1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
|
<!doctype html>
<html class="no-js" lang="en">
<head><meta charset="utf-8"/>
<meta name="viewport" content="width=device-width,initial-scale=1"/>
<meta name="color-scheme" content="light dark"><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
<link rel="index" title="Index" href="../genindex.html" /><link rel="search" title="Search" href="../search.html" /><link rel="prev" title="Welcome to the Synapse Developer Documentation!" href="../index.html" />
<!-- Generated with Sphinx 6.2.1 and Furo 2023.07.26 -->
<title>Federation Sender - Synapse development documentation</title>
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../_static/styles/furo.css?digest=369552022d0b975c8e74270ce6eabe0fb7978f24" />
<link rel="stylesheet" type="text/css" href="../_static/styles/furo-extensions.css?digest=30d1aed668e5c3a91c3e3bf6a60b675221979f0e" />
<style>
body {
--color-code-background: #f8f8f8;
--color-code-foreground: black;
}
@media not print {
body[data-theme="dark"] {
--color-code-background: #202020;
--color-code-foreground: #d0d0d0;
}
@media (prefers-color-scheme: dark) {
body:not([data-theme="light"]) {
--color-code-background: #202020;
--color-code-foreground: #d0d0d0;
}
}
}
</style></head>
<body>
<script>
document.body.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
<symbol id="svg-toc" viewBox="0 0 24 24">
<title>Contents</title>
<svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
<path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
</svg>
</symbol>
<symbol id="svg-menu" viewBox="0 0 24 24">
<title>Menu</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
<line x1="3" y1="12" x2="21" y2="12"></line>
<line x1="3" y1="6" x2="21" y2="6"></line>
<line x1="3" y1="18" x2="21" y2="18"></line>
</svg>
</symbol>
<symbol id="svg-arrow-right" viewBox="0 0 24 24">
<title>Expand</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
<polyline points="9 18 15 12 9 6"></polyline>
</svg>
</symbol>
<symbol id="svg-sun" viewBox="0 0 24 24">
<title>Light mode</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
<circle cx="12" cy="12" r="5"></circle>
<line x1="12" y1="1" x2="12" y2="3"></line>
<line x1="12" y1="21" x2="12" y2="23"></line>
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
<line x1="1" y1="12" x2="3" y2="12"></line>
<line x1="21" y1="12" x2="23" y2="12"></line>
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
</svg>
</symbol>
<symbol id="svg-moon" viewBox="0 0 24 24">
<title>Dark mode</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
</svg>
</symbol>
<symbol id="svg-sun-half" viewBox="0 0 24 24">
<title>Auto light/dark mode</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-shadow">
<path stroke="none" d="M0 0h24v24H0z" fill="none"/>
<circle cx="12" cy="12" r="9" />
<path d="M13 12h5" />
<path d="M13 15h4" />
<path d="M13 18h1" />
<path d="M13 9h4" />
<path d="M13 6h1" />
</svg>
</symbol>
</svg>
<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation">
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc">
<label class="overlay sidebar-overlay" for="__navigation">
<div class="visually-hidden">Hide navigation sidebar</div>
</label>
<label class="overlay toc-overlay" for="__toc">
<div class="visually-hidden">Hide table of contents sidebar</div>
</label>
<div class="page">
<header class="mobile-header">
<div class="header-left">
<label class="nav-overlay-icon" for="__navigation">
<div class="visually-hidden">Toggle site navigation sidebar</div>
<i class="icon"><svg><use href="#svg-menu"></use></svg></i>
</label>
</div>
<div class="header-center">
<a href="../index.html"><div class="brand">Synapse development documentation</div></a>
</div>
<div class="header-right">
<div class="theme-toggle-container theme-toggle-header">
<button class="theme-toggle">
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
</button>
</div>
<label class="toc-overlay-icon toc-header-icon no-toc" for="__toc">
<div class="visually-hidden">Toggle table of contents sidebar</div>
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
</label>
</div>
</header>
<aside class="sidebar-drawer">
<div class="sidebar-container">
<div class="sidebar-sticky"><a class="sidebar-brand" href="../index.html">
<span class="sidebar-brand-text">Synapse development documentation</span>
</a><form class="sidebar-search-container" method="get" action="../search.html" role="search">
<input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
<input type="hidden" name="check_keywords" value="yes">
<input type="hidden" name="area" value="default">
</form>
<div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1 current current-page"><a class="current reference internal" href="#">Federation Sender</a></li>
</ul>
</div>
</div>
</div>
</div>
</aside>
<div class="main">
<div class="content">
<div class="article-container">
<a href="#" class="back-to-top muted-link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
<path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
</svg>
<span>Back to top</span>
</a>
<div class="content-icon-container">
<div class="theme-toggle-container theme-toggle-content">
<button class="theme-toggle">
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
</button>
</div>
<label class="toc-overlay-icon toc-content-icon no-toc" for="__toc">
<div class="visually-hidden">Toggle table of contents sidebar</div>
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
</label>
</div>
<article role="main">
<section id="federation-sender">
<h1>Federation Sender<a class="headerlink" href="#federation-sender" title="Permalink to this heading">#</a></h1>
<p>The Federation Sender is responsible for sending Persistent Data Units (PDUs)
and Ephemeral Data Units (EDUs) to other homeservers using
the <code class="docutils literal notranslate"><span class="pre">/send</span></code> Federation API.</p>
<h2 class="rubric" id="how-do-pdus-get-sent">How do PDUs get sent?</h2>
<p>The Federation Sender is made aware of new PDUs due to <code class="docutils literal notranslate"><span class="pre">FederationSender.notify_new_events</span></code>.
When the sender is notified about a newly-persisted PDU that originates from this homeserver
and is not an out-of-band event, we pass the PDU to the <code class="docutils literal notranslate"><span class="pre">_PerDestinationQueue</span></code> for each
remote homeserver that is in the room at that point in the DAG.</p>
<h3 class="rubric" id="per-destination-queues">Per-Destination Queues</h3>
<p>There is one <code class="docutils literal notranslate"><span class="pre">PerDestinationQueue</span></code> per ‘destination’ homeserver.
The <code class="docutils literal notranslate"><span class="pre">PerDestinationQueue</span></code> maintains the following information about the destination:</p>
<ul class="simple">
<li><p>whether the destination is currently in <a class="reference internal" href="#catch-up-mode"><span class="xref myst">catch-up mode (see below)</span></a>;</p></li>
<li><p>a queue of PDUs to be sent to the destination; and</p></li>
<li><p>a queue of EDUs to be sent to the destination (not considered in this section).</p></li>
</ul>
<p>Upon a new PDU being enqueued, <code class="docutils literal notranslate"><span class="pre">attempt_new_transaction</span></code> is called to start a new
transaction if there is not already one in progress.</p>
<h3 class="rubric" id="transactions-and-the-transaction-transmission-loop">Transactions and the Transaction Transmission Loop</h3>
<p>Each federation HTTP request to the <code class="docutils literal notranslate"><span class="pre">/send</span></code> endpoint is referred to as a ‘transaction’.
The body of the HTTP request contains a list of PDUs and EDUs to send to the destination.</p>
<p>The <em>Transaction Transmission Loop</em> (<code class="docutils literal notranslate"><span class="pre">_transaction_transmission_loop</span></code>) is responsible
for emptying the queued PDUs (and EDUs) from a <code class="docutils literal notranslate"><span class="pre">PerDestinationQueue</span></code> by sending
them to the destination.</p>
<p>There can only be one transaction in flight for a given destination at any time.
(Other than preventing us from overloading the destination, this also makes it easier to
reason about because we process events sequentially for each destination.
This is useful for <em>Catch-Up Mode</em>, described later.)</p>
<p>The loop continues so long as there is anything to send. At each iteration of the loop, we:</p>
<ul class="simple">
<li><p>dequeue up to 50 PDUs (and up to 100 EDUs).</p></li>
<li><p>make the <code class="docutils literal notranslate"><span class="pre">/send</span></code> request to the destination homeserver with the dequeued PDUs and EDUs.</p></li>
<li><p>if successful, make note of the fact that we succeeded in transmitting PDUs up to
the given <code class="docutils literal notranslate"><span class="pre">stream_ordering</span></code> of the latest PDU by</p></li>
<li><p>if unsuccessful, back off from the remote homeserver for some time.
If we have been unsuccessful for too long (when the backoff interval grows to exceed 1 hour),
the in-memory queues are emptied and we enter <a class="reference internal" href="#catch-up-mode"><span class="xref myst"><em>Catch-Up Mode</em>, described below</span></a>.</p></li>
</ul>
<h3 class="rubric" id="catch-up-mode">Catch-Up Mode</h3>
<p>When the <code class="docutils literal notranslate"><span class="pre">PerDestinationQueue</span></code> has the catch-up flag set, the <em>Catch-Up Transmission Loop</em>
(<code class="docutils literal notranslate"><span class="pre">_catch_up_transmission_loop</span></code>) is used in lieu of the regular <code class="docutils literal notranslate"><span class="pre">_transaction_transmission_loop</span></code>.
(Only once the catch-up mode has been exited can the regular tranaction transmission behaviour
be resumed.)</p>
<p><em>Catch-Up Mode</em>, entered upon Synapse startup or once a homeserver has fallen behind due to
connection problems, is responsible for sending PDUs that have been missed by the destination
homeserver. (PDUs can be missed because the <code class="docutils literal notranslate"><span class="pre">PerDestinationQueue</span></code> is volatile — i.e. resets
on startup — and it does not hold PDUs forever if <code class="docutils literal notranslate"><span class="pre">/send</span></code> requests to the destination fail.)</p>
<p>The catch-up mechanism makes use of the <code class="docutils literal notranslate"><span class="pre">last_successful_stream_ordering</span></code> column in the
<code class="docutils literal notranslate"><span class="pre">destinations</span></code> table (which gives the <code class="docutils literal notranslate"><span class="pre">stream_ordering</span></code> of the most recent successfully
sent PDU) and the <code class="docutils literal notranslate"><span class="pre">stream_ordering</span></code> column in the <code class="docutils literal notranslate"><span class="pre">destination_rooms</span></code> table (which gives,
for each room, the <code class="docutils literal notranslate"><span class="pre">stream_ordering</span></code> of the most recent PDU that needs to be sent to this
destination).</p>
<p>Each iteration of the loop pulls out 50 <code class="docutils literal notranslate"><span class="pre">destination_rooms</span></code> entries with the oldest
<code class="docutils literal notranslate"><span class="pre">stream_ordering</span></code>s that are greater than the <code class="docutils literal notranslate"><span class="pre">last_successful_stream_ordering</span></code>.
In other words, from the set of latest PDUs in each room to be sent to the destination,
the 50 oldest such PDUs are pulled out.</p>
<p>These PDUs could, in principle, now be directly sent to the destination. However, as an
optimisation intended to prevent overloading destination homeservers, we instead attempt
to send the latest forward extremities so long as the destination homeserver is still
eligible to receive those.
This reduces load on the destination <strong>in aggregate</strong> because all Synapse homeservers
will behave according to this principle and therefore avoid sending lots of different PDUs
at different points in the DAG to a recovering homeserver.
<em>This optimisation is not currently valid in rooms which are partial-state on this homeserver,
since we are unable to determine whether the destination homeserver is eligible to receive
the latest forward extremities unless this homeserver sent those PDUs — in this case, we
just send the latest PDUs originating from this server and skip this optimisation.</em></p>
<p>Whilst PDUs are sent through this mechanism, the position of <code class="docutils literal notranslate"><span class="pre">last_successful_stream_ordering</span></code>
is advanced as normal.
Once there are no longer any rooms containing outstanding PDUs to be sent to the destination
<em>that are not already in the <code class="docutils literal notranslate"><span class="pre">PerDestinationQueue</span></code> because they arrived since Catch-Up Mode
was enabled</em>, Catch-Up Mode is exited and we return to <code class="docutils literal notranslate"><span class="pre">_transaction_transmission_loop</span></code>.</p>
<h4 class="rubric" id="a-note-on-failures-and-back-offs">A note on failures and back-offs</h4>
<p>If a remote server is unreachable over federation, we back off from that server,
with an exponentially-increasing retry interval.
We automatically retry after the retry interval expires (roughly, the logic to do so
being triggered every minute).</p>
<p>If the backoff grows too large (> 1 hour), the in-memory queue is emptied (to prevent
unbounded growth) and Catch-Up Mode is entered.</p>
<p>It is worth noting that the back-off for a remote server is cleared once an inbound
request from that remote server is received (see <code class="docutils literal notranslate"><span class="pre">notify_remote_server_up</span></code>).
At this point, the transaction transmission loop is also started up, to proactively
send missed PDUs and EDUs to the destination (i.e. you don’t need to wait for a new PDU
or EDU, destined for that destination, to be created in order to send out missed PDUs and
EDUs).</p>
</section>
</article>
</div>
<footer>
<div class="related-pages">
<a class="prev-page" href="../index.html">
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
<div class="page-info">
<div class="context">
<span>Previous</span>
</div>
<div class="title">Home</div>
</div>
</a>
</div>
<div class="bottom-of-page">
<div class="left-details">
<div class="copyright">
Copyright © 2023, The Matrix.org Foundation C.I.C.
</div>
Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
<a href="https://github.com/pradyunsg/furo">Furo</a>
</div>
<div class="right-details">
</div>
</div>
</footer>
</div>
<aside class="toc-drawer no-toc">
</aside>
</div>
</div><script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
<script src="../_static/doctools.js"></script>
<script src="../_static/sphinx_highlight.js"></script>
<script src="../_static/scripts/furo.js"></script>
</body>
</html>
|
