diff options
| -rw-r--r-- | changelog.d/3902.feature | 1 | ||||
| -rw-r--r-- | changelog.d/4513.misc | 1 | ||||
| -rw-r--r-- | changelog.d/4530.bugfix | 1 | ||||
| -rw-r--r-- | changelog.d/4539.misc | 1 | ||||
| -rw-r--r-- | changelog.d/4542.misc | 1 | ||||
| -rw-r--r-- | changelog.d/4544.misc | 1 | ||||
| -rw-r--r-- | synapse/api/constants.py | 1 | ||||
| -rw-r--r-- | synapse/config/api.py | 2 | ||||
| -rw-r--r-- | synapse/federation/federation_server.py | 16 | ||||
| -rw-r--r-- | synapse/handlers/room.py | 10 | ||||
| -rw-r--r-- | synapse/http/federation/matrix_federation_agent.py | 90 | ||||
| -rw-r--r-- | synapse/storage/state.py | 38 | ||||
| -rw-r--r-- | tests/http/federation/test_matrix_federation_agent.py | 130 |
13 files changed, 211 insertions, 82 deletions
diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature new file mode 100644 index 0000000000..eb8d9f2393 --- /dev/null +++ b/changelog.d/3902.feature @@ -0,0 +1 @@ +Include m.room.encryption on invites by default diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc new file mode 100644 index 0000000000..1f64a96465 --- /dev/null +++ b/changelog.d/4513.misc @@ -0,0 +1 @@ +Reject federation transactions if they include more than 50 PDUs or 100 EDUs. \ No newline at end of file diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix new file mode 100644 index 0000000000..d010af927e --- /dev/null +++ b/changelog.d/4530.bugfix @@ -0,0 +1 @@ +Copy over room federation ability on room upgrade. \ No newline at end of file diff --git a/changelog.d/4539.misc b/changelog.d/4539.misc new file mode 100644 index 0000000000..b222c8d23f --- /dev/null +++ b/changelog.d/4539.misc @@ -0,0 +1 @@ +Update federation routing logic to check .well-known before SRV diff --git a/changelog.d/4542.misc b/changelog.d/4542.misc new file mode 100644 index 0000000000..74c84e0209 --- /dev/null +++ b/changelog.d/4542.misc @@ -0,0 +1 @@ +Improve performance of handling servers with invalid .well-known diff --git a/changelog.d/4544.misc b/changelog.d/4544.misc new file mode 100644 index 0000000000..b29fc8575c --- /dev/null +++ b/changelog.d/4544.misc @@ -0,0 +1 @@ +Treat an invalid .well-known file the same as an absent one \ No newline at end of file diff --git a/synapse/api/constants.py b/synapse/api/constants.py index fedfb92b3e..f47c33a074 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -73,6 +73,7 @@ class EventTypes(object): RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" RoomAvatar = "m.room.avatar" + RoomEncryption = "m.room.encryption" GuestAccess = "m.room.guest_access" # These are used for validation diff --git a/synapse/config/api.py b/synapse/config/api.py index 403d96ba76..9f25bbc5cb 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -24,6 +24,7 @@ class ApiConfig(Config): EventTypes.JoinRules, EventTypes.CanonicalAlias, EventTypes.RoomAvatar, + EventTypes.RoomEncryption, EventTypes.Name, ]) @@ -36,5 +37,6 @@ class ApiConfig(Config): - "{JoinRules}" - "{CanonicalAlias}" - "{RoomAvatar}" + - "{RoomEncryption}" - "{Name}" """.format(**vars(EventTypes)) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index aeadc9c564..3da86d4ba6 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -148,6 +148,22 @@ class FederationServer(FederationBase): logger.debug("[%s] Transaction is new", transaction.transaction_id) + # Reject if PDU count > 50 and EDU count > 100 + if (len(transaction.pdus) > 50 + or (hasattr(transaction, "edus") and len(transaction.edus) > 100)): + + logger.info( + "Transaction PDU or EDU count too large. Returning 400", + ) + + response = {} + yield self.transaction_actions.set_response( + origin, + transaction, + 400, response + ) + defer.returnValue((400, response)) + received_pdus_counter.inc(len(transaction.pdus)) origin_host, _ = parse_server_name(origin) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13ba9291b0..5e40e9ea46 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -263,6 +263,16 @@ class RoomCreationHandler(BaseHandler): } } + # Check if old room was non-federatable + + # Get old room's create event + old_room_create_event = yield self.store.get_create_event_for_room(old_room_id) + + # Check if the create event specified a non-federatable room + if not old_room_create_event.content.get("m.federate", True): + # If so, mark the new room as non-federatable as well + creation_content["m.federate"] = False + initial_state = dict() # Replicate relevant room events diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 26649e70be..384d8a37a2 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -23,14 +23,17 @@ from zope.interface import implementer from twisted.internet import defer from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS +from twisted.internet.interfaces import IStreamClientEndpoint from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, readBody from twisted.web.http import stringToDatetime from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent from synapse.http.federation.srv_resolver import SrvResolver, pick_server_from_list +from synapse.util import Clock from synapse.util.caches.ttlcache import TTLCache from synapse.util.logcontext import make_deferred_yieldable +from synapse.util.metrics import Measure # period to cache .well-known results for by default WELL_KNOWN_DEFAULT_CACHE_PERIOD = 24 * 3600 @@ -44,7 +47,6 @@ WELL_KNOWN_INVALID_CACHE_PERIOD = 1 * 3600 # cap for .well-known cache period WELL_KNOWN_MAX_CACHE_PERIOD = 48 * 3600 - logger = logging.getLogger(__name__) well_known_cache = TTLCache('well-known') @@ -78,6 +80,8 @@ class MatrixFederationAgent(object): _well_known_cache=well_known_cache, ): self._reactor = reactor + self._clock = Clock(reactor) + self._tls_client_options_factory = tls_client_options_factory if _srv_resolver is None: _srv_resolver = SrvResolver() @@ -98,6 +102,10 @@ class MatrixFederationAgent(object): ) self._well_known_agent = _well_known_agent + # our cache of .well-known lookup results, mapping from server name + # to delegated name. The values can be: + # `bytes`: a valid server-name + # `None`: there is no (valid) .well-known here self._well_known_cache = _well_known_cache @defer.inlineCallbacks @@ -152,12 +160,9 @@ class MatrixFederationAgent(object): class EndpointFactory(object): @staticmethod def endpointForURI(_uri): - logger.info( - "Connecting to %s:%i", - res.target_host.decode("ascii"), - res.target_port, + ep = LoggingHostnameEndpoint( + self._reactor, res.target_host, res.target_port, ) - ep = HostnameEndpoint(self._reactor, res.target_host, res.target_port) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep @@ -210,11 +215,7 @@ class MatrixFederationAgent(object): target_port=parsed_uri.port, )) - # try a SRV lookup - service_name = b"_matrix._tcp.%s" % (parsed_uri.host,) - server_list = yield self._srv_resolver.resolve_service(service_name) - - if not server_list and lookup_well_known: + if lookup_well_known: # try a .well-known lookup well_known_server = yield self._get_well_known(parsed_uri.host) @@ -250,6 +251,10 @@ class MatrixFederationAgent(object): res = yield self._route_matrix_uri(new_uri, lookup_well_known=False) defer.returnValue(res) + # try a SRV lookup + service_name = b"_matrix._tcp.%s" % (parsed_uri.host,) + server_list = yield self._srv_resolver.resolve_service(service_name) + if not server_list: target_host = parsed_uri.host port = 8448 @@ -283,14 +288,32 @@ class MatrixFederationAgent(object): None if there was no .well-known file. """ try: - cached = self._well_known_cache[server_name] - defer.returnValue(cached) + result = self._well_known_cache[server_name] except KeyError: - pass + # TODO: should we linearise so that we don't end up doing two .well-known + # requests for the same server in parallel? + with Measure(self._clock, "get_well_known"): + result, cache_period = yield self._do_get_well_known(server_name) + + if cache_period > 0: + self._well_known_cache.set(server_name, result, cache_period) - # TODO: should we linearise so that we don't end up doing two .well-known requests - # for the same server in parallel? + defer.returnValue(result) + + @defer.inlineCallbacks + def _do_get_well_known(self, server_name): + """Actually fetch and parse a .well-known, without checking the cache + Args: + server_name (bytes): name of the server, from the requested url + + Returns: + Deferred[Tuple[bytes|None|object],int]: + result, cache period, where result is one of: + - the new server name from the .well-known (as a `bytes`) + - None if there was no .well-known file. + - INVALID_WELL_KNOWN if the .well-known was invalid + """ uri = b"https://%s/.well-known/matrix/server" % (server_name, ) uri_str = uri.decode("ascii") logger.info("Fetching %s", uri_str) @@ -301,18 +324,7 @@ class MatrixFederationAgent(object): body = yield make_deferred_yieldable(readBody(response)) if response.code != 200: raise Exception("Non-200 response %s" % (response.code, )) - except Exception as e: - logger.info("Error fetching %s: %s", uri_str, e) - - # add some randomness to the TTL to avoid a stampeding herd every hour - # after startup - cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD - cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER) - self._well_known_cache.set(server_name, None, cache_period) - defer.returnValue(None) - - try: parsed_body = json.loads(body.decode('utf-8')) logger.info("Response from .well-known: %s", parsed_body) if not isinstance(parsed_body, dict): @@ -320,7 +332,13 @@ class MatrixFederationAgent(object): if "m.server" not in parsed_body: raise Exception("Missing key 'm.server'") except Exception as e: - raise Exception("invalid .well-known response from %s: %s" % (uri_str, e,)) + logger.info("Error fetching %s: %s", uri_str, e) + + # add some randomness to the TTL to avoid a stampeding herd every hour + # after startup + cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD + cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER) + defer.returnValue((None, cache_period)) result = parsed_body["m.server"].encode("ascii") @@ -336,10 +354,20 @@ class MatrixFederationAgent(object): else: cache_period = min(cache_period, WELL_KNOWN_MAX_CACHE_PERIOD) - if cache_period > 0: - self._well_known_cache.set(server_name, result, cache_period) + defer.returnValue((result, cache_period)) - defer.returnValue(result) + +@implementer(IStreamClientEndpoint) +class LoggingHostnameEndpoint(object): + """A wrapper for HostnameEndpint which logs when it connects""" + def __init__(self, reactor, host, port, *args, **kwargs): + self.host = host + self.port = port + self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs) + + def connect(self, protocol_factory): + logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port) + return self.ep.connect(protocol_factory) def _cache_period_from_headers(headers, time_now=time.time): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index c3ab7db7ae..d14a7b2538 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -428,13 +428,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): """ # for now we do this by looking at the create event. We may want to cache this # more intelligently in future. - state_ids = yield self.get_current_state_ids(room_id) - create_id = state_ids.get((EventTypes.Create, "")) - - if not create_id: - raise NotFoundError("Unknown room %s" % (room_id)) - create_event = yield self.get_event(create_id) + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) defer.returnValue(create_event.content.get("room_version", "1")) @defer.inlineCallbacks @@ -447,19 +443,39 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[unicode|None]: predecessor room id + + Raises: + NotFoundError if the room is unknown + """ + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) + + # Return predecessor if present + defer.returnValue(create_event.content.get("predecessor", None)) + + @defer.inlineCallbacks + def get_create_event_for_room(self, room_id): + """Get the create state event for a room. + + Args: + room_id (str) + + Returns: + Deferred[EventBase]: The room creation event. + + Raises: + NotFoundError if the room is unknown """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) # If we can't find the create event, assume we've hit a dead end if not create_id: - defer.returnValue(None) + raise NotFoundError("Unknown room %s" % (room_id)) - # Retrieve the room's create event + # Retrieve the room's create event and return create_event = yield self.get_event(create_id) - - # Return predecessor if present - defer.returnValue(create_event.content.get("predecessor", None)) + defer.returnValue(create_event) @cached(max_entries=100000, iterable=True) def get_current_state_ids(self, room_id): diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 7b2800021f..dcf184d3cf 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -124,7 +124,7 @@ class MatrixFederationAgentTests(TestCase): _check_logcontext(context) def _handle_well_known_connection( - self, client_factory, expected_sni, target_server, response_headers={}, + self, client_factory, expected_sni, content, response_headers={}, ): """Handle an outgoing HTTPs connection: wire it up to a server, check that the request is for a .well-known, and send the response. @@ -132,8 +132,7 @@ class MatrixFederationAgentTests(TestCase): Args: client_factory (IProtocolFactory): outgoing connection expected_sni (bytes): SNI that we expect the outgoing connection to send - target_server (bytes): target server that we should redirect to in the - .well-known response. + content (bytes): content to send back as the .well-known Returns: HTTPChannel: server impl """ @@ -145,10 +144,10 @@ class MatrixFederationAgentTests(TestCase): # check the .well-known request and send a response self.assertEqual(len(well_known_server.requests), 1) request = well_known_server.requests[0] - self._send_well_known_response(request, target_server, headers=response_headers) + self._send_well_known_response(request, content, headers=response_headers) return well_known_server - def _send_well_known_response(self, request, target_server, headers={}): + def _send_well_known_response(self, request, content, headers={}): """Check that an incoming request looks like a valid .well-known request, and send back the response. """ @@ -161,7 +160,7 @@ class MatrixFederationAgentTests(TestCase): # send back a response for k, v in headers.items(): request.setHeader(k, v) - request.write(b'{ "m.server": "%s" }' % (target_server,)) + request.write(content) request.finish() self.reactor.pump((0.1, )) @@ -358,9 +357,8 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.testserv", - ) + # No SRV record lookup yet + self.mock_resolver.resolve_service.assert_not_called() # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients @@ -376,6 +374,11 @@ class MatrixFederationAgentTests(TestCase): # .well-known request fails. self.reactor.pump((0.4,)) + # now there should be a SRV lookup + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.testserv", + ) + # we should fall back to a direct connection self.assertEqual(len(clients), 2) (host, port, client_factory, _timeout, _bindAddress) = clients[1] @@ -403,8 +406,7 @@ class MatrixFederationAgentTests(TestCase): self.successResultOf(test_d) def test_get_well_known(self): - """Test the behaviour when the server name has no port and no SRV record, but - the .well-known redirects elsewhere + """Test the behaviour when the .well-known delegates elsewhere """ self.mock_resolver.resolve_service.side_effect = lambda _: [] @@ -416,11 +418,6 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.testserv", - ) - self.mock_resolver.resolve_service.reset_mock() - # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) @@ -429,10 +426,11 @@ class MatrixFederationAgentTests(TestCase): self.assertEqual(port, 443) self._handle_well_known_connection( - client_factory, expected_sni=b"testserv", target_server=b"target-server", + client_factory, expected_sni=b"testserv", + content=b'{ "m.server": "target-server" }', ) - # there should be another SRV lookup + # there should be a SRV lookup self.mock_resolver.resolve_service.assert_called_once_with( b"_matrix._tcp.target-server", ) @@ -483,11 +481,6 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.testserv", - ) - self.mock_resolver.resolve_service.reset_mock() - # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) @@ -529,7 +522,7 @@ class MatrixFederationAgentTests(TestCase): self.reactor.pump((0.1, )) - # there should be another SRV lookup + # there should be a SRV lookup self.mock_resolver.resolve_service.assert_called_once_with( b"_matrix._tcp.target-server", ) @@ -567,6 +560,64 @@ class MatrixFederationAgentTests(TestCase): self.well_known_cache.expire() self.assertNotIn(b"testserv", self.well_known_cache) + def test_get_invalid_well_known(self): + """ + Test the behaviour when the server name has an *invalid* well-known (and no SRV) + """ + + self.mock_resolver.resolve_service.side_effect = lambda _: [] + self.reactor.lookups["testserv"] = "1.2.3.4" + + test_d = self._make_get_request(b"matrix://testserv/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + # No SRV record lookup yet + self.mock_resolver.resolve_service.assert_not_called() + + # there should be an attempt to connect on port 443 for the .well-known + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients.pop() + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 443) + + self._handle_well_known_connection( + client_factory, expected_sni=b"testserv", content=b'NOT JSON', + ) + + # now there should be a SRV lookup + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.testserv", + ) + + # we should fall back to a direct connection + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients.pop() + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 8448) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'testserv', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'testserv'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + def test_get_hostname_srv(self): """ Test the behaviour when there is a single SRV record @@ -581,6 +632,7 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) + # the request for a .well-known will have failed with a DNS lookup error. self.mock_resolver.resolve_service.assert_called_once_with( b"_matrix._tcp.testserv", ) @@ -613,11 +665,9 @@ class MatrixFederationAgentTests(TestCase): self.successResultOf(test_d) def test_get_well_known_srv(self): - """Test the behaviour when the server name has no port and no SRV record, but - the .well-known redirects to a place where there is a SRV. + """Test the behaviour when the .well-known redirects to a place where there + is a SRV. """ - - self.mock_resolver.resolve_service.side_effect = lambda _: [] self.reactor.lookups["testserv"] = "1.2.3.4" self.reactor.lookups["srvtarget"] = "5.6.7.8" @@ -626,11 +676,6 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.testserv", - ) - self.mock_resolver.resolve_service.reset_mock() - # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) @@ -643,10 +688,11 @@ class MatrixFederationAgentTests(TestCase): ] self._handle_well_known_connection( - client_factory, expected_sni=b"testserv", target_server=b"target-server", + client_factory, expected_sni=b"testserv", + content=b'{ "m.server": "target-server" }', ) - # there should be another SRV lookup + # there should be a SRV lookup self.mock_resolver.resolve_service.assert_called_once_with( b"_matrix._tcp.target-server", ) @@ -691,9 +737,8 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.xn--bcher-kva.com", - ) + # No SRV record lookup yet + self.mock_resolver.resolve_service.assert_not_called() # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients @@ -709,6 +754,11 @@ class MatrixFederationAgentTests(TestCase): # .well-known request fails. self.reactor.pump((0.4,)) + # now there should have been a SRV lookup + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.xn--bcher-kva.com", + ) + # We should fall back to port 8448 clients = self.reactor.tcpClients self.assertEqual(len(clients), 2) @@ -806,7 +856,7 @@ class MatrixFederationAgentTests(TestCase): client_factory, expected_sni=b"testserv", response_headers={b'Cache-Control': b'max-age=10'}, - target_server=b"target-server", + content=b'{ "m.server": "target-server" }', ) r = self.successResultOf(fetch_d) @@ -834,7 +884,7 @@ class MatrixFederationAgentTests(TestCase): self._handle_well_known_connection( client_factory, expected_sni=b"testserv", - target_server=b"other-server", + content=b'{ "m.server": "other-server" }', ) r = self.successResultOf(fetch_d) |