diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature
new file mode 100644
index 0000000000..eb8d9f2393
--- /dev/null
+++ b/changelog.d/3902.feature
@@ -0,0 +1 @@
+Include m.room.encryption on invites by default
diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc
new file mode 100644
index 0000000000..1f64a96465
--- /dev/null
+++ b/changelog.d/4513.misc
@@ -0,0 +1 @@
+Reject federation transactions if they include more than 50 PDUs or 100 EDUs.
\ No newline at end of file
diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix
new file mode 100644
index 0000000000..d010af927e
--- /dev/null
+++ b/changelog.d/4530.bugfix
@@ -0,0 +1 @@
+Copy over room federation ability on room upgrade.
\ No newline at end of file
diff --git a/changelog.d/4539.misc b/changelog.d/4539.misc
new file mode 100644
index 0000000000..b222c8d23f
--- /dev/null
+++ b/changelog.d/4539.misc
@@ -0,0 +1 @@
+Update federation routing logic to check .well-known before SRV
diff --git a/changelog.d/4542.misc b/changelog.d/4542.misc
new file mode 100644
index 0000000000..74c84e0209
--- /dev/null
+++ b/changelog.d/4542.misc
@@ -0,0 +1 @@
+Improve performance of handling servers with invalid .well-known
diff --git a/changelog.d/4544.misc b/changelog.d/4544.misc
new file mode 100644
index 0000000000..b29fc8575c
--- /dev/null
+++ b/changelog.d/4544.misc
@@ -0,0 +1 @@
+Treat an invalid .well-known file the same as an absent one
\ No newline at end of file
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index fedfb92b3e..f47c33a074 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -73,6 +73,7 @@ class EventTypes(object):
RoomHistoryVisibility = "m.room.history_visibility"
CanonicalAlias = "m.room.canonical_alias"
RoomAvatar = "m.room.avatar"
+ RoomEncryption = "m.room.encryption"
GuestAccess = "m.room.guest_access"
# These are used for validation
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 403d96ba76..9f25bbc5cb 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -24,6 +24,7 @@ class ApiConfig(Config):
EventTypes.JoinRules,
EventTypes.CanonicalAlias,
EventTypes.RoomAvatar,
+ EventTypes.RoomEncryption,
EventTypes.Name,
])
@@ -36,5 +37,6 @@ class ApiConfig(Config):
- "{JoinRules}"
- "{CanonicalAlias}"
- "{RoomAvatar}"
+ - "{RoomEncryption}"
- "{Name}"
""".format(**vars(EventTypes))
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index aeadc9c564..3da86d4ba6 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -148,6 +148,22 @@ class FederationServer(FederationBase):
logger.debug("[%s] Transaction is new", transaction.transaction_id)
+ # Reject if PDU count > 50 and EDU count > 100
+ if (len(transaction.pdus) > 50
+ or (hasattr(transaction, "edus") and len(transaction.edus) > 100)):
+
+ logger.info(
+ "Transaction PDU or EDU count too large. Returning 400",
+ )
+
+ response = {}
+ yield self.transaction_actions.set_response(
+ origin,
+ transaction,
+ 400, response
+ )
+ defer.returnValue((400, response))
+
received_pdus_counter.inc(len(transaction.pdus))
origin_host, _ = parse_server_name(origin)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 13ba9291b0..5e40e9ea46 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -263,6 +263,16 @@ class RoomCreationHandler(BaseHandler):
}
}
+ # Check if old room was non-federatable
+
+ # Get old room's create event
+ old_room_create_event = yield self.store.get_create_event_for_room(old_room_id)
+
+ # Check if the create event specified a non-federatable room
+ if not old_room_create_event.content.get("m.federate", True):
+ # If so, mark the new room as non-federatable as well
+ creation_content["m.federate"] = False
+
initial_state = dict()
# Replicate relevant room events
diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py
index 26649e70be..384d8a37a2 100644
--- a/synapse/http/federation/matrix_federation_agent.py
+++ b/synapse/http/federation/matrix_federation_agent.py
@@ -23,14 +23,17 @@ from zope.interface import implementer
from twisted.internet import defer
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
+from twisted.internet.interfaces import IStreamClientEndpoint
from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, readBody
from twisted.web.http import stringToDatetime
from twisted.web.http_headers import Headers
from twisted.web.iweb import IAgent
from synapse.http.federation.srv_resolver import SrvResolver, pick_server_from_list
+from synapse.util import Clock
from synapse.util.caches.ttlcache import TTLCache
from synapse.util.logcontext import make_deferred_yieldable
+from synapse.util.metrics import Measure
# period to cache .well-known results for by default
WELL_KNOWN_DEFAULT_CACHE_PERIOD = 24 * 3600
@@ -44,7 +47,6 @@ WELL_KNOWN_INVALID_CACHE_PERIOD = 1 * 3600
# cap for .well-known cache period
WELL_KNOWN_MAX_CACHE_PERIOD = 48 * 3600
-
logger = logging.getLogger(__name__)
well_known_cache = TTLCache('well-known')
@@ -78,6 +80,8 @@ class MatrixFederationAgent(object):
_well_known_cache=well_known_cache,
):
self._reactor = reactor
+ self._clock = Clock(reactor)
+
self._tls_client_options_factory = tls_client_options_factory
if _srv_resolver is None:
_srv_resolver = SrvResolver()
@@ -98,6 +102,10 @@ class MatrixFederationAgent(object):
)
self._well_known_agent = _well_known_agent
+ # our cache of .well-known lookup results, mapping from server name
+ # to delegated name. The values can be:
+ # `bytes`: a valid server-name
+ # `None`: there is no (valid) .well-known here
self._well_known_cache = _well_known_cache
@defer.inlineCallbacks
@@ -152,12 +160,9 @@ class MatrixFederationAgent(object):
class EndpointFactory(object):
@staticmethod
def endpointForURI(_uri):
- logger.info(
- "Connecting to %s:%i",
- res.target_host.decode("ascii"),
- res.target_port,
+ ep = LoggingHostnameEndpoint(
+ self._reactor, res.target_host, res.target_port,
)
- ep = HostnameEndpoint(self._reactor, res.target_host, res.target_port)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
@@ -210,11 +215,7 @@ class MatrixFederationAgent(object):
target_port=parsed_uri.port,
))
- # try a SRV lookup
- service_name = b"_matrix._tcp.%s" % (parsed_uri.host,)
- server_list = yield self._srv_resolver.resolve_service(service_name)
-
- if not server_list and lookup_well_known:
+ if lookup_well_known:
# try a .well-known lookup
well_known_server = yield self._get_well_known(parsed_uri.host)
@@ -250,6 +251,10 @@ class MatrixFederationAgent(object):
res = yield self._route_matrix_uri(new_uri, lookup_well_known=False)
defer.returnValue(res)
+ # try a SRV lookup
+ service_name = b"_matrix._tcp.%s" % (parsed_uri.host,)
+ server_list = yield self._srv_resolver.resolve_service(service_name)
+
if not server_list:
target_host = parsed_uri.host
port = 8448
@@ -283,14 +288,32 @@ class MatrixFederationAgent(object):
None if there was no .well-known file.
"""
try:
- cached = self._well_known_cache[server_name]
- defer.returnValue(cached)
+ result = self._well_known_cache[server_name]
except KeyError:
- pass
+ # TODO: should we linearise so that we don't end up doing two .well-known
+ # requests for the same server in parallel?
+ with Measure(self._clock, "get_well_known"):
+ result, cache_period = yield self._do_get_well_known(server_name)
+
+ if cache_period > 0:
+ self._well_known_cache.set(server_name, result, cache_period)
- # TODO: should we linearise so that we don't end up doing two .well-known requests
- # for the same server in parallel?
+ defer.returnValue(result)
+
+ @defer.inlineCallbacks
+ def _do_get_well_known(self, server_name):
+ """Actually fetch and parse a .well-known, without checking the cache
+ Args:
+ server_name (bytes): name of the server, from the requested url
+
+ Returns:
+ Deferred[Tuple[bytes|None|object],int]:
+ result, cache period, where result is one of:
+ - the new server name from the .well-known (as a `bytes`)
+ - None if there was no .well-known file.
+ - INVALID_WELL_KNOWN if the .well-known was invalid
+ """
uri = b"https://%s/.well-known/matrix/server" % (server_name, )
uri_str = uri.decode("ascii")
logger.info("Fetching %s", uri_str)
@@ -301,18 +324,7 @@ class MatrixFederationAgent(object):
body = yield make_deferred_yieldable(readBody(response))
if response.code != 200:
raise Exception("Non-200 response %s" % (response.code, ))
- except Exception as e:
- logger.info("Error fetching %s: %s", uri_str, e)
-
- # add some randomness to the TTL to avoid a stampeding herd every hour
- # after startup
- cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD
- cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER)
- self._well_known_cache.set(server_name, None, cache_period)
- defer.returnValue(None)
-
- try:
parsed_body = json.loads(body.decode('utf-8'))
logger.info("Response from .well-known: %s", parsed_body)
if not isinstance(parsed_body, dict):
@@ -320,7 +332,13 @@ class MatrixFederationAgent(object):
if "m.server" not in parsed_body:
raise Exception("Missing key 'm.server'")
except Exception as e:
- raise Exception("invalid .well-known response from %s: %s" % (uri_str, e,))
+ logger.info("Error fetching %s: %s", uri_str, e)
+
+ # add some randomness to the TTL to avoid a stampeding herd every hour
+ # after startup
+ cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD
+ cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER)
+ defer.returnValue((None, cache_period))
result = parsed_body["m.server"].encode("ascii")
@@ -336,10 +354,20 @@ class MatrixFederationAgent(object):
else:
cache_period = min(cache_period, WELL_KNOWN_MAX_CACHE_PERIOD)
- if cache_period > 0:
- self._well_known_cache.set(server_name, result, cache_period)
+ defer.returnValue((result, cache_period))
- defer.returnValue(result)
+
+@implementer(IStreamClientEndpoint)
+class LoggingHostnameEndpoint(object):
+ """A wrapper for HostnameEndpint which logs when it connects"""
+ def __init__(self, reactor, host, port, *args, **kwargs):
+ self.host = host
+ self.port = port
+ self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs)
+
+ def connect(self, protocol_factory):
+ logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port)
+ return self.ep.connect(protocol_factory)
def _cache_period_from_headers(headers, time_now=time.time):
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index c3ab7db7ae..d14a7b2538 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -428,13 +428,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
"""
# for now we do this by looking at the create event. We may want to cache this
# more intelligently in future.
- state_ids = yield self.get_current_state_ids(room_id)
- create_id = state_ids.get((EventTypes.Create, ""))
-
- if not create_id:
- raise NotFoundError("Unknown room %s" % (room_id))
- create_event = yield self.get_event(create_id)
+ # Retrieve the room's create event
+ create_event = yield self.get_create_event_for_room(room_id)
defer.returnValue(create_event.content.get("room_version", "1"))
@defer.inlineCallbacks
@@ -447,19 +443,39 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
Returns:
Deferred[unicode|None]: predecessor room id
+
+ Raises:
+ NotFoundError if the room is unknown
+ """
+ # Retrieve the room's create event
+ create_event = yield self.get_create_event_for_room(room_id)
+
+ # Return predecessor if present
+ defer.returnValue(create_event.content.get("predecessor", None))
+
+ @defer.inlineCallbacks
+ def get_create_event_for_room(self, room_id):
+ """Get the create state event for a room.
+
+ Args:
+ room_id (str)
+
+ Returns:
+ Deferred[EventBase]: The room creation event.
+
+ Raises:
+ NotFoundError if the room is unknown
"""
state_ids = yield self.get_current_state_ids(room_id)
create_id = state_ids.get((EventTypes.Create, ""))
# If we can't find the create event, assume we've hit a dead end
if not create_id:
- defer.returnValue(None)
+ raise NotFoundError("Unknown room %s" % (room_id))
- # Retrieve the room's create event
+ # Retrieve the room's create event and return
create_event = yield self.get_event(create_id)
-
- # Return predecessor if present
- defer.returnValue(create_event.content.get("predecessor", None))
+ defer.returnValue(create_event)
@cached(max_entries=100000, iterable=True)
def get_current_state_ids(self, room_id):
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index 7b2800021f..dcf184d3cf 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -124,7 +124,7 @@ class MatrixFederationAgentTests(TestCase):
_check_logcontext(context)
def _handle_well_known_connection(
- self, client_factory, expected_sni, target_server, response_headers={},
+ self, client_factory, expected_sni, content, response_headers={},
):
"""Handle an outgoing HTTPs connection: wire it up to a server, check that the
request is for a .well-known, and send the response.
@@ -132,8 +132,7 @@ class MatrixFederationAgentTests(TestCase):
Args:
client_factory (IProtocolFactory): outgoing connection
expected_sni (bytes): SNI that we expect the outgoing connection to send
- target_server (bytes): target server that we should redirect to in the
- .well-known response.
+ content (bytes): content to send back as the .well-known
Returns:
HTTPChannel: server impl
"""
@@ -145,10 +144,10 @@ class MatrixFederationAgentTests(TestCase):
# check the .well-known request and send a response
self.assertEqual(len(well_known_server.requests), 1)
request = well_known_server.requests[0]
- self._send_well_known_response(request, target_server, headers=response_headers)
+ self._send_well_known_response(request, content, headers=response_headers)
return well_known_server
- def _send_well_known_response(self, request, target_server, headers={}):
+ def _send_well_known_response(self, request, content, headers={}):
"""Check that an incoming request looks like a valid .well-known request, and
send back the response.
"""
@@ -161,7 +160,7 @@ class MatrixFederationAgentTests(TestCase):
# send back a response
for k, v in headers.items():
request.setHeader(k, v)
- request.write(b'{ "m.server": "%s" }' % (target_server,))
+ request.write(content)
request.finish()
self.reactor.pump((0.1, ))
@@ -358,9 +357,8 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv",
- )
+ # No SRV record lookup yet
+ self.mock_resolver.resolve_service.assert_not_called()
# there should be an attempt to connect on port 443 for the .well-known
clients = self.reactor.tcpClients
@@ -376,6 +374,11 @@ class MatrixFederationAgentTests(TestCase):
# .well-known request fails.
self.reactor.pump((0.4,))
+ # now there should be a SRV lookup
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.testserv",
+ )
+
# we should fall back to a direct connection
self.assertEqual(len(clients), 2)
(host, port, client_factory, _timeout, _bindAddress) = clients[1]
@@ -403,8 +406,7 @@ class MatrixFederationAgentTests(TestCase):
self.successResultOf(test_d)
def test_get_well_known(self):
- """Test the behaviour when the server name has no port and no SRV record, but
- the .well-known redirects elsewhere
+ """Test the behaviour when the .well-known delegates elsewhere
"""
self.mock_resolver.resolve_service.side_effect = lambda _: []
@@ -416,11 +418,6 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv",
- )
- self.mock_resolver.resolve_service.reset_mock()
-
# there should be an attempt to connect on port 443 for the .well-known
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 1)
@@ -429,10 +426,11 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual(port, 443)
self._handle_well_known_connection(
- client_factory, expected_sni=b"testserv", target_server=b"target-server",
+ client_factory, expected_sni=b"testserv",
+ content=b'{ "m.server": "target-server" }',
)
- # there should be another SRV lookup
+ # there should be a SRV lookup
self.mock_resolver.resolve_service.assert_called_once_with(
b"_matrix._tcp.target-server",
)
@@ -483,11 +481,6 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv",
- )
- self.mock_resolver.resolve_service.reset_mock()
-
# there should be an attempt to connect on port 443 for the .well-known
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 1)
@@ -529,7 +522,7 @@ class MatrixFederationAgentTests(TestCase):
self.reactor.pump((0.1, ))
- # there should be another SRV lookup
+ # there should be a SRV lookup
self.mock_resolver.resolve_service.assert_called_once_with(
b"_matrix._tcp.target-server",
)
@@ -567,6 +560,64 @@ class MatrixFederationAgentTests(TestCase):
self.well_known_cache.expire()
self.assertNotIn(b"testserv", self.well_known_cache)
+ def test_get_invalid_well_known(self):
+ """
+ Test the behaviour when the server name has an *invalid* well-known (and no SRV)
+ """
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: []
+ self.reactor.lookups["testserv"] = "1.2.3.4"
+
+ test_d = self._make_get_request(b"matrix://testserv/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ # No SRV record lookup yet
+ self.mock_resolver.resolve_service.assert_not_called()
+
+ # there should be an attempt to connect on port 443 for the .well-known
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ self._handle_well_known_connection(
+ client_factory, expected_sni=b"testserv", content=b'NOT JSON',
+ )
+
+ # now there should be a SRV lookup
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.testserv",
+ )
+
+ # we should fall back to a direct connection
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients.pop()
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 8448)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=b'testserv',
+ )
+
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'testserv'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
def test_get_hostname_srv(self):
"""
Test the behaviour when there is a single SRV record
@@ -581,6 +632,7 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
+ # the request for a .well-known will have failed with a DNS lookup error.
self.mock_resolver.resolve_service.assert_called_once_with(
b"_matrix._tcp.testserv",
)
@@ -613,11 +665,9 @@ class MatrixFederationAgentTests(TestCase):
self.successResultOf(test_d)
def test_get_well_known_srv(self):
- """Test the behaviour when the server name has no port and no SRV record, but
- the .well-known redirects to a place where there is a SRV.
+ """Test the behaviour when the .well-known redirects to a place where there
+ is a SRV.
"""
-
- self.mock_resolver.resolve_service.side_effect = lambda _: []
self.reactor.lookups["testserv"] = "1.2.3.4"
self.reactor.lookups["srvtarget"] = "5.6.7.8"
@@ -626,11 +676,6 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.testserv",
- )
- self.mock_resolver.resolve_service.reset_mock()
-
# there should be an attempt to connect on port 443 for the .well-known
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 1)
@@ -643,10 +688,11 @@ class MatrixFederationAgentTests(TestCase):
]
self._handle_well_known_connection(
- client_factory, expected_sni=b"testserv", target_server=b"target-server",
+ client_factory, expected_sni=b"testserv",
+ content=b'{ "m.server": "target-server" }',
)
- # there should be another SRV lookup
+ # there should be a SRV lookup
self.mock_resolver.resolve_service.assert_called_once_with(
b"_matrix._tcp.target-server",
)
@@ -691,9 +737,8 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.xn--bcher-kva.com",
- )
+ # No SRV record lookup yet
+ self.mock_resolver.resolve_service.assert_not_called()
# there should be an attempt to connect on port 443 for the .well-known
clients = self.reactor.tcpClients
@@ -709,6 +754,11 @@ class MatrixFederationAgentTests(TestCase):
# .well-known request fails.
self.reactor.pump((0.4,))
+ # now there should have been a SRV lookup
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.xn--bcher-kva.com",
+ )
+
# We should fall back to port 8448
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 2)
@@ -806,7 +856,7 @@ class MatrixFederationAgentTests(TestCase):
client_factory,
expected_sni=b"testserv",
response_headers={b'Cache-Control': b'max-age=10'},
- target_server=b"target-server",
+ content=b'{ "m.server": "target-server" }',
)
r = self.successResultOf(fetch_d)
@@ -834,7 +884,7 @@ class MatrixFederationAgentTests(TestCase):
self._handle_well_known_connection(
client_factory,
expected_sni=b"testserv",
- target_server=b"other-server",
+ content=b'{ "m.server": "other-server" }',
)
r = self.successResultOf(fetch_d)
|
