From e79ba9eb34697cb6205c50282882b294c0acafa7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 24 Jan 2019 09:28:16 +0000 Subject: Fix tests --- tests/storage/test_redaction.py | 5 ++++- tests/storage/test_roommember.py | 3 ++- tests/storage/test_state.py | 3 ++- tests/test_visibility.py | 4 ++++ tests/utils.py | 3 ++- 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 02bf975fbf..3957561b1e 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -18,7 +18,7 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RoomVersions from synapse.types import RoomID, UserID from tests import unittest @@ -52,6 +52,7 @@ class RedactionTestCase(unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Member, "sender": user.to_string(), @@ -74,6 +75,7 @@ class RedactionTestCase(unittest.TestCase): self.depth += 1 builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Message, "sender": user.to_string(), @@ -94,6 +96,7 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Redaction, "sender": user.to_string(), diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 978c66133d..7fa2f4fd70 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -18,7 +18,7 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RoomVersions from synapse.types import RoomID, UserID from tests import unittest @@ -50,6 +50,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Member, "sender": user.to_string(), diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index 086a39d834..a1f99134dc 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RoomVersions from synapse.storage.state import StateFilter from synapse.types import RoomID, UserID @@ -52,6 +52,7 @@ class StateStoreTestCase(tests.unittest.TestCase): @defer.inlineCallbacks def inject_state_event(self, room, sender, typ, state_key, content): builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": typ, "sender": sender.to_string(), diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 2eea3b098b..82d63ce00e 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -17,6 +17,7 @@ import logging from twisted.internet import defer from twisted.internet.defer import succeed +from synapse.api.constants import RoomVersions from synapse.events import FrozenEvent from synapse.visibility import filter_events_for_server @@ -124,6 +125,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): def inject_visibility(self, user_id, visibility): content = {"history_visibility": visibility} builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": "m.room.history_visibility", "sender": user_id, @@ -144,6 +146,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": "m.room.member", "sender": user_id, @@ -165,6 +168,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): if content is None: content = {"body": "testytest"} builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": "m.room.message", "sender": user_id, diff --git a/tests/utils.py b/tests/utils.py index 08d6faa0a6..1aa8f98094 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -26,7 +26,7 @@ from six.moves.urllib import parse as urlparse from twisted.internet import defer, reactor -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, RoomVersions from synapse.api.errors import CodeMessageException, cs_error from synapse.config.server import ServerConfig from synapse.federation.transport import server @@ -622,6 +622,7 @@ def create_room(hs, room_id, creator_id): event_creation_handler = hs.get_event_creation_handler() builder = event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Create, "state_key": "", -- cgit 1.5.1 From be6a7e47fa75d72466a356c254376b8eb0707bb2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Jan 2019 10:23:51 +0000 Subject: Revert "Require event format version to parse or create events" --- changelog.d/4447.misc | 1 - changelog.d/4451.misc | 1 - synapse/events/__init__.py | 39 +---------------- synapse/events/builder.py | 51 +---------------------- synapse/federation/federation_base.py | 9 ++-- synapse/federation/federation_client.py | 74 ++++++++++++--------------------- synapse/federation/federation_server.py | 41 ++++++------------ synapse/federation/transport/server.py | 4 +- synapse/handlers/federation.py | 72 +++++++++++++------------------- synapse/handlers/message.py | 10 +---- synapse/replication/http/federation.py | 8 +--- synapse/replication/http/send_event.py | 8 +--- synapse/storage/events_worker.py | 8 +++- tests/storage/test_redaction.py | 5 +-- tests/storage/test_roommember.py | 3 +- tests/storage/test_state.py | 3 +- tests/test_visibility.py | 4 -- tests/utils.py | 3 +- 18 files changed, 91 insertions(+), 253 deletions(-) delete mode 100644 changelog.d/4447.misc delete mode 100644 changelog.d/4451.misc (limited to 'tests') diff --git a/changelog.d/4447.misc b/changelog.d/4447.misc deleted file mode 100644 index 43f8963614..0000000000 --- a/changelog.d/4447.misc +++ /dev/null @@ -1 +0,0 @@ -Add infrastructure to support different event formats diff --git a/changelog.d/4451.misc b/changelog.d/4451.misc deleted file mode 100644 index 43f8963614..0000000000 --- a/changelog.d/4451.misc +++ /dev/null @@ -1 +0,0 @@ -Add infrastructure to support different event formats diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index c3e6caf597..888296933b 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -18,11 +18,7 @@ from distutils.util import strtobool import six -from synapse.api.constants import ( - KNOWN_EVENT_FORMAT_VERSIONS, - KNOWN_ROOM_VERSIONS, - EventFormatVersions, -) +from synapse.api.constants import EventFormatVersions from synapse.util.caches import intern_dict from synapse.util.frozenutils import freeze @@ -244,36 +240,3 @@ class FrozenEvent(EventBase): self.get("type", None), self.get("state_key", None), ) - - -def room_version_to_event_format(room_version): - """Converts a room version string to the event format - - Args: - room_version (str) - - Returns: - int - """ - if room_version not in KNOWN_ROOM_VERSIONS: - raise - - return EventFormatVersions.V1 - - -def event_type_from_format_version(format_version): - """Returns the python type to use to construct an Event object for the - given event format version. - - Args: - format_version (int): The event format version - - Returns: - type: A type that can be initialized as per the initializer of - `FrozenEvent` - """ - if format_version not in KNOWN_EVENT_FORMAT_VERSIONS: - raise Exception( - "No event format %r" % (format_version,) - ) - return FrozenEvent diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 7e63371095..e662eaef10 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -15,39 +15,12 @@ import copy -from synapse.api.constants import RoomVersions from synapse.types import EventID from synapse.util.stringutils import random_string from . import EventBase, FrozenEvent, _event_dict_property -def get_event_builder(room_version, key_values={}, internal_metadata_dict={}): - """Generate an event builder appropriate for the given room version - - Args: - room_version (str): Version of the room that we're creating an - event builder for - key_values (dict): Fields used as the basis of the new event - internal_metadata_dict (dict): Used to create the `_EventInternalMetadata` - object. - - Returns: - EventBuilder - """ - if room_version in { - RoomVersions.V1, - RoomVersions.V2, - RoomVersions.VDH_TEST, - RoomVersions.STATE_V2_TEST, - }: - return EventBuilder(key_values, internal_metadata_dict) - else: - raise Exception( - "No event format defined for version %r" % (room_version,) - ) - - class EventBuilder(EventBase): def __init__(self, key_values={}, internal_metadata_dict={}): signatures = copy.deepcopy(key_values.pop("signatures", {})) @@ -85,29 +58,7 @@ class EventBuilderFactory(object): return e_id.to_string() - def new(self, room_version, key_values={}): - """Generate an event builder appropriate for the given room version - - Args: - room_version (str): Version of the room that we're creating an - event builder for - key_values (dict): Fields used as the basis of the new event - - Returns: - EventBuilder - """ - - # There's currently only the one event version defined - if room_version not in { - RoomVersions.V1, - RoomVersions.V2, - RoomVersions.VDH_TEST, - RoomVersions.STATE_V2_TEST, - }: - raise Exception( - "No event format defined for version %r" % (room_version,) - ) - + def new(self, key_values={}): key_values["event_id"] = self.create_event_id() time_now = int(self.clock.time_msec()) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 5c31e5f85f..d749bfdd3a 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -23,7 +23,7 @@ from twisted.internet.defer import DeferredList from synapse.api.constants import MAX_DEPTH, EventTypes, Membership from synapse.api.errors import Codes, SynapseError from synapse.crypto.event_signing import check_event_content_hash -from synapse.events import event_type_from_format_version +from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.http.servlet import assert_params_in_dict from synapse.types import get_domain_from_id @@ -302,12 +302,11 @@ def _is_invite_via_3pid(event): ) -def event_from_pdu_json(pdu_json, event_format_version, outlier=False): +def event_from_pdu_json(pdu_json, outlier=False): """Construct a FrozenEvent from an event json received over federation Args: pdu_json (object): pdu as received over federation - event_format_version (int): The event format version outlier (bool): True to mark this event as an outlier Returns: @@ -331,8 +330,8 @@ def event_from_pdu_json(pdu_json, event_format_version, outlier=False): elif depth > MAX_DEPTH: raise SynapseError(400, "Depth too large", Codes.BAD_JSON) - event = event_type_from_format_version(event_format_version)( - pdu_json, + event = FrozenEvent( + pdu_json ) event.internal_metadata.outlier = outlier diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4b25f891ca..777deabdf7 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -38,7 +38,6 @@ from synapse.api.errors import ( SynapseError, ) from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import room_version_to_event_format from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.util import logcontext, unwrapFirstError from synapse.util.caches.expiringcache import ExpiringCache @@ -170,13 +169,13 @@ class FederationClient(FederationBase): @defer.inlineCallbacks @log_function - def backfill(self, dest, room_id, limit, extremities): + def backfill(self, dest, context, limit, extremities): """Requests some more historic PDUs for the given context from the given destination server. Args: dest (str): The remote home server to ask. - room_id (str): The room_id to backfill. + context (str): The context to backfill. limit (int): The maximum number of PDUs to return. extremities (list): List of PDU id and origins of the first pdus we have seen from the context @@ -191,15 +190,12 @@ class FederationClient(FederationBase): return transaction_data = yield self.transport_layer.backfill( - dest, room_id, extremities, limit) + dest, context, extremities, limit) logger.debug("backfill transaction_data=%s", repr(transaction_data)) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - pdus = [ - event_from_pdu_json(p, format_ver, outlier=False) + event_from_pdu_json(p, outlier=False) for p in transaction_data["pdus"] ] @@ -243,8 +239,6 @@ class FederationClient(FederationBase): pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {}) - format_ver = room_version_to_event_format(room_version) - signed_pdu = None for destination in destinations: now = self._clock.time_msec() @@ -260,7 +254,7 @@ class FederationClient(FederationBase): logger.debug("transaction_data %r", transaction_data) pdu_list = [ - event_from_pdu_json(p, format_ver, outlier=outlier) + event_from_pdu_json(p, outlier=outlier) for p in transaction_data["pdus"] ] @@ -354,16 +348,12 @@ class FederationClient(FederationBase): destination, room_id, event_id=event_id, ) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - pdus = [ - event_from_pdu_json(p, format_ver, outlier=True) - for p in result["pdus"] + event_from_pdu_json(p, outlier=True) for p in result["pdus"] ] auth_chain = [ - event_from_pdu_json(p, format_ver, outlier=True) + event_from_pdu_json(p, outlier=True) for p in result.get("auth_chain", []) ] @@ -371,6 +361,8 @@ class FederationClient(FederationBase): ev.event_id for ev in itertools.chain(pdus, auth_chain) ]) + room_version = yield self.store.get_room_version(room_id) + signed_pdus = yield self._check_sigs_and_hash_and_fetch( destination, [p for p in pdus if p.event_id not in seen_events], @@ -469,14 +461,13 @@ class FederationClient(FederationBase): destination, room_id, event_id, ) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - auth_chain = [ - event_from_pdu_json(p, format_ver, outlier=True) + event_from_pdu_json(p, outlier=True) for p in res["auth_chain"] ] + room_version = yield self.store.get_room_version(room_id) + signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, auth_chain, outlier=True, room_version=room_version, @@ -566,9 +557,9 @@ class FederationClient(FederationBase): params (dict[str, str|Iterable[str]]): Query parameters to include in the request. Return: - Deferred[tuple[str, FrozenEvent, int]]: resolves to a tuple of - `(origin, event, event_format)` where origin is the remote - homeserver which generated the event. + Deferred[tuple[str, FrozenEvent]]: resolves to a tuple of `origin` + and event where origin is the remote homeserver which generated + the event. Fails with a ``SynapseError`` if the chosen remote server returns a 300/400 code. @@ -588,11 +579,6 @@ class FederationClient(FederationBase): destination, room_id, user_id, membership, params, ) - # Note: If not supplied, the room version may be either v1 or v2, - # however either way the event format version will be v1. - room_version = ret.get("room_version", RoomVersions.V1) - event_format = room_version_to_event_format(room_version) - pdu_dict = ret.get("event", None) if not isinstance(pdu_dict, dict): raise InvalidResponseError("Bad 'event' field in response") @@ -612,7 +598,7 @@ class FederationClient(FederationBase): pdu_dict.pop("origin_server_ts", None) pdu_dict.pop("unsigned", None) - builder = self.event_builder_factory.new(room_version, pdu_dict) + builder = self.event_builder_factory.new(pdu_dict) add_hashes_and_signatures( builder, self.hs.hostname, @@ -621,14 +607,14 @@ class FederationClient(FederationBase): ev = builder.build() defer.returnValue( - (destination, ev, event_format) + (destination, ev) ) return self._try_destination_list( "make_" + membership, destinations, send_request, ) - def send_join(self, destinations, pdu, event_format_version): + def send_join(self, destinations, pdu): """Sends a join event to one of a list of homeservers. Doing so will cause the remote server to add the event to the graph, @@ -638,7 +624,6 @@ class FederationClient(FederationBase): destinations (str): Candidate homeservers which are probably participating in the room. pdu (BaseEvent): event to be sent - event_format_version (int): The event format version Return: Deferred: resolves to a dict with members ``origin`` (a string @@ -684,12 +669,12 @@ class FederationClient(FederationBase): logger.debug("Got content: %s", content) state = [ - event_from_pdu_json(p, event_format_version, outlier=True) + event_from_pdu_json(p, outlier=True) for p in content.get("state", []) ] auth_chain = [ - event_from_pdu_json(p, event_format_version, outlier=True) + event_from_pdu_json(p, outlier=True) for p in content.get("auth_chain", []) ] @@ -767,10 +752,7 @@ class FederationClient(FederationBase): logger.debug("Got response to send_invite: %s", pdu_dict) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - - pdu = event_from_pdu_json(pdu_dict, format_ver) + pdu = event_from_pdu_json(pdu_dict) # Check signatures are correct. pdu = yield self._check_sigs_and_hash(pdu) @@ -848,14 +830,13 @@ class FederationClient(FederationBase): content=send_content, ) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - auth_chain = [ - event_from_pdu_json(e, format_ver) + event_from_pdu_json(e) for e in content["auth_chain"] ] + room_version = yield self.store.get_room_version(room_id) + signed_auth = yield self._check_sigs_and_hash_and_fetch( destination, auth_chain, outlier=True, room_version=room_version, ) @@ -899,14 +880,13 @@ class FederationClient(FederationBase): timeout=timeout, ) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - events = [ - event_from_pdu_json(e, format_ver) + event_from_pdu_json(e) for e in content.get("events", []) ] + room_version = yield self.store.get_room_version(room_id) + signed_events = yield self._check_sigs_and_hash_and_fetch( destination, events, outlier=False, room_version=room_version, ) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4aa04b9588..cb729c69ea 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -34,7 +34,6 @@ from synapse.api.errors import ( SynapseError, ) from synapse.crypto.event_signing import compute_event_signature -from synapse.events import room_version_to_event_format from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.federation.persistence import TransactionActions from synapse.federation.units import Edu, Transaction @@ -179,13 +178,14 @@ class FederationServer(FederationBase): continue try: - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) + # In future we will actually use the room version to parse the + # PDU into an event. + yield self.store.get_room_version(room_id) except NotFoundError: logger.info("Ignoring PDU for unknown room_id: %s", room_id) continue - event = event_from_pdu_json(p, format_ver) + event = event_from_pdu_json(p) pdus_by_room.setdefault(room_id, []).append(event) pdu_results = {} @@ -370,9 +370,7 @@ class FederationServer(FederationBase): @defer.inlineCallbacks def on_invite_request(self, origin, content, room_version): - format_ver = room_version_to_event_format(room_version) - - pdu = event_from_pdu_json(content, format_ver) + pdu = event_from_pdu_json(content) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) ret_pdu = yield self.handler.on_invite_request(origin, pdu) @@ -380,12 +378,9 @@ class FederationServer(FederationBase): defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)}) @defer.inlineCallbacks - def on_send_join_request(self, origin, content, room_id): + def on_send_join_request(self, origin, content): logger.debug("on_send_join_request: content: %s", content) - - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - pdu = event_from_pdu_json(content, format_ver) + pdu = event_from_pdu_json(content) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) @@ -405,22 +400,13 @@ class FederationServer(FederationBase): origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, room_id) pdu = yield self.handler.on_make_leave_request(room_id, user_id) - - room_version = yield self.store.get_room_version(room_id) - time_now = self._clock.time_msec() - defer.returnValue({ - "event": pdu.get_pdu_json(time_now), - "room_version": room_version, - }) + defer.returnValue({"event": pdu.get_pdu_json(time_now)}) @defer.inlineCallbacks - def on_send_leave_request(self, origin, content, room_id): + def on_send_leave_request(self, origin, content): logger.debug("on_send_leave_request: content: %s", content) - - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - pdu = event_from_pdu_json(content, format_ver) + pdu = event_from_pdu_json(content) origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, pdu.room_id) @@ -466,14 +452,13 @@ class FederationServer(FederationBase): origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, room_id) - room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) - auth_chain = [ - event_from_pdu_json(e, format_ver) + event_from_pdu_json(e) for e in content["auth_chain"] ] + room_version = yield self.store.get_room_version(room_id) + signed_auth = yield self._check_sigs_and_hash_and_fetch( origin, auth_chain, outlier=True, room_version=room_version, ) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 67ae0212c3..4557a9e66e 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -469,7 +469,7 @@ class FederationSendLeaveServlet(BaseFederationServlet): @defer.inlineCallbacks def on_PUT(self, origin, content, query, room_id, event_id): - content = yield self.handler.on_send_leave_request(origin, content, room_id) + content = yield self.handler.on_send_leave_request(origin, content) defer.returnValue((200, content)) @@ -487,7 +487,7 @@ class FederationSendJoinServlet(BaseFederationServlet): def on_PUT(self, origin, content, query, context, event_id): # TODO(paul): assert that context/event_id parsed from path actually # match those given in content - content = yield self.handler.on_send_join_request(origin, content, context) + content = yield self.handler.on_send_join_request(origin, content) defer.returnValue((200, content)) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a4b771049c..453d393ce1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1061,7 +1061,7 @@ class FederationHandler(BaseHandler): """ logger.debug("Joining %s to %s", joinee, room_id) - origin, event, event_format_version = yield self._make_and_verify_event( + origin, event = yield self._make_and_verify_event( target_hosts, room_id, joinee, @@ -1091,9 +1091,7 @@ class FederationHandler(BaseHandler): target_hosts.insert(0, origin) except ValueError: pass - ret = yield self.federation_client.send_join( - target_hosts, event, event_format_version, - ) + ret = yield self.federation_client.send_join(target_hosts, event) origin = ret["origin"] state = ret["state"] @@ -1166,18 +1164,13 @@ class FederationHandler(BaseHandler): """ event_content = {"membership": Membership.JOIN} - room_version = yield self.store.get_room_version(room_id) - - builder = self.event_builder_factory.new( - room_version, - { - "type": EventTypes.Member, - "content": event_content, - "room_id": room_id, - "sender": user_id, - "state_key": user_id, - } - ) + builder = self.event_builder_factory.new({ + "type": EventTypes.Member, + "content": event_content, + "room_id": room_id, + "sender": user_id, + "state_key": user_id, + }) try: event, context = yield self.event_creation_handler.create_new_client_event( @@ -1311,7 +1304,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def do_remotely_reject_invite(self, target_hosts, room_id, user_id): - origin, event, event_format_version = yield self._make_and_verify_event( + origin, event = yield self._make_and_verify_event( target_hosts, room_id, user_id, @@ -1343,7 +1336,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _make_and_verify_event(self, target_hosts, room_id, user_id, membership, content={}, params=None): - origin, event, format_ver = yield self.federation_client.make_membership_event( + origin, pdu = yield self.federation_client.make_membership_event( target_hosts, room_id, user_id, @@ -1352,7 +1345,9 @@ class FederationHandler(BaseHandler): params=params, ) - logger.debug("Got response to make_%s: %s", membership, event) + logger.debug("Got response to make_%s: %s", membership, pdu) + + event = pdu # We should assert some things. # FIXME: Do this in a nicer way @@ -1360,7 +1355,7 @@ class FederationHandler(BaseHandler): assert(event.user_id == user_id) assert(event.state_key == user_id) assert(event.room_id == room_id) - defer.returnValue((origin, event, format_ver)) + defer.returnValue((origin, event)) @defer.inlineCallbacks @log_function @@ -1369,17 +1364,13 @@ class FederationHandler(BaseHandler): leave event for the room and return that. We do *not* persist or process it until the other server has signed it and sent it back. """ - room_version = yield self.store.get_room_version(room_id) - builder = self.event_builder_factory.new( - room_version, - { - "type": EventTypes.Member, - "content": {"membership": Membership.LEAVE}, - "room_id": room_id, - "sender": user_id, - "state_key": user_id, - } - ) + builder = self.event_builder_factory.new({ + "type": EventTypes.Member, + "content": {"membership": Membership.LEAVE}, + "room_id": room_id, + "sender": user_id, + "state_key": user_id, + }) event, context = yield self.event_creation_handler.create_new_client_event( builder=builder, @@ -2275,16 +2266,14 @@ class FederationHandler(BaseHandler): } if (yield self.auth.check_host_in_room(room_id, self.hs.hostname)): - room_version = yield self.store.get_room_version(room_id) - builder = self.event_builder_factory.new(room_version, event_dict) - + builder = self.event_builder_factory.new(event_dict) EventValidator().validate_new(builder) event, context = yield self.event_creation_handler.create_new_client_event( builder=builder ) event, context = yield self.add_display_name_to_third_party_invite( - room_version, event_dict, event, context + event_dict, event, context ) try: @@ -2315,18 +2304,14 @@ class FederationHandler(BaseHandler): Returns: Deferred: resolves (to None) """ - room_version = yield self.store.get_room_version(room_id) - - # NB: event_dict has a particular specced format we might need to fudge - # if we change event formats too much. - builder = self.event_builder_factory.new(room_version, event_dict) + builder = self.event_builder_factory.new(event_dict) event, context = yield self.event_creation_handler.create_new_client_event( builder=builder, ) event, context = yield self.add_display_name_to_third_party_invite( - room_version, event_dict, event, context + event_dict, event, context ) try: @@ -2346,8 +2331,7 @@ class FederationHandler(BaseHandler): yield member_handler.send_membership_event(None, event, context) @defer.inlineCallbacks - def add_display_name_to_third_party_invite(self, room_version, event_dict, - event, context): + def add_display_name_to_third_party_invite(self, event_dict, event, context): key = ( EventTypes.ThirdPartyInvite, event.content["third_party_invite"]["signed"]["token"] @@ -2371,7 +2355,7 @@ class FederationHandler(BaseHandler): # auth checks. If we need the invite and don't have it then the # auth check code will explode appropriately. - builder = self.event_builder_factory.new(room_version, event_dict) + builder = self.event_builder_factory.new(event_dict) EventValidator().validate_new(builder) event, context = yield self.event_creation_handler.create_new_client_event( builder=builder, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7aaa4fba33..a7cd779b02 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -278,15 +278,7 @@ class EventCreationHandler(object): """ yield self.auth.check_auth_blocking(requester.user.to_string()) - if event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "": - room_version = event_dict["content"]["room_version"] - else: - try: - room_version = yield self.store.get_room_version(event_dict["room_id"]) - except NotFoundError: - raise AuthError(403, "Unknown room") - - builder = self.event_builder_factory.new(room_version, event_dict) + builder = self.event_builder_factory.new(event_dict) self.validator.validate_new(builder) diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index 2e16c69666..64a79da162 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.events import event_type_from_format_version +from synapse.events import FrozenEvent from synapse.events.snapshot import EventContext from synapse.http.servlet import parse_json_object_from_request from synapse.replication.http._base import ReplicationEndpoint @@ -70,7 +70,6 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): event_payloads.append({ "event": event.get_pdu_json(), - "event_format_version": event.format_version, "internal_metadata": event.internal_metadata.get_dict(), "rejected_reason": event.rejected_reason, "context": serialized_context, @@ -95,12 +94,9 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): event_and_contexts = [] for event_payload in event_payloads: event_dict = event_payload["event"] - format_ver = content["event_format_version"] internal_metadata = event_payload["internal_metadata"] rejected_reason = event_payload["rejected_reason"] - - EventType = event_type_from_format_version(format_ver) - event = EventType(event_dict, internal_metadata, rejected_reason) + event = FrozenEvent(event_dict, internal_metadata, rejected_reason) context = yield EventContext.deserialize( self.store, event_payload["context"], diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py index 3635015eda..5b52c91650 100644 --- a/synapse/replication/http/send_event.py +++ b/synapse/replication/http/send_event.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.events import event_type_from_format_version +from synapse.events import FrozenEvent from synapse.events.snapshot import EventContext from synapse.http.servlet import parse_json_object_from_request from synapse.replication.http._base import ReplicationEndpoint @@ -74,7 +74,6 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint): payload = { "event": event.get_pdu_json(), - "event_format_version": event.format_version, "internal_metadata": event.internal_metadata.get_dict(), "rejected_reason": event.rejected_reason, "context": serialized_context, @@ -91,12 +90,9 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint): content = parse_json_object_from_request(request) event_dict = content["event"] - format_ver = content["event_format_version"] internal_metadata = content["internal_metadata"] rejected_reason = content["rejected_reason"] - - EventType = event_type_from_format_version(format_ver) - event = EventType(event_dict, internal_metadata, rejected_reason) + event = FrozenEvent(event_dict, internal_metadata, rejected_reason) requester = Requester.deserialize(self.store, content["requester"]) context = yield EventContext.deserialize(self.store, content["context"]) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index 0a0ca58fc4..599f892858 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -23,7 +23,7 @@ from twisted.internet import defer from synapse.api.constants import EventFormatVersions from synapse.api.errors import NotFoundError -from synapse.events import FrozenEvent, event_type_from_format_version # noqa: F401 +from synapse.events import FrozenEvent # these are only included to make the type annotations work from synapse.events.snapshot import EventContext # noqa: F401 from synapse.events.utils import prune_event @@ -412,7 +412,11 @@ class EventsWorkerStore(SQLBaseStore): # of a event format version, so it must be a V1 event. format_version = EventFormatVersions.V1 - original_ev = event_type_from_format_version(format_version)( + # TODO: When we implement new event formats we'll need to use a + # different event python type + assert format_version == EventFormatVersions.V1 + + original_ev = FrozenEvent( event_dict=d, internal_metadata_dict=internal_metadata, rejected_reason=rejected_reason, diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 3957561b1e..02bf975fbf 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -18,7 +18,7 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership from synapse.types import RoomID, UserID from tests import unittest @@ -52,7 +52,6 @@ class RedactionTestCase(unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": EventTypes.Member, "sender": user.to_string(), @@ -75,7 +74,6 @@ class RedactionTestCase(unittest.TestCase): self.depth += 1 builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": EventTypes.Message, "sender": user.to_string(), @@ -96,7 +94,6 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": EventTypes.Redaction, "sender": user.to_string(), diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 7fa2f4fd70..978c66133d 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -18,7 +18,7 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership from synapse.types import RoomID, UserID from tests import unittest @@ -50,7 +50,6 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": EventTypes.Member, "sender": user.to_string(), diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index a1f99134dc..086a39d834 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership, RoomVersions +from synapse.api.constants import EventTypes, Membership from synapse.storage.state import StateFilter from synapse.types import RoomID, UserID @@ -52,7 +52,6 @@ class StateStoreTestCase(tests.unittest.TestCase): @defer.inlineCallbacks def inject_state_event(self, room, sender, typ, state_key, content): builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": typ, "sender": sender.to_string(), diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 82d63ce00e..2eea3b098b 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -17,7 +17,6 @@ import logging from twisted.internet import defer from twisted.internet.defer import succeed -from synapse.api.constants import RoomVersions from synapse.events import FrozenEvent from synapse.visibility import filter_events_for_server @@ -125,7 +124,6 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): def inject_visibility(self, user_id, visibility): content = {"history_visibility": visibility} builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": "m.room.history_visibility", "sender": user_id, @@ -146,7 +144,6 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": "m.room.member", "sender": user_id, @@ -168,7 +165,6 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): if content is None: content = {"body": "testytest"} builder = self.event_builder_factory.new( - RoomVersions.V1, { "type": "m.room.message", "sender": user_id, diff --git a/tests/utils.py b/tests/utils.py index 2dfcb70a93..df73c539c3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -26,7 +26,7 @@ from six.moves.urllib import parse as urlparse from twisted.internet import defer, reactor -from synapse.api.constants import EventTypes, RoomVersions +from synapse.api.constants import EventTypes from synapse.api.errors import CodeMessageException, cs_error from synapse.config.server import ServerConfig from synapse.federation.transport import server @@ -624,7 +624,6 @@ def create_room(hs, room_id, creator_id): event_creation_handler = hs.get_event_creation_handler() builder = event_builder_factory.new( - RoomVersions.V1, { "type": EventTypes.Create, "state_key": "", -- cgit 1.5.1 From 9770ed91c223051d940f773e0b1fa2746003eb4e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 24 Jan 2019 09:28:16 +0000 Subject: Fix tests --- tests/storage/test_redaction.py | 5 ++++- tests/storage/test_roommember.py | 3 ++- tests/storage/test_state.py | 3 ++- tests/test_visibility.py | 4 ++++ tests/utils.py | 3 ++- 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 02bf975fbf..3957561b1e 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -18,7 +18,7 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RoomVersions from synapse.types import RoomID, UserID from tests import unittest @@ -52,6 +52,7 @@ class RedactionTestCase(unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Member, "sender": user.to_string(), @@ -74,6 +75,7 @@ class RedactionTestCase(unittest.TestCase): self.depth += 1 builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Message, "sender": user.to_string(), @@ -94,6 +96,7 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Redaction, "sender": user.to_string(), diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 978c66133d..7fa2f4fd70 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -18,7 +18,7 @@ from mock import Mock from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RoomVersions from synapse.types import RoomID, UserID from tests import unittest @@ -50,6 +50,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Member, "sender": user.to_string(), diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index 086a39d834..a1f99134dc 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.constants import EventTypes, Membership +from synapse.api.constants import EventTypes, Membership, RoomVersions from synapse.storage.state import StateFilter from synapse.types import RoomID, UserID @@ -52,6 +52,7 @@ class StateStoreTestCase(tests.unittest.TestCase): @defer.inlineCallbacks def inject_state_event(self, room, sender, typ, state_key, content): builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": typ, "sender": sender.to_string(), diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 2eea3b098b..82d63ce00e 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -17,6 +17,7 @@ import logging from twisted.internet import defer from twisted.internet.defer import succeed +from synapse.api.constants import RoomVersions from synapse.events import FrozenEvent from synapse.visibility import filter_events_for_server @@ -124,6 +125,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): def inject_visibility(self, user_id, visibility): content = {"history_visibility": visibility} builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": "m.room.history_visibility", "sender": user_id, @@ -144,6 +146,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": "m.room.member", "sender": user_id, @@ -165,6 +168,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): if content is None: content = {"body": "testytest"} builder = self.event_builder_factory.new( + RoomVersions.V1, { "type": "m.room.message", "sender": user_id, diff --git a/tests/utils.py b/tests/utils.py index df73c539c3..2dfcb70a93 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -26,7 +26,7 @@ from six.moves.urllib import parse as urlparse from twisted.internet import defer, reactor -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, RoomVersions from synapse.api.errors import CodeMessageException, cs_error from synapse.config.server import ServerConfig from synapse.federation.transport import server @@ -624,6 +624,7 @@ def create_room(hs, room_id, creator_id): event_creation_handler = hs.get_event_creation_handler() builder = event_builder_factory.new( + RoomVersions.V1, { "type": EventTypes.Create, "state_key": "", -- cgit 1.5.1 From 8520bc3109c2de6175391497941f3fc0b74c08e5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 25 Jan 2019 12:38:16 +0000 Subject: Fix Host header sent by MatrixFederationAgent (#4468) Move the Host header logic down here so that (a) it is used if we reuse the agent elsewhere, and (b) we can mess about with it with .well-known. --- changelog.d/4468.misc | 1 + synapse/http/federation/matrix_federation_agent.py | 10 ++++++++++ synapse/http/matrixfederationclient.py | 1 - tests/http/federation/test_matrix_federation_agent.py | 16 ++++++++++++++++ tests/http/test_fedclient.py | 2 +- 5 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4468.misc (limited to 'tests') diff --git a/changelog.d/4468.misc b/changelog.d/4468.misc new file mode 100644 index 0000000000..9a51434755 --- /dev/null +++ b/changelog.d/4468.misc @@ -0,0 +1 @@ +Move SRV logic into the Agent layer diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 0ec28c6696..1788e9a34a 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -19,6 +19,7 @@ from zope.interface import implementer from twisted.internet import defer from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent from synapse.http.endpoint import parse_server_name @@ -109,6 +110,15 @@ class MatrixFederationAgent(object): else: target = pick_server_from_list(server_list) + # make sure that the Host header is set correctly + if headers is None: + headers = Headers() + else: + headers = headers.copy() + + if not headers.hasHeader(b'host'): + headers.addRawHeader(b'host', server_name_bytes) + class EndpointFactory(object): @staticmethod def endpointForURI(_uri): diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 980e912348..bb2e64ed80 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -255,7 +255,6 @@ class MatrixFederationHttpClient(object): headers_dict = { b"User-Agent": [self.version_string_bytes], - b"Host": [destination_bytes], } with limiter: diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index b32d7566a5..261afb5f41 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -131,6 +131,10 @@ class MatrixFederationAgentTests(TestCase): request = http_server.requests[0] self.assertEqual(request.method, b'GET') self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'testserv:8448'] + ) content = request.content.read() self.assertEqual(content, b'') @@ -195,6 +199,10 @@ class MatrixFederationAgentTests(TestCase): request = http_server.requests[0] self.assertEqual(request.method, b'GET') self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'1.2.3.4'], + ) # finish the request request.finish() @@ -235,6 +243,10 @@ class MatrixFederationAgentTests(TestCase): request = http_server.requests[0] self.assertEqual(request.method, b'GET') self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'testserv'], + ) # finish the request request.finish() @@ -276,6 +288,10 @@ class MatrixFederationAgentTests(TestCase): request = http_server.requests[0] self.assertEqual(request.method, b'GET') self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'testserv'], + ) # finish the request request.finish() diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py index d37f8f9981..018c77ebcd 100644 --- a/tests/http/test_fedclient.py +++ b/tests/http/test_fedclient.py @@ -49,7 +49,6 @@ class FederationClientTests(HomeserverTestCase): return hs def prepare(self, reactor, clock, homeserver): - self.cl = MatrixFederationHttpClient(self.hs) self.reactor.lookups["testserv"] = "1.2.3.4" @@ -95,6 +94,7 @@ class FederationClientTests(HomeserverTestCase): # that should have made it send the request to the transport self.assertRegex(transport.value(), b"^GET /foo/bar") + self.assertRegex(transport.value(), b"Host: testserv:8008") # Deferred is still without a result self.assertNoResult(test_d) -- cgit 1.5.1 From ae2a957dbacc38f1126e2eca160f17322c710d26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Jan 2019 18:31:41 +0000 Subject: Pass through room version to event auth --- synapse/api/auth.py | 14 ++++++++++---- synapse/event_auth.py | 3 ++- synapse/handlers/federation.py | 20 ++++++++++++-------- synapse/handlers/message.py | 7 ++++++- synapse/handlers/room.py | 5 ++++- synapse/state/__init__.py | 2 +- synapse/state/v1.py | 14 +++++++++++--- synapse/state/v2.py | 14 +++++++++----- tests/state/test_v2.py | 4 +++- tests/test_event_auth.py | 13 +++++++++++-- 10 files changed, 69 insertions(+), 27 deletions(-) (limited to 'tests') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e37b807c94..7b213e54c8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -65,7 +65,7 @@ class Auth(object): register_cache("cache", "token_cache", self.token_cache) @defer.inlineCallbacks - def check_from_context(self, event, context, do_sig_check=True): + def check_from_context(self, room_version, event, context, do_sig_check=True): prev_state_ids = yield context.get_prev_state_ids(self.store) auth_events_ids = yield self.compute_auth_events( event, prev_state_ids, for_verification=True, @@ -74,12 +74,16 @@ class Auth(object): auth_events = { (e.type, e.state_key): e for e in itervalues(auth_events) } - self.check(event, auth_events=auth_events, do_sig_check=do_sig_check) + self.check( + room_version, event, + auth_events=auth_events, do_sig_check=do_sig_check, + ) - def check(self, event, auth_events, do_sig_check=True): + def check(self, room_version, event, auth_events, do_sig_check=True): """ Checks if this event is correctly authed. Args: + room_version (str): version of the room event: the event being checked. auth_events (dict: event-key -> event): the existing room state. @@ -88,7 +92,9 @@ class Auth(object): True if the auth checks pass. """ with Measure(self.clock, "auth.check"): - event_auth.check(event, auth_events, do_sig_check=do_sig_check) + event_auth.check( + room_version, event, auth_events, do_sig_check=do_sig_check + ) @defer.inlineCallbacks def check_joined_room(self, room_id, user_id, current_state=None): diff --git a/synapse/event_auth.py b/synapse/event_auth.py index c81d8e6729..9adedbbb02 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -27,10 +27,11 @@ from synapse.types import UserID, get_domain_from_id logger = logging.getLogger(__name__) -def check(event, auth_events, do_sig_check=True, do_size_check=True): +def check(room_version, event, auth_events, do_sig_check=True, do_size_check=True): """ Checks if this event is correctly authed. Args: + room_version (str): the version of the room event: the event being checked. auth_events (dict: event-key -> event): the existing room state. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a4b771049c..5adbe7b538 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1189,7 +1189,9 @@ class FederationHandler(BaseHandler): # The remote hasn't signed it yet, obviously. We'll do the full checks # when we get the event back in `on_send_join_request` - yield self.auth.check_from_context(event, context, do_sig_check=False) + yield self.auth.check_from_context( + room_version, event, context, do_sig_check=False, + ) defer.returnValue(event) @@ -1388,7 +1390,9 @@ class FederationHandler(BaseHandler): try: # The remote hasn't signed it yet, obviously. We'll do the full checks # when we get the event back in `on_send_leave_request` - yield self.auth.check_from_context(event, context, do_sig_check=False) + yield self.auth.check_from_context( + room_version, event, context, do_sig_check=False, + ) except AuthError as e: logger.warn("Failed to create new leave %r because %s", event, e) raise e @@ -1683,7 +1687,7 @@ class FederationHandler(BaseHandler): auth_for_e[(EventTypes.Create, "")] = create_event try: - self.auth.check(e, auth_events=auth_for_e) + self.auth.check(room_version, e, auth_events=auth_for_e) except SynapseError as err: # we may get SynapseErrors here as well as AuthErrors. For # instance, there are a couple of (ancient) events in some @@ -1927,6 +1931,8 @@ class FederationHandler(BaseHandler): current_state = set(e.event_id for e in auth_events.values()) different_auth = event_auth_events - current_state + room_version = yield self.store.get_room_version(event.room_id) + if different_auth and not event.internal_metadata.is_outlier(): # Do auth conflict res. logger.info("Different auth: %s", different_auth) @@ -1951,8 +1957,6 @@ class FederationHandler(BaseHandler): (d.type, d.state_key): d for d in different_events if d }) - room_version = yield self.store.get_room_version(event.room_id) - new_state = yield self.state_handler.resolve_events( room_version, [list(local_view.values()), list(remote_view.values())], @@ -2052,7 +2056,7 @@ class FederationHandler(BaseHandler): ) try: - self.auth.check(event, auth_events=auth_events) + self.auth.check(room_version, event, auth_events=auth_events) except AuthError as e: logger.warn("Failed auth resolution for %r because %s", event, e) raise e @@ -2288,7 +2292,7 @@ class FederationHandler(BaseHandler): ) try: - yield self.auth.check_from_context(event, context) + yield self.auth.check_from_context(room_version, event, context) except AuthError as e: logger.warn("Denying new third party invite %r because %s", event, e) raise e @@ -2330,7 +2334,7 @@ class FederationHandler(BaseHandler): ) try: - self.auth.check_from_context(event, context) + self.auth.check_from_context(room_version, event, context) except AuthError as e: logger.warn("Denying third party invite %r because %s", event, e) raise e diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7aaa4fba33..10a7ed4c5e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -611,8 +611,13 @@ class EventCreationHandler(object): extra_users (list(UserID)): Any extra users to notify about event """ + if event.is_state() and (event.type, event.state_key) == (EventTypes.Create, ""): + room_version = event.content["room_version"] + else: + room_version = yield self.store.get_room_version(event.room_id) + try: - yield self.auth.check_from_context(event, context) + yield self.auth.check_from_context(room_version, event, context) except AuthError as err: logger.warn("Denying new event %r because %s", event, err) raise err diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index cb8c5f77dd..19b4ee35d2 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -123,7 +123,10 @@ class RoomCreationHandler(BaseHandler): token_id=requester.access_token_id, ) ) - yield self.auth.check_from_context(tombstone_event, tombstone_context) + old_room_version = yield self.store.get_room_version(old_room_id) + yield self.auth.check_from_context( + old_room_version, tombstone_event, tombstone_context, + ) yield self.clone_exiting_room( requester, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index e9ecb00277..2fca51d0b2 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -611,7 +611,7 @@ def resolve_events_with_store(room_version, state_sets, event_map, state_res_sto RoomVersions.VDH_TEST, RoomVersions.STATE_V2_TEST, RoomVersions.V2, ): return v2.resolve_events_with_store( - state_sets, event_map, state_res_store, + room_version, state_sets, event_map, state_res_store, ) else: # This should only happen if we added a version but forgot to add it to diff --git a/synapse/state/v1.py b/synapse/state/v1.py index 19e091ce3b..6d3afcae7c 100644 --- a/synapse/state/v1.py +++ b/synapse/state/v1.py @@ -21,7 +21,7 @@ from six import iteritems, iterkeys, itervalues from twisted.internet import defer from synapse import event_auth -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, RoomVersions from synapse.api.errors import AuthError logger = logging.getLogger(__name__) @@ -274,7 +274,11 @@ def _resolve_auth_events(events, auth_events): auth_events[(prev_event.type, prev_event.state_key)] = prev_event try: # The signatures have already been checked at this point - event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False) + event_auth.check( + RoomVersions.V1, event, auth_events, + do_sig_check=False, + do_size_check=False, + ) prev_event = event except AuthError: return prev_event @@ -286,7 +290,11 @@ def _resolve_normal_events(events, auth_events): for event in _ordered_events(events): try: # The signatures have already been checked at this point - event_auth.check(event, auth_events, do_sig_check=False, do_size_check=False) + event_auth.check( + RoomVersions.V1, event, auth_events, + do_sig_check=False, + do_size_check=False, + ) return event except AuthError: pass diff --git a/synapse/state/v2.py b/synapse/state/v2.py index 3573bb0028..650995c92c 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -29,10 +29,12 @@ logger = logging.getLogger(__name__) @defer.inlineCallbacks -def resolve_events_with_store(state_sets, event_map, state_res_store): +def resolve_events_with_store(room_version, state_sets, event_map, state_res_store): """Resolves the state using the v2 state resolution algorithm Args: + room_version (str): The room version + state_sets(list): List of dicts of (type, state_key) -> event_id, which are the different state groups to resolve. @@ -104,7 +106,7 @@ def resolve_events_with_store(state_sets, event_map, state_res_store): # Now sequentially auth each one resolved_state = yield _iterative_auth_checks( - sorted_power_events, unconflicted_state, event_map, + room_version, sorted_power_events, unconflicted_state, event_map, state_res_store, ) @@ -129,7 +131,7 @@ def resolve_events_with_store(state_sets, event_map, state_res_store): logger.debug("resolving remaining events") resolved_state = yield _iterative_auth_checks( - leftover_events, resolved_state, event_map, + room_version, leftover_events, resolved_state, event_map, state_res_store, ) @@ -350,11 +352,13 @@ def _reverse_topological_power_sort(event_ids, event_map, state_res_store, auth_ @defer.inlineCallbacks -def _iterative_auth_checks(event_ids, base_state, event_map, state_res_store): +def _iterative_auth_checks(room_version, event_ids, base_state, event_map, + state_res_store): """Sequentially apply auth checks to each event in given list, updating the state as it goes along. Args: + room_version (str) event_ids (list[str]): Ordered list of events to apply auth checks to base_state (dict[tuple[str, str], str]): The set of state to start with event_map (dict[str,FrozenEvent]) @@ -385,7 +389,7 @@ def _iterative_auth_checks(event_ids, base_state, event_map, state_res_store): try: event_auth.check( - event, auth_events, + room_version, event, auth_events, do_sig_check=False, do_size_check=False ) diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index 2e073a3afc..9a5c816927 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -19,7 +19,7 @@ from six.moves import zip import attr -from synapse.api.constants import EventTypes, JoinRules, Membership +from synapse.api.constants import EventTypes, JoinRules, Membership, RoomVersions from synapse.event_auth import auth_types_for_event from synapse.events import FrozenEvent from synapse.state.v2 import lexicographical_topological_sort, resolve_events_with_store @@ -539,6 +539,7 @@ class StateTestCase(unittest.TestCase): state_before = dict(state_at_event[prev_events[0]]) else: state_d = resolve_events_with_store( + RoomVersions.V2, [state_at_event[n] for n in prev_events], event_map=event_map, state_res_store=TestStateResolutionStore(event_map), @@ -685,6 +686,7 @@ class SimpleParamStateTestCase(unittest.TestCase): # Test that we correctly handle passing `None` as the event_map state_d = resolve_events_with_store( + RoomVersions.V2, [self.state_at_bob, self.state_at_charlie], event_map=None, state_res_store=TestStateResolutionStore(self.event_map), diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 411b4a9f86..7ee318e4e8 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -16,6 +16,7 @@ import unittest from synapse import event_auth +from synapse.api.constants import RoomVersions from synapse.api.errors import AuthError from synapse.events import FrozenEvent @@ -35,12 +36,16 @@ class EventAuthTestCase(unittest.TestCase): } # creator should be able to send state - event_auth.check(_random_state_event(creator), auth_events, do_sig_check=False) + event_auth.check( + RoomVersions.V1, _random_state_event(creator), auth_events, + do_sig_check=False, + ) # joiner should not be able to send state self.assertRaises( AuthError, event_auth.check, + RoomVersions.V1, _random_state_event(joiner), auth_events, do_sig_check=False, @@ -69,13 +74,17 @@ class EventAuthTestCase(unittest.TestCase): self.assertRaises( AuthError, event_auth.check, + RoomVersions.V1, _random_state_event(pleb), auth_events, do_sig_check=False, ), # king should be able to send state - event_auth.check(_random_state_event(king), auth_events, do_sig_check=False) + event_auth.check( + RoomVersions.V1, _random_state_event(king), auth_events, + do_sig_check=False, + ) # helpers for making events -- cgit 1.5.1 From d840019192769f900f6a1a5c768368a080e651cd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 28 Jan 2019 09:56:59 +0000 Subject: Fix idna and ipv6 literal handling in MatrixFederationAgent (#4487) Turns out that the library does a better job of parsing URIs than our reinvented wheel. Who knew. There are two things going on here. The first is that, unlike parse_server_name, URI.fromBytes will strip off square brackets from IPv6 literals, which means that it is valid input to ClientTLSOptionsFactory and HostnameEndpoint. The second is that we stay in `bytes` throughout (except for the argument to ClientTLSOptionsFactory), which avoids the weirdness of (sometimes) ending up with idna-encoded values being held in `unicode` variables. TBH it probably would have been ok but it made the tests fragile. --- changelog.d/4487.misc | 1 + synapse/http/federation/matrix_federation_agent.py | 23 +-- .../federation/test_matrix_federation_agent.py | 181 ++++++++++++++++++++- 3 files changed, 193 insertions(+), 12 deletions(-) create mode 100644 changelog.d/4487.misc (limited to 'tests') diff --git a/changelog.d/4487.misc b/changelog.d/4487.misc new file mode 100644 index 0000000000..79de8eb3ad --- /dev/null +++ b/changelog.d/4487.misc @@ -0,0 +1 @@ +Fix idna and ipv6 literal handling in MatrixFederationAgent diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 1788e9a34a..9526f39cca 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -22,7 +22,6 @@ from twisted.web.client import URI, Agent, HTTPConnectionPool from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent -from synapse.http.endpoint import parse_server_name from synapse.http.federation.srv_resolver import SrvResolver, pick_server_from_list from synapse.util.logcontext import make_deferred_yieldable @@ -87,9 +86,7 @@ class MatrixFederationAgent(object): from being sent). """ - parsed_uri = URI.fromBytes(uri) - server_name_bytes = parsed_uri.netloc - host, port = parse_server_name(server_name_bytes.decode("ascii")) + parsed_uri = URI.fromBytes(uri, defaultPort=-1) # XXX disabling TLS is really only supported here for the benefit of the # unit tests. We should make the UTs cope with TLS rather than having to make @@ -97,16 +94,20 @@ class MatrixFederationAgent(object): if self._tls_client_options_factory is None: tls_options = None else: - tls_options = self._tls_client_options_factory.get_options(host) + tls_options = self._tls_client_options_factory.get_options( + parsed_uri.host.decode("ascii") + ) - if port is not None: - target = (host, port) + if parsed_uri.port != -1: + # there was an explicit port in the URI + target = parsed_uri.host, parsed_uri.port else: - service_name = b"_matrix._tcp.%s" % (server_name_bytes, ) + service_name = b"_matrix._tcp.%s" % (parsed_uri.host, ) server_list = yield self._srv_resolver.resolve_service(service_name) if not server_list: - target = (host, 8448) - logger.debug("No SRV record for %s, using %s", host, target) + target = (parsed_uri.host, 8448) + logger.debug( + "No SRV record for %s, using %s", service_name, target) else: target = pick_server_from_list(server_list) @@ -117,7 +118,7 @@ class MatrixFederationAgent(object): headers = headers.copy() if not headers.hasHeader(b'host'): - headers.addRawHeader(b'host', server_name_bytes) + headers.addRawHeader(b'host', parsed_uri.netloc) class EndpointFactory(object): @staticmethod diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 261afb5f41..f144092a51 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -209,6 +209,95 @@ class MatrixFederationAgentTests(TestCase): self.reactor.pump((0.1,)) self.successResultOf(test_d) + def test_get_ipv6_address(self): + """ + Test the behaviour when the server name contains an explicit IPv6 address + (with no port) + """ + + # the SRV lookup will return an empty list (XXX: why do we even do an SRV lookup?) + self.mock_resolver.resolve_service.side_effect = lambda _: [] + + # then there will be a getaddrinfo on the IP + self.reactor.lookups["::1"] = "::1" + + test_d = self._make_get_request(b"matrix://[::1]/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.::1", + ) + + # Make sure treq is trying to connect + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '::1') + self.assertEqual(port, 8448) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=None, + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'[::1]'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + + def test_get_ipv6_address_with_port(self): + """ + Test the behaviour when the server name contains an explicit IPv6 address + (with explicit port) + """ + + # there will be a getaddrinfo on the IP + self.reactor.lookups["::1"] = "::1" + + test_d = self._make_get_request(b"matrix://[::1]:80/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + # Make sure treq is trying to connect + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '::1') + self.assertEqual(port, 80) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=None, + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'[::1]:80'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + def test_get_hostname_no_srv(self): """ Test the behaviour when the server name has no port, and no SRV record @@ -258,7 +347,7 @@ class MatrixFederationAgentTests(TestCase): Test the behaviour when there is a single SRV record """ self.mock_resolver.resolve_service.side_effect = lambda _: [ - Server(host="srvtarget", port=8443) + Server(host=b"srvtarget", port=8443) ] self.reactor.lookups["srvtarget"] = "1.2.3.4" @@ -298,6 +387,96 @@ class MatrixFederationAgentTests(TestCase): self.reactor.pump((0.1,)) self.successResultOf(test_d) + def test_idna_servername(self): + """test the behaviour when the server name has idna chars in""" + + self.mock_resolver.resolve_service.side_effect = lambda _: [] + + # hostnameendpoint does the lookup on the unicode value (getaddrinfo encodes + # it back to idna) + self.reactor.lookups[u"bücher.com"] = "1.2.3.4" + + # this is idna for bücher.com + test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.xn--bcher-kva.com", + ) + + # Make sure treq is trying to connect + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 8448) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'xn--bcher-kva.com', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'xn--bcher-kva.com'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + + def test_idna_srv_target(self): + """test the behaviour when the target of a SRV record has idna chars""" + + self.mock_resolver.resolve_service.side_effect = lambda _: [ + Server(host=b"xn--trget-3qa.com", port=8443) # târget.com + ] + self.reactor.lookups[u"târget.com"] = "1.2.3.4" + + test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.xn--bcher-kva.com", + ) + + # Make sure treq is trying to connect + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 8443) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'xn--bcher-kva.com', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'xn--bcher-kva.com'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + def _check_logcontext(context): current = LoggingContext.current_context() -- cgit 1.5.1 From 0fd5b3b53e312a48d98afec27ff3686d8ffce199 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 26 Jan 2019 21:48:50 +0000 Subject: Handle IP literals explicitly We don't want to be doing .well-known lookups on these guys. --- synapse/http/federation/matrix_federation_agent.py | 19 +++++++++++++++++++ tests/http/federation/test_matrix_federation_agent.py | 19 ++----------------- 2 files changed, 21 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 2a8bceed9a..4d674cdb93 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -15,6 +15,7 @@ import logging import attr +from netaddr import IPAddress from zope.interface import implementer from twisted.internet import defer @@ -137,6 +138,24 @@ class MatrixFederationAgent(object): Returns: Deferred[_RoutingResult] """ + # check for an IP literal + try: + ip_address = IPAddress(parsed_uri.host.decode("ascii")) + except Exception: + # not an IP address + ip_address = None + + if ip_address: + port = parsed_uri.port + if port == -1: + port = 8448 + defer.returnValue(_RoutingResult( + host_header=parsed_uri.netloc, + tls_server_name=parsed_uri.host, + target_host=parsed_uri.host, + target_port=port, + )) + if parsed_uri.port != -1: # there is an explicit port defer.returnValue(_RoutingResult( diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index f144092a51..8257594fb8 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -166,11 +166,7 @@ class MatrixFederationAgentTests(TestCase): """ Test the behaviour when the server name contains an explicit IP (with no port) """ - - # the SRV lookup will return an empty list (XXX: why do we even do an SRV lookup?) - self.mock_resolver.resolve_service.side_effect = lambda _: [] - - # then there will be a getaddrinfo on the IP + # there will be a getaddrinfo on the IP self.reactor.lookups["1.2.3.4"] = "1.2.3.4" test_d = self._make_get_request(b"matrix://1.2.3.4/foo/bar") @@ -178,10 +174,6 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.1.2.3.4", - ) - # Make sure treq is trying to connect clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) @@ -215,10 +207,7 @@ class MatrixFederationAgentTests(TestCase): (with no port) """ - # the SRV lookup will return an empty list (XXX: why do we even do an SRV lookup?) - self.mock_resolver.resolve_service.side_effect = lambda _: [] - - # then there will be a getaddrinfo on the IP + # there will be a getaddrinfo on the IP self.reactor.lookups["::1"] = "::1" test_d = self._make_get_request(b"matrix://[::1]/foo/bar") @@ -226,10 +215,6 @@ class MatrixFederationAgentTests(TestCase): # Nothing happened yet self.assertNoResult(test_d) - self.mock_resolver.resolve_service.assert_called_once_with( - b"_matrix._tcp.::1", - ) - # Make sure treq is trying to connect clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) -- cgit 1.5.1 From 7072fe30846c47849c3cb142acef03ef3825a892 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 Jan 2019 15:43:32 +0000 Subject: Fix UPSERTs on SQLite 3.24+ (#4477) --- changelog.d/4306.misc | 2 +- changelog.d/4471.misc | 2 +- changelog.d/4477.misc | 1 + synapse/storage/_base.py | 10 ++++++++-- synapse/storage/engines/sqlite.py | 10 +++------- synapse/storage/monthly_active_users.py | 12 +++++++++--- tests/storage/test_base.py | 7 +++++-- tests/storage/test_monthly_active_users.py | 4 ++-- 8 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 changelog.d/4477.misc (limited to 'tests') diff --git a/changelog.d/4306.misc b/changelog.d/4306.misc index 7f48b02fbf..58130b6190 100644 --- a/changelog.d/4306.misc +++ b/changelog.d/4306.misc @@ -1 +1 @@ -Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+. +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. diff --git a/changelog.d/4471.misc b/changelog.d/4471.misc index 7f48b02fbf..994801fd1e 100644 --- a/changelog.d/4471.misc +++ b/changelog.d/4471.misc @@ -1 +1 @@ -Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+. + Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. diff --git a/changelog.d/4477.misc b/changelog.d/4477.misc new file mode 100644 index 0000000000..58130b6190 --- /dev/null +++ b/changelog.d/4477.misc @@ -0,0 +1 @@ +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f62f70b9f1..5109bc3e2e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,7 +27,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.stringutils import exception_to_unicode @@ -196,6 +196,12 @@ class SQLBaseStore(object): # A set of tables that are not safe to use native upserts in. self._unsafe_to_upsert_tables = {"user_ips"} + # We add the user_directory_search table to the blacklist on SQLite + # because the existing search table does not have an index, making it + # unsafe to use native upserts. + if isinstance(self.database_engine, Sqlite3Engine): + self._unsafe_to_upsert_tables.add("user_directory_search") + if self.database_engine.can_native_upsert: # Check ASAP (and then later, every 1s) to see if we have finished # background updates of tables that aren't safe to update. @@ -230,7 +236,7 @@ class SQLBaseStore(object): self._unsafe_to_upsert_tables.discard("user_ips") # If there's any tables left to check, reschedule to run. - if self._unsafe_to_upsert_tables: + if self.updates: self._clock.call_later( 15.0, run_as_background_process, diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 31b8449ca1..059ab81055 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -33,14 +33,10 @@ class Sqlite3Engine(object): @property def can_native_upsert(self): """ - Do we support native UPSERTs? + Do we support native UPSERTs? This requires SQLite3 3.24+, plus some + more work we haven't done yet to tell what was inserted vs updated. """ - # SQLite3 3.24+ supports them, but empirically the unit tests don't work - # when its enabled. - # FIXME: Figure out what is wrong so we can re-enable native upserts - - # return self.module.sqlite_version_info >= (3, 24, 0) - return False + return self.module.sqlite_version_info >= (3, 24, 0) def check_database(self, txn): pass diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py index d6fc8edd4c..9e7e09b8c1 100644 --- a/synapse/storage/monthly_active_users.py +++ b/synapse/storage/monthly_active_users.py @@ -197,15 +197,21 @@ class MonthlyActiveUsersStore(SQLBaseStore): if is_support: return - is_insert = yield self.runInteraction( + yield self.runInteraction( "upsert_monthly_active_user", self.upsert_monthly_active_user_txn, user_id ) - if is_insert: - self.user_last_seen_monthly_active.invalidate((user_id,)) + user_in_mau = self.user_last_seen_monthly_active.cache.get( + (user_id,), + None, + update_metrics=False + ) + if user_in_mau is None: self.get_monthly_active_count.invalidate(()) + self.user_last_seen_monthly_active.invalidate((user_id,)) + def upsert_monthly_active_user_txn(self, txn, user_id): """Updates or inserts monthly active user member diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 452d76ddd5..f18db8c384 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -49,14 +49,17 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.db_pool.runWithConnection = runWithConnection config = Mock() - config._enable_native_upserts = False + config._disable_native_upserts = True config.event_cache_size = 1 config.database_config = {"name": "sqlite3"} + engine = create_engine(config.database_config) + fake_engine = Mock(wraps=engine) + fake_engine.can_native_upsert = False hs = TestHomeServer( "test", db_pool=self.db_pool, config=config, - database_engine=create_engine(config.database_config), + database_engine=fake_engine, ) self.datastore = SQLBaseStore(None, hs) diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 9605301b59..d6569a82bb 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -18,12 +18,12 @@ from twisted.internet import defer from synapse.api.constants import UserTypes -from tests.unittest import HomeserverTestCase +from tests import unittest FORTY_DAYS = 40 * 24 * 60 * 60 -class MonthlyActiveUsersTestCase(HomeserverTestCase): +class MonthlyActiveUsersTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver() -- cgit 1.5.1 From f2b553d65692ad6dffd54fc5e911ae76b3f27aeb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 29 Jan 2019 09:38:29 +0000 Subject: Use SimpleResolverComplexifier in tests (#4497) two reasons for this. One, it saves a bunch of boilerplate. Two, it squashes unicode to IDNA-in-a-`str` (even on python 3) in a way that it turns out we rely on to give consistent behaviour between python 2 and 3. --- changelog.d/4497.feature | 1 + .../federation/test_matrix_federation_agent.py | 7 ++-- tests/server.py | 41 +++++++--------------- 3 files changed, 17 insertions(+), 32 deletions(-) create mode 100644 changelog.d/4497.feature (limited to 'tests') diff --git a/changelog.d/4497.feature b/changelog.d/4497.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4497.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 8257594fb8..53b52ace59 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -377,9 +377,8 @@ class MatrixFederationAgentTests(TestCase): self.mock_resolver.resolve_service.side_effect = lambda _: [] - # hostnameendpoint does the lookup on the unicode value (getaddrinfo encodes - # it back to idna) - self.reactor.lookups[u"bücher.com"] = "1.2.3.4" + # the resolver is always called with the IDNA hostname as a native string. + self.reactor.lookups["xn--bcher-kva.com"] = "1.2.3.4" # this is idna for bücher.com test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar") @@ -424,7 +423,7 @@ class MatrixFederationAgentTests(TestCase): self.mock_resolver.resolve_service.side_effect = lambda _: [ Server(host=b"xn--trget-3qa.com", port=8443) # târget.com ] - self.reactor.lookups[u"târget.com"] = "1.2.3.4" + self.reactor.lookups["xn--trget-3qa.com"] = "1.2.3.4" test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar") diff --git a/tests/server.py b/tests/server.py index ed2a046ae6..6adcc73f91 100644 --- a/tests/server.py +++ b/tests/server.py @@ -8,11 +8,10 @@ import attr from zope.interface import implementer from twisted.internet import address, threads, udp -from twisted.internet._resolver import HostResolution -from twisted.internet.address import IPv4Address -from twisted.internet.defer import Deferred +from twisted.internet._resolver import SimpleResolverComplexifier +from twisted.internet.defer import Deferred, fail, succeed from twisted.internet.error import DNSLookupError -from twisted.internet.interfaces import IReactorPluggableNameResolver +from twisted.internet.interfaces import IReactorPluggableNameResolver, IResolverSimple from twisted.python.failure import Failure from twisted.test.proto_helpers import MemoryReactorClock from twisted.web.http import unquote @@ -227,30 +226,16 @@ class ThreadedMemoryReactorClock(MemoryReactorClock): def __init__(self): self._udp = [] - self.lookups = {} - - class Resolver(object): - def resolveHostName( - _self, - resolutionReceiver, - hostName, - portNumber=0, - addressTypes=None, - transportSemantics='TCP', - ): - - resolution = HostResolution(hostName) - resolutionReceiver.resolutionBegan(resolution) - if hostName not in self.lookups: - raise DNSLookupError("OH NO") - - resolutionReceiver.addressResolved( - IPv4Address('TCP', self.lookups[hostName], portNumber) - ) - resolutionReceiver.resolutionComplete() - return resolution - - self.nameResolver = Resolver() + lookups = self.lookups = {} + + @implementer(IResolverSimple) + class FakeResolver(object): + def getHostByName(self, name, timeout=None): + if name not in lookups: + return fail(DNSLookupError("OH NO: unknown %s" % (name, ))) + return succeed(lookups[name]) + + self.nameResolver = SimpleResolverComplexifier(FakeResolver()) super(ThreadedMemoryReactorClock, self).__init__() def listenUDP(self, port, protocol, interface='', maxPacketSize=8196): -- cgit 1.5.1 From 554ca58ea1729804f413630b98fd158f324041a0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 25 Jan 2019 17:17:38 +0000 Subject: Make add_hashes_and_signatures operate on dicts --- synapse/crypto/event_signing.py | 16 ++++------- tests/crypto/test_event_signing.py | 56 ++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 40 deletions(-) (limited to 'tests') diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d01ac5075c..1dfa727fcf 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -131,12 +131,12 @@ def compute_event_signature(event_dict, signature_name, signing_key): return redact_json["signatures"] -def add_hashes_and_signatures(event, signature_name, signing_key, +def add_hashes_and_signatures(event_dict, signature_name, signing_key, hash_algorithm=hashlib.sha256): """Add content hash and sign the event Args: - event_dict (EventBuilder): The event to add hashes to and sign + event_dict (dict): The event to add hashes to and sign signature_name (str): The name of the entity signing the event (typically the server's hostname). signing_key (syutil.crypto.SigningKey): The key to sign with @@ -144,16 +144,12 @@ def add_hashes_and_signatures(event, signature_name, signing_key, to hash the event """ - name, digest = compute_content_hash( - event.get_pdu_json(), hash_algorithm=hash_algorithm, - ) + name, digest = compute_content_hash(event_dict, hash_algorithm=hash_algorithm) - if not hasattr(event, "hashes"): - event.hashes = {} - event.hashes[name] = encode_base64(digest) + event_dict.setdefault("hashes", {})[name] = encode_base64(digest) - event.signatures = compute_event_signature( - event.get_pdu_json(), + event_dict["signatures"] = compute_event_signature( + event_dict, signature_name=signature_name, signing_key=signing_key, ) diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index b2536c1e69..71aa731439 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -18,7 +18,7 @@ import nacl.signing from unpaddedbase64 import decode_base64 from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events.builder import EventBuilder +from synapse.events import FrozenEvent from tests import unittest @@ -40,20 +40,18 @@ class EventSigningTestCase(unittest.TestCase): self.signing_key.version = KEY_VER def test_sign_minimal(self): - builder = EventBuilder( - { - 'event_id': "$0:domain", - 'origin': "domain", - 'origin_server_ts': 1000000, - 'signatures': {}, - 'type': "X", - 'unsigned': {'age_ts': 1000000}, - } - ) + event_dict = { + 'event_id': "$0:domain", + 'origin': "domain", + 'origin_server_ts': 1000000, + 'signatures': {}, + 'type': "X", + 'unsigned': {'age_ts': 1000000}, + } - add_hashes_and_signatures(builder, HOSTNAME, self.signing_key) + add_hashes_and_signatures(event_dict, HOSTNAME, self.signing_key) - event = builder.build() + event = FrozenEvent(event_dict) self.assertTrue(hasattr(event, 'hashes')) self.assertIn('sha256', event.hashes) @@ -71,23 +69,21 @@ class EventSigningTestCase(unittest.TestCase): ) def test_sign_message(self): - builder = EventBuilder( - { - 'content': {'body': "Here is the message content"}, - 'event_id': "$0:domain", - 'origin': "domain", - 'origin_server_ts': 1000000, - 'type': "m.room.message", - 'room_id': "!r:domain", - 'sender': "@u:domain", - 'signatures': {}, - 'unsigned': {'age_ts': 1000000}, - } - ) - - add_hashes_and_signatures(builder, HOSTNAME, self.signing_key) - - event = builder.build() + event_dict = { + 'content': {'body': "Here is the message content"}, + 'event_id': "$0:domain", + 'origin': "domain", + 'origin_server_ts': 1000000, + 'type': "m.room.message", + 'room_id': "!r:domain", + 'sender': "@u:domain", + 'signatures': {}, + 'unsigned': {'age_ts': 1000000}, + } + + add_hashes_and_signatures(event_dict, HOSTNAME, self.signing_key) + + event = FrozenEvent(event_dict) self.assertTrue(hasattr(event, 'hashes')) self.assertIn('sha256', event.hashes) -- cgit 1.5.1 From aee39f7de8c79acb780c9903dd17f7d0b6a2b760 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 29 Jan 2019 11:19:50 +0000 Subject: Fix test to use valid event format --- tests/test_visibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/test_visibility.py b/tests/test_visibility.py index 82d63ce00e..455db9f276 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -166,7 +166,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase): @defer.inlineCallbacks def inject_message(self, user_id, content=None): if content is None: - content = {"body": "testytest"} + content = {"body": "testytest", "msgtype": "m.text"} builder = self.event_builder_factory.new( RoomVersions.V1, { -- cgit 1.5.1 From 5488cadaae37af4e6d4f847f6eddb38eca6e7c04 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 29 Jan 2019 12:07:00 +0000 Subject: Enable configuring test log level via env var (#4506) I got fed up with always adding '@unittest.DEBUG' every time I needed to debug a test. --- changelog.d/4506.misc | 1 + tests/test_utils/__init__.py | 18 +++++++++++++ tests/test_utils/logging_setup.py | 54 +++++++++++++++++++++++++++++++++++++++ tests/unittest.py | 37 +++++---------------------- 4 files changed, 80 insertions(+), 30 deletions(-) create mode 100644 changelog.d/4506.misc create mode 100644 tests/test_utils/__init__.py create mode 100644 tests/test_utils/logging_setup.py (limited to 'tests') diff --git a/changelog.d/4506.misc b/changelog.d/4506.misc new file mode 100644 index 0000000000..ea0e7d9580 --- /dev/null +++ b/changelog.d/4506.misc @@ -0,0 +1 @@ +Make it possible to set the log level for tests via an environment variable \ No newline at end of file diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 0000000000..a7310cf12a --- /dev/null +++ b/tests/test_utils/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utilities for running the unit tests +""" diff --git a/tests/test_utils/logging_setup.py b/tests/test_utils/logging_setup.py new file mode 100644 index 0000000000..d0bc8e2112 --- /dev/null +++ b/tests/test_utils/logging_setup.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os + +import twisted.logger + +from synapse.util.logcontext import LoggingContextFilter + + +class ToTwistedHandler(logging.Handler): + """logging handler which sends the logs to the twisted log""" + tx_log = twisted.logger.Logger() + + def emit(self, record): + log_entry = self.format(record) + log_level = record.levelname.lower().replace('warning', 'warn') + self.tx_log.emit( + twisted.logger.LogLevel.levelWithName(log_level), + log_entry.replace("{", r"(").replace("}", r")"), + ) + + +def setup_logging(): + """Configure the python logging appropriately for the tests. + + (Logs will end up in _trial_temp.) + """ + root_logger = logging.getLogger() + + log_format = ( + "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s" + ) + + handler = ToTwistedHandler() + formatter = logging.Formatter(log_format) + handler.setFormatter(formatter) + handler.addFilter(LoggingContextFilter(request="")) + root_logger.addHandler(handler) + + log_level = os.environ.get("SYNAPSE_TEST_LOG_LEVEL", "ERROR") + root_logger.setLevel(log_level) diff --git a/tests/unittest.py b/tests/unittest.py index cda549c783..fac254ff10 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -31,38 +31,14 @@ from synapse.http.server import JsonResource from synapse.http.site import SynapseRequest from synapse.server import HomeServer from synapse.types import UserID, create_requester -from synapse.util.logcontext import LoggingContext, LoggingContextFilter +from synapse.util.logcontext import LoggingContext from tests.server import get_clock, make_request, render, setup_test_homeserver +from tests.test_utils.logging_setup import setup_logging from tests.utils import default_config, setupdb setupdb() - -# Set up putting Synapse's logs into Trial's. -rootLogger = logging.getLogger() - -log_format = ( - "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s" -) - - -class ToTwistedHandler(logging.Handler): - tx_log = twisted.logger.Logger() - - def emit(self, record): - log_entry = self.format(record) - log_level = record.levelname.lower().replace('warning', 'warn') - self.tx_log.emit( - twisted.logger.LogLevel.levelWithName(log_level), - log_entry.replace("{", r"(").replace("}", r")"), - ) - - -handler = ToTwistedHandler() -formatter = logging.Formatter(log_format) -handler.setFormatter(formatter) -handler.addFilter(LoggingContextFilter(request="")) -rootLogger.addHandler(handler) +setup_logging() def around(target): @@ -96,7 +72,7 @@ class TestCase(unittest.TestCase): method = getattr(self, methodName) - level = getattr(method, "loglevel", getattr(self, "loglevel", logging.WARNING)) + level = getattr(method, "loglevel", getattr(self, "loglevel", None)) @around(self) def setUp(orig): @@ -114,7 +90,7 @@ class TestCase(unittest.TestCase): ) old_level = logging.getLogger().level - if old_level != level: + if level is not None and old_level != level: @around(self) def tearDown(orig): @@ -122,7 +98,8 @@ class TestCase(unittest.TestCase): logging.getLogger().setLevel(old_level) return ret - logging.getLogger().setLevel(level) + logging.getLogger().setLevel(level) + return orig() @around(self) -- cgit 1.5.1 From 99e36d5e243059a4c92a769a2920714b2d4c84d9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 29 Jan 2019 13:53:02 +0000 Subject: Implement MSC1708 (.well-known lookups for server routing) (#4489) --- MANIFEST.in | 1 + changelog.d/4408.feature | 1 + changelog.d/4408.misc | 1 - changelog.d/4409.feature | 1 + changelog.d/4409.misc | 1 - changelog.d/4426.feature | 1 + changelog.d/4426.misc | 1 - changelog.d/4427.feature | 1 + changelog.d/4427.misc | 1 - changelog.d/4428.feature | 1 + changelog.d/4428.misc | 1 - changelog.d/4464.feature | 1 + changelog.d/4464.misc | 1 - changelog.d/4468.feature | 1 + changelog.d/4468.misc | 1 - changelog.d/4487.feature | 1 + changelog.d/4487.misc | 1 - changelog.d/4489.feature | 1 + synapse/http/federation/matrix_federation_agent.py | 114 ++++++++++- tests/http/__init__.py | 42 ++++ .../federation/test_matrix_federation_agent.py | 223 ++++++++++++++++++++- tests/http/server.pem | 81 ++++++++ tests/server.py | 13 +- 23 files changed, 470 insertions(+), 21 deletions(-) create mode 100644 changelog.d/4408.feature delete mode 100644 changelog.d/4408.misc create mode 100644 changelog.d/4409.feature delete mode 100644 changelog.d/4409.misc create mode 100644 changelog.d/4426.feature delete mode 100644 changelog.d/4426.misc create mode 100644 changelog.d/4427.feature delete mode 100644 changelog.d/4427.misc create mode 100644 changelog.d/4428.feature delete mode 100644 changelog.d/4428.misc create mode 100644 changelog.d/4464.feature delete mode 100644 changelog.d/4464.misc create mode 100644 changelog.d/4468.feature delete mode 100644 changelog.d/4468.misc create mode 100644 changelog.d/4487.feature delete mode 100644 changelog.d/4487.misc create mode 100644 changelog.d/4489.feature create mode 100644 tests/http/server.pem (limited to 'tests') diff --git a/MANIFEST.in b/MANIFEST.in index 7e4c031b79..eb2de60f72 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,6 +15,7 @@ recursive-include docs * recursive-include scripts * recursive-include scripts-dev * recursive-include synapse *.pyi +recursive-include tests *.pem recursive-include tests *.py recursive-include synapse/res * diff --git a/changelog.d/4408.feature b/changelog.d/4408.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4408.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4408.misc b/changelog.d/4408.misc deleted file mode 100644 index 729bafd62e..0000000000 --- a/changelog.d/4408.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor 'sign_request' as 'build_auth_headers' \ No newline at end of file diff --git a/changelog.d/4409.feature b/changelog.d/4409.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4409.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4409.misc b/changelog.d/4409.misc deleted file mode 100644 index 9cf2adfbb1..0000000000 --- a/changelog.d/4409.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant federation connection wrapping code diff --git a/changelog.d/4426.feature b/changelog.d/4426.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4426.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4426.misc b/changelog.d/4426.misc deleted file mode 100644 index cda50438e0..0000000000 --- a/changelog.d/4426.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant SynapseKeyClientProtocol magic \ No newline at end of file diff --git a/changelog.d/4427.feature b/changelog.d/4427.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4427.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4427.misc b/changelog.d/4427.misc deleted file mode 100644 index 75500bdbc2..0000000000 --- a/changelog.d/4427.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor and cleanup for SRV record lookup diff --git a/changelog.d/4428.feature b/changelog.d/4428.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4428.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4428.misc b/changelog.d/4428.misc deleted file mode 100644 index 9a51434755..0000000000 --- a/changelog.d/4428.misc +++ /dev/null @@ -1 +0,0 @@ -Move SRV logic into the Agent layer diff --git a/changelog.d/4464.feature b/changelog.d/4464.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4464.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4464.misc b/changelog.d/4464.misc deleted file mode 100644 index 9a51434755..0000000000 --- a/changelog.d/4464.misc +++ /dev/null @@ -1 +0,0 @@ -Move SRV logic into the Agent layer diff --git a/changelog.d/4468.feature b/changelog.d/4468.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4468.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4468.misc b/changelog.d/4468.misc deleted file mode 100644 index 9a51434755..0000000000 --- a/changelog.d/4468.misc +++ /dev/null @@ -1 +0,0 @@ -Move SRV logic into the Agent layer diff --git a/changelog.d/4487.feature b/changelog.d/4487.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4487.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4487.misc b/changelog.d/4487.misc deleted file mode 100644 index 79de8eb3ad..0000000000 --- a/changelog.d/4487.misc +++ /dev/null @@ -1 +0,0 @@ -Fix idna and ipv6 literal handling in MatrixFederationAgent diff --git a/changelog.d/4489.feature b/changelog.d/4489.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4489.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 4a6f634c8b..07c72c9351 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import cgi +import json import logging import attr @@ -20,7 +22,7 @@ from zope.interface import implementer from twisted.internet import defer from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS -from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import URI, Agent, HTTPConnectionPool, readBody from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent @@ -43,13 +45,19 @@ class MatrixFederationAgent(object): tls_client_options_factory (ClientTLSOptionsFactory|None): factory to use for fetching client tls options, or none to disable TLS. + _well_known_tls_policy (IPolicyForHTTPS|None): + TLS policy to use for fetching .well-known files. None to use a default + (browser-like) implementation. + srv_resolver (SrvResolver|None): SRVResolver impl to use for looking up SRV records. None to use a default implementation. """ def __init__( - self, reactor, tls_client_options_factory, _srv_resolver=None, + self, reactor, tls_client_options_factory, + _well_known_tls_policy=None, + _srv_resolver=None, ): self._reactor = reactor self._tls_client_options_factory = tls_client_options_factory @@ -62,6 +70,14 @@ class MatrixFederationAgent(object): self._pool.maxPersistentPerHost = 5 self._pool.cachedConnectionTimeout = 2 * 60 + agent_args = {} + if _well_known_tls_policy is not None: + # the param is called 'contextFactory', but actually passing a + # contextfactory is deprecated, and it expects an IPolicyForHTTPS. + agent_args['contextFactory'] = _well_known_tls_policy + _well_known_agent = Agent(self._reactor, pool=self._pool, **agent_args) + self._well_known_agent = _well_known_agent + @defer.inlineCallbacks def request(self, method, uri, headers=None, bodyProducer=None): """ @@ -114,7 +130,11 @@ class MatrixFederationAgent(object): class EndpointFactory(object): @staticmethod def endpointForURI(_uri): - logger.info("Connecting to %s:%s", res.target_host, res.target_port) + logger.info( + "Connecting to %s:%i", + res.target_host.decode("ascii"), + res.target_port, + ) ep = HostnameEndpoint(self._reactor, res.target_host, res.target_port) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) @@ -127,7 +147,7 @@ class MatrixFederationAgent(object): defer.returnValue(res) @defer.inlineCallbacks - def _route_matrix_uri(self, parsed_uri): + def _route_matrix_uri(self, parsed_uri, lookup_well_known=True): """Helper for `request`: determine the routing for a Matrix URI Args: @@ -135,6 +155,9 @@ class MatrixFederationAgent(object): parsed with URI.fromBytes(uri, defaultPort=-1) to set the `port` to -1 if there is no explicit port given. + lookup_well_known (bool): True if we should look up the .well-known file if + there is no SRV record. + Returns: Deferred[_RoutingResult] """ @@ -169,6 +192,42 @@ class MatrixFederationAgent(object): service_name = b"_matrix._tcp.%s" % (parsed_uri.host,) server_list = yield self._srv_resolver.resolve_service(service_name) + if not server_list and lookup_well_known: + # try a .well-known lookup + well_known_server = yield self._get_well_known(parsed_uri.host) + + if well_known_server: + # if we found a .well-known, start again, but don't do another + # .well-known lookup. + + # parse the server name in the .well-known response into host/port. + # (This code is lifted from twisted.web.client.URI.fromBytes). + if b':' in well_known_server: + well_known_host, well_known_port = well_known_server.rsplit(b':', 1) + try: + well_known_port = int(well_known_port) + except ValueError: + # the part after the colon could not be parsed as an int + # - we assume it is an IPv6 literal with no port (the closing + # ']' stops it being parsed as an int) + well_known_host, well_known_port = well_known_server, -1 + else: + well_known_host, well_known_port = well_known_server, -1 + + new_uri = URI( + scheme=parsed_uri.scheme, + netloc=well_known_server, + host=well_known_host, + port=well_known_port, + path=parsed_uri.path, + params=parsed_uri.params, + query=parsed_uri.query, + fragment=parsed_uri.fragment, + ) + + res = yield self._route_matrix_uri(new_uri, lookup_well_known=False) + defer.returnValue(res) + if not server_list: target_host = parsed_uri.host port = 8448 @@ -190,6 +249,53 @@ class MatrixFederationAgent(object): target_port=port, )) + @defer.inlineCallbacks + def _get_well_known(self, server_name): + """Attempt to fetch and parse a .well-known file for the given server + + Args: + server_name (bytes): name of the server, from the requested url + + Returns: + Deferred[bytes|None]: either the new server name, from the .well-known, or + None if there was no .well-known file. + """ + # FIXME: add a cache + + uri = b"https://%s/.well-known/matrix/server" % (server_name, ) + logger.info("Fetching %s", uri.decode("ascii")) + try: + response = yield make_deferred_yieldable( + self._well_known_agent.request(b"GET", uri), + ) + except Exception as e: + logger.info( + "Connection error fetching %s: %s", + uri.decode("ascii"), e, + ) + defer.returnValue(None) + + body = yield make_deferred_yieldable(readBody(response)) + + if response.code != 200: + logger.info( + "Error response %i from %s: %s", + response.code, uri.decode("ascii"), body, + ) + defer.returnValue(None) + + content_types = response.headers.getRawHeaders(u'content-type') + if content_types is None: + raise Exception("no content-type header on .well-known response") + content_type, _opts = cgi.parse_header(content_types[-1]) + if content_type != 'application/json': + raise Exception("content-type not application/json on .well-known response") + parsed_body = json.loads(body.decode('utf-8')) + logger.info("Response from .well-known: %s", parsed_body) + if not isinstance(parsed_body, dict) or "m.server" not in parsed_body: + raise Exception("invalid .well-known response") + defer.returnValue(parsed_body["m.server"].encode("ascii")) + @attr.s class _RoutingResult(object): diff --git a/tests/http/__init__.py b/tests/http/__init__.py index e69de29bb2..ee8010f598 100644 --- a/tests/http/__init__.py +++ b/tests/http/__init__.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os.path + +from OpenSSL import SSL + + +def get_test_cert_file(): + """get the path to the test cert""" + + # the cert file itself is made with: + # + # openssl req -x509 -newkey rsa:4096 -keyout server.pem -out server.pem -days 36500 \ + # -nodes -subj '/CN=testserv' + return os.path.join( + os.path.dirname(__file__), + 'server.pem', + ) + + +class ServerTLSContext(object): + """A TLS Context which presents our test cert.""" + def __init__(self): + self.filename = get_test_cert_file() + + def getContext(self): + ctx = SSL.Context(SSL.TLSv1_METHOD) + ctx.use_certificate_file(self.filename) + ctx.use_privatekey_file(self.filename) + return ctx diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 53b52ace59..bd80dd0cb6 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -17,18 +17,21 @@ import logging from mock import Mock import treq +from zope.interface import implementer from twisted.internet import defer +from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions from twisted.internet.protocol import Factory from twisted.protocols.tls import TLSMemoryBIOFactory -from twisted.test.ssl_helpers import ServerTLSContext from twisted.web.http import HTTPChannel +from twisted.web.iweb import IPolicyForHTTPS from synapse.crypto.context_factory import ClientTLSOptionsFactory from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent from synapse.http.federation.srv_resolver import Server from synapse.util.logcontext import LoggingContext +from tests.http import ServerTLSContext from tests.server import FakeTransport, ThreadedMemoryReactorClock from tests.unittest import TestCase @@ -44,6 +47,7 @@ class MatrixFederationAgentTests(TestCase): self.agent = MatrixFederationAgent( reactor=self.reactor, tls_client_options_factory=ClientTLSOptionsFactory(None), + _well_known_tls_policy=TrustingTLSPolicyForHTTPS(), _srv_resolver=self.mock_resolver, ) @@ -65,10 +69,14 @@ class MatrixFederationAgentTests(TestCase): # Normally this would be done by the TCP socket code in Twisted, but we are # stubbing that out here. client_protocol = client_factory.buildProtocol(None) - client_protocol.makeConnection(FakeTransport(server_tls_protocol, self.reactor)) + client_protocol.makeConnection( + FakeTransport(server_tls_protocol, self.reactor, client_protocol), + ) # tell the server tls protocol to send its stuff back to the client, too - server_tls_protocol.makeConnection(FakeTransport(client_protocol, self.reactor)) + server_tls_protocol.makeConnection( + FakeTransport(client_protocol, self.reactor, server_tls_protocol), + ) # give the reactor a pump to get the TLS juices flowing. self.reactor.pump((0.1,)) @@ -101,9 +109,49 @@ class MatrixFederationAgentTests(TestCase): try: fetch_res = yield fetch_d defer.returnValue(fetch_res) + except Exception as e: + logger.info("Fetch of %s failed: %s", uri.decode("ascii"), e) + raise finally: _check_logcontext(context) + def _handle_well_known_connection(self, client_factory, expected_sni, target_server): + """Handle an outgoing HTTPs connection: wire it up to a server, check that the + request is for a .well-known, and send the response. + + Args: + client_factory (IProtocolFactory): outgoing connection + expected_sni (bytes): SNI that we expect the outgoing connection to send + target_server (bytes): target server that we should redirect to in the + .well-known response. + """ + # make the connection for .well-known + well_known_server = self._make_connection( + client_factory, + expected_sni=expected_sni, + ) + # check the .well-known request and send a response + self.assertEqual(len(well_known_server.requests), 1) + request = well_known_server.requests[0] + self._send_well_known_response(request, target_server) + + def _send_well_known_response(self, request, target_server): + """Check that an incoming request looks like a valid .well-known request, and + send back the response. + """ + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/.well-known/matrix/server') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'testserv'], + ) + # send back a response + request.responseHeaders.setRawHeaders(b'Content-Type', [b'application/json']) + request.write(b'{ "m.server": "%s" }' % (target_server,)) + request.finish() + + self.reactor.pump((0.1, )) + def test_get(self): """ happy-path test of a GET request with an explicit port @@ -283,9 +331,9 @@ class MatrixFederationAgentTests(TestCase): self.reactor.pump((0.1,)) self.successResultOf(test_d) - def test_get_hostname_no_srv(self): + def test_get_no_srv_no_well_known(self): """ - Test the behaviour when the server name has no port, and no SRV record + Test the behaviour when the server name has no port, no SRV, and no well-known """ self.mock_resolver.resolve_service.side_effect = lambda _: [] @@ -300,11 +348,24 @@ class MatrixFederationAgentTests(TestCase): b"_matrix._tcp.testserv", ) - # Make sure treq is trying to connect + # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) (host, port, client_factory, _timeout, _bindAddress) = clients[0] self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 443) + + # fonx the connection + client_factory.clientConnectionFailed(None, Exception("nope")) + + # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the + # .well-known request fails. + self.reactor.pump((0.4,)) + + # we should fall back to a direct connection + self.assertEqual(len(clients), 2) + (host, port, client_factory, _timeout, _bindAddress) = clients[1] + self.assertEqual(host, '1.2.3.4') self.assertEqual(port, 8448) # make a test server, and wire up the client @@ -327,6 +388,67 @@ class MatrixFederationAgentTests(TestCase): self.reactor.pump((0.1,)) self.successResultOf(test_d) + def test_get_well_known(self): + """Test the behaviour when the server name has no port and no SRV record, but + the .well-known redirects elsewhere + """ + + self.mock_resolver.resolve_service.side_effect = lambda _: [] + self.reactor.lookups["testserv"] = "1.2.3.4" + self.reactor.lookups["target-server"] = "1::f" + + test_d = self._make_get_request(b"matrix://testserv/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.testserv", + ) + self.mock_resolver.resolve_service.reset_mock() + + # there should be an attempt to connect on port 443 for the .well-known + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 443) + + self._handle_well_known_connection( + client_factory, expected_sni=b"testserv", target_server=b"target-server", + ) + + # there should be another SRV lookup + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.target-server", + ) + + # now we should get a connection to the target server + self.assertEqual(len(clients), 2) + (host, port, client_factory, _timeout, _bindAddress) = clients[1] + self.assertEqual(host, '1::f') + self.assertEqual(port, 8448) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'target-server', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'target-server'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + def test_get_hostname_srv(self): """ Test the behaviour when there is a single SRV record @@ -372,6 +494,71 @@ class MatrixFederationAgentTests(TestCase): self.reactor.pump((0.1,)) self.successResultOf(test_d) + def test_get_well_known_srv(self): + """Test the behaviour when the server name has no port and no SRV record, but + the .well-known redirects to a place where there is a SRV. + """ + + self.mock_resolver.resolve_service.side_effect = lambda _: [] + self.reactor.lookups["testserv"] = "1.2.3.4" + self.reactor.lookups["srvtarget"] = "5.6.7.8" + + test_d = self._make_get_request(b"matrix://testserv/foo/bar") + + # Nothing happened yet + self.assertNoResult(test_d) + + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.testserv", + ) + self.mock_resolver.resolve_service.reset_mock() + + # there should be an attempt to connect on port 443 for the .well-known + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 1) + (host, port, client_factory, _timeout, _bindAddress) = clients[0] + self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 443) + + self.mock_resolver.resolve_service.side_effect = lambda _: [ + Server(host=b"srvtarget", port=8443), + ] + + self._handle_well_known_connection( + client_factory, expected_sni=b"testserv", target_server=b"target-server", + ) + + # there should be another SRV lookup + self.mock_resolver.resolve_service.assert_called_once_with( + b"_matrix._tcp.target-server", + ) + + # now we should get a connection to the target of the SRV record + self.assertEqual(len(clients), 2) + (host, port, client_factory, _timeout, _bindAddress) = clients[1] + self.assertEqual(host, '5.6.7.8') + self.assertEqual(port, 8443) + + # make a test server, and wire up the client + http_server = self._make_connection( + client_factory, + expected_sni=b'target-server', + ) + + self.assertEqual(len(http_server.requests), 1) + request = http_server.requests[0] + self.assertEqual(request.method, b'GET') + self.assertEqual(request.path, b'/foo/bar') + self.assertEqual( + request.requestHeaders.getRawHeaders(b'host'), + [b'target-server'], + ) + + # finish the request + request.finish() + self.reactor.pump((0.1,)) + self.successResultOf(test_d) + def test_idna_servername(self): """test the behaviour when the server name has idna chars in""" @@ -390,11 +577,25 @@ class MatrixFederationAgentTests(TestCase): b"_matrix._tcp.xn--bcher-kva.com", ) - # Make sure treq is trying to connect + # there should be an attempt to connect on port 443 for the .well-known clients = self.reactor.tcpClients self.assertEqual(len(clients), 1) (host, port, client_factory, _timeout, _bindAddress) = clients[0] self.assertEqual(host, '1.2.3.4') + self.assertEqual(port, 443) + + # fonx the connection + client_factory.clientConnectionFailed(None, Exception("nope")) + + # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the + # .well-known request fails. + self.reactor.pump((0.4,)) + + # We should fall back to port 8448 + clients = self.reactor.tcpClients + self.assertEqual(len(clients), 2) + (host, port, client_factory, _timeout, _bindAddress) = clients[1] + self.assertEqual(host, '1.2.3.4') self.assertEqual(port, 8448) # make a test server, and wire up the client @@ -492,3 +693,11 @@ def _build_test_server(): def _log_request(request): """Implements Factory.log, which is expected by Request.finish""" logger.info("Completed request %s", request) + + +@implementer(IPolicyForHTTPS) +class TrustingTLSPolicyForHTTPS(object): + """An IPolicyForHTTPS which doesn't do any certificate verification""" + def creatorForNetloc(self, hostname, port): + certificateOptions = OpenSSLCertificateOptions() + return ClientTLSOptions(hostname, certificateOptions.getContext()) diff --git a/tests/http/server.pem b/tests/http/server.pem new file mode 100644 index 0000000000..0584cf1a80 --- /dev/null +++ b/tests/http/server.pem @@ -0,0 +1,81 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCgF43/3lAgJ+p0 +x7Rn8UcL8a4fctvdkikvZrCngw96LkB34Evfq8YGWlOVjU+f9naUJLAKMatmAfEN +r+rMX4VOXmpTwuu6iLtqwreUrRFMESyrmvQxa15p+y85gkY0CFmXMblv6ORbxHTG +ncBGwST4WK4Poewcgt6jcISFCESTUKu1zc3cw1ANIDRyDLB5K44KwIe36dcKckyN +Kdtv4BJ+3fcIZIkPJH62zqCypgFF1oiFt40uJzClxgHdJZlKYpgkfnDTckw4Y/Mx +9k8BbE310KAzUNMV9H7I1eEolzrNr66FQj1eN64X/dqO8lTbwCqAd4diCT4sIUk0 +0SVsAUjNd3g8j651hx+Qb1t8fuOjrny8dmeMxtUgIBHoQcpcj76R55Fs7KZ9uar0 +8OFTyGIze51W1jG2K/7/5M1zxIqrA+7lsXu5OR81s7I+Ng/UUAhiHA/z+42/aiNa +qEuk6tqj3rHfLctnCbtZ+JrRNqSSwEi8F0lMA021ivEd2eJV+284OyJjhXOmKHrX +QADHrmS7Sh4syTZvRNm9n+qWID0KdDr2Sji/KnS3Enp44HDQ4xriT6/xhwEGsyuX +oH5aAkdLznulbWkHBbyx1SUQSTLpOqzaioF9m1vRrLsFvrkrY3D253mPJ5eU9HM/ +dilduFcUgj4rz+6cdXUAh+KK/v95zwIDAQABAoICAFG5tJPaOa0ws0/KYx5s3YgL +aIhFalhCNSQtmCDrlwsYcXDA3/rfBchYdDL0YKGYgBBAal3J3WXFt/j0xThvyu2m +5UC9UPl4s7RckrsjXqEmY1d3UxGnbhtMT19cUdpeKN42VCP9EBaIw9Rg07dLAkSF +gNYaIx6q8F0fI4eGIPvTQtUcqur4CfWpaxyNvckdovV6M85/YXfDwbCOnacPDGIX +jfSK3i0MxGMuOHr6o8uzKR6aBUh6WStHWcw7VXXTvzdiFNbckmx3Gb93rf1b/LBw +QFfx+tBKcC62gKroCOzXso/0sL9YTVeSD/DJZOiJwSiz3Dj/3u1IUMbVvfTU8wSi +CYS7Z+jHxwSOCSSNTXm1wO/MtDsNKbI1+R0cohr/J9pOMQvrVh1+2zSDOFvXAQ1S +yvjn+uqdmijRoV2VEGVHd+34C+ci7eJGAhL/f92PohuuFR2shUETgGWzpACZSJwg +j1d90Hs81hj07vWRb+xCeDh00vimQngz9AD8vYvv/S4mqRGQ6TZdfjLoUwSTg0JD +6sQgRXX026gQhLhn687vLKZfHwzQPZkpQdxOR0dTZ/ho/RyGGRJXH4kN4cA2tPr+ +AKYQ29YXGlEzGG7OqikaZcprNWG6UFgEpuXyBxCgp9r4ladZo3J+1Rhgus8ZYatd +uO98q3WEBmP6CZ2n32mBAoIBAQDS/c/ybFTos0YpGHakwdmSfj5OOQJto2y8ywfG +qDHwO0ebcpNnS1+MA+7XbKUQb/3Iq7iJljkkzJG2DIJ6rpKynYts1ViYpM7M/t0T +W3V1gvUcUL62iqkgws4pnpWmubFkqV31cPSHcfIIclnzeQ1aOEGsGHNAvhty0ciC +DnkJACbqApvopFLOR5f6UFTtKExE+hDH0WqgpsCAKJ1L4g6pBzZatI32/CN9JEVU +tDbxLV75hHlFFjUrG7nT1rPyr/gI8Ceh9/2xeXPfjJUR0PrG3U1nwLqUCZkvFzO6 +XpN2+A+/v4v5xqMjKDKDFy1oq6SCMomwv/viw6wl/84TMbolAoIBAQDCPiMecnR8 +REik6tqVzQO/uSe9ZHjz6J15t5xdwaI6HpSwLlIkQPkLTjyXtFpemK5DOYRxrJvQ +remfrZrN2qtLlb/DKpuGPWRsPOvWCrSuNEp48ivUehtclljrzxAFfy0sM+fWeJ48 +nTnR+td9KNhjNtZixzWdAy/mE+jdaMsXVnk66L73Uz+2WsnvVMW2R6cpCR0F2eP/ +B4zDWRqlT2w47sePAB81mFYSQLvPC6Xcgg1OqMubfiizJI49c8DO6Jt+FFYdsxhd +kG52Eqa/Net6rN3ueiS6yXL5TU3Y6g96bPA2KyNCypucGcddcBfqaiVx/o4AH6yT +NrdsrYtyvk/jAoIBAQDHUwKVeeRJJbvdbQAArCV4MI155n+1xhMe1AuXkCQFWGtQ +nlBE4D72jmyf1UKnIbW2Uwv15xY6/ouVWYIWlj9+QDmMaozVP7Uiko+WDuwLRNl8 +k4dn+dzHV2HejbPBG2JLv3lFOx23q1zEwArcaXrExaq9Ayg2fKJ/uVHcFAIiD6Oz +pR1XDY4w1A/uaN+iYFSVQUyDCQLbnEz1hej73CaPZoHh9Pq83vxD5/UbjVjuRTeZ +L55FNzKpc/r89rNvTPBcuUwnxplDhYKDKVNWzn9rSXwrzTY2Tk8J3rh+k4RqevSd +6D47jH1n5Dy7/TRn0ueKHGZZtTUnyEUkbOJo3ayFAoIBAHKDyZaQqaX9Z8p6fwWj +yVsFoK0ih8BcWkLBAdmwZ6DWGJjJpjmjaG/G3ygc9s4gO1R8m12dAnuDnGE8KzDD +gwtbrKM2Alyg4wyA2hTlWOH/CAzH0RlCJ9Fs/d1/xJVJBeuyajLiB3/6vXTS6qnq +I7BSSxAPG8eGcn21LSsjNeB7ZZtaTgNnu/8ZBUYo9yrgkWc67TZe3/ChldYxOOlO +qqHh/BqNWtjxB4VZTp/g4RbgQVInZ2ozdXEv0v/dt0UEk29ANAjsZif7F3RayJ2f +/0TilzCaJ/9K9pKNhaClVRy7Dt8QjYg6BIWCGSw4ApF7pLnQ9gySn95mersCkVzD +YDsCggEAb0E/TORjQhKfNQvahyLfQFm151e+HIoqBqa4WFyfFxe/IJUaLH/JSSFw +VohbQqPdCmaAeuQ8ERL564DdkcY5BgKcax79fLLCOYP5bT11aQx6uFpfl2Dcm6Z9 +QdCRI4jzPftsd5fxLNH1XtGyC4t6vTic4Pji2O71WgWzx0j5v4aeDY4sZQeFxqCV +/q7Ee8hem1Rn5RFHu14FV45RS4LAWl6wvf5pQtneSKzx8YL0GZIRRytOzdEfnGKr +FeUlAj5uL+5/p0ZEgM7gPsEBwdm8scF79qSUn8UWSoXNeIauF9D4BDg8RZcFFxka +KILVFsq3cQC+bEnoM4eVbjEQkGs1RQ== +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIE/jCCAuagAwIBAgIJANFtVaGvJWZlMA0GCSqGSIb3DQEBCwUAMBMxETAPBgNV +BAMMCHRlc3RzZXJ2MCAXDTE5MDEyNzIyMDIzNloYDzIxMTkwMTAzMjIwMjM2WjAT +MREwDwYDVQQDDAh0ZXN0c2VydjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC +ggIBAKAXjf/eUCAn6nTHtGfxRwvxrh9y292SKS9msKeDD3ouQHfgS9+rxgZaU5WN +T5/2dpQksAoxq2YB8Q2v6sxfhU5ealPC67qIu2rCt5StEUwRLKua9DFrXmn7LzmC +RjQIWZcxuW/o5FvEdMadwEbBJPhYrg+h7ByC3qNwhIUIRJNQq7XNzdzDUA0gNHIM +sHkrjgrAh7fp1wpyTI0p22/gEn7d9whkiQ8kfrbOoLKmAUXWiIW3jS4nMKXGAd0l +mUpimCR+cNNyTDhj8zH2TwFsTfXQoDNQ0xX0fsjV4SiXOs2vroVCPV43rhf92o7y +VNvAKoB3h2IJPiwhSTTRJWwBSM13eDyPrnWHH5BvW3x+46OufLx2Z4zG1SAgEehB +ylyPvpHnkWzspn25qvTw4VPIYjN7nVbWMbYr/v/kzXPEiqsD7uWxe7k5HzWzsj42 +D9RQCGIcD/P7jb9qI1qoS6Tq2qPesd8ty2cJu1n4mtE2pJLASLwXSUwDTbWK8R3Z +4lX7bzg7ImOFc6YoetdAAMeuZLtKHizJNm9E2b2f6pYgPQp0OvZKOL8qdLcSenjg +cNDjGuJPr/GHAQazK5egfloCR0vOe6VtaQcFvLHVJRBJMuk6rNqKgX2bW9GsuwW+ +uStjcPbneY8nl5T0cz92KV24VxSCPivP7px1dQCH4or+/3nPAgMBAAGjUzBRMB0G +A1UdDgQWBBQcQZpzLzTk5KdS/Iz7sGCV7gTd/zAfBgNVHSMEGDAWgBQcQZpzLzTk +5KdS/Iz7sGCV7gTd/zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IC +AQAr/Pgha57jqYsDDX1LyRrVdqoVBpLBeB7x/p9dKYm7S6tBTDFNMZ0SZyQP8VEG +7UoC9/OQ9nCdEMoR7ZKpQsmipwcIqpXHS6l4YOkf5EEq5jpMgvlEesHmBJJeJew/ +FEPDl1bl8d0tSrmWaL3qepmwzA+2lwAAouWk2n+rLiP8CZ3jZeoTXFqYYrUlEqO9 +fHMvuWqTV4KCSyNY+GWCrnHetulgKHlg+W2J1mZnrCKcBhWf9C2DesTJO+JldIeM +ornTFquSt21hZi+k3aySuMn2N3MWiNL8XsZVsAnPSs0zA+2fxjJkShls8Gc7cCvd +a6XrNC+PY6pONguo7rEU4HiwbvnawSTngFFglmH/ImdA/HkaAekW6o82aI8/UxFx +V9fFMO3iKDQdOrg77hI1bx9RlzKNZZinE2/Pu26fWd5d2zqDWCjl8ykGQRAfXgYN +H3BjgyXLl+ao5/pOUYYtzm3ruTXTgRcy5hhL6hVTYhSrf9vYh4LNIeXNKnZ78tyG +TX77/kU2qXhBGCFEUUMqUNV/+ITir2lmoxVjknt19M07aGr8C7SgYt6Rs+qDpMiy +JurgvRh8LpVq4pHx1efxzxCFmo58DMrG40I0+CF3y/niNpOb1gp2wAqByRiORkds +f0ytW6qZ0TpHbD6gOtQLYDnhx3ISuX+QYSekVwQUpffeWQ== +-----END CERTIFICATE----- diff --git a/tests/server.py b/tests/server.py index 6adcc73f91..3d7ae9875c 100644 --- a/tests/server.py +++ b/tests/server.py @@ -354,6 +354,11 @@ class FakeTransport(object): :type: twisted.internet.interfaces.IReactorTime """ + _protocol = attr.ib(default=None) + """The Protocol which is producing data for this transport. Optional, but if set + will get called back for connectionLost() notifications etc. + """ + disconnecting = False buffer = attr.ib(default=b'') producer = attr.ib(default=None) @@ -364,8 +369,12 @@ class FakeTransport(object): def getHost(self): return None - def loseConnection(self): - self.disconnecting = True + def loseConnection(self, reason=None): + logger.info("FakeTransport: loseConnection(%s)", reason) + if not self.disconnecting: + self.disconnecting = True + if self._protocol: + self._protocol.connectionLost(reason) def abortConnection(self): self.disconnecting = True -- cgit 1.5.1 From 6bd43746367028849b0942bf7f8050a85175dffc Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 14:09:10 +0000 Subject: Do not generate self-signed TLS certificates by default. (#4509) --- changelog.d/4509.removal | 1 + synapse/config/tls.py | 61 +++++++++++------------------------ tests/config/test_generate.py | 2 -- tests/config/test_tls.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 45 deletions(-) create mode 100644 changelog.d/4509.removal create mode 100644 tests/config/test_tls.py (limited to 'tests') diff --git a/changelog.d/4509.removal b/changelog.d/4509.removal new file mode 100644 index 0000000000..9165009813 --- /dev/null +++ b/changelog.d/4509.removal @@ -0,0 +1 @@ +Synapse no longer generates self-signed TLS certificates when generating a configuration file. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index a75e233aa0..734f612db7 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -15,6 +15,7 @@ import logging import os +import warnings from datetime import datetime from hashlib import sha256 @@ -39,8 +40,8 @@ class TlsConfig(Config): self.acme_bind_addresses = acme_config.get("bind_addresses", ["127.0.0.1"]) self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30) - self.tls_certificate_file = os.path.abspath(config.get("tls_certificate_path")) - self.tls_private_key_file = os.path.abspath(config.get("tls_private_key_path")) + self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) + self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) self._original_tls_fingerprints = config["tls_fingerprints"] self.tls_fingerprints = list(self._original_tls_fingerprints) self.no_tls = config.get("no_tls", False) @@ -94,6 +95,16 @@ class TlsConfig(Config): """ self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file) + # Check if it is self-signed, and issue a warning if so. + if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject(): + warnings.warn( + ( + "Self-signed TLS certificates will not be accepted by Synapse 1.0. " + "Please either provide a valid certificate, or use Synapse's ACME " + "support to provision one." + ) + ) + if not self.no_tls: self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) @@ -118,10 +129,11 @@ class TlsConfig(Config): return ( """\ # PEM encoded X509 certificate for TLS. - # You can replace the self-signed certificate that synapse - # autogenerates on launch with your own SSL certificate + key pair - # if you like. Any required intermediary certificates can be - # appended after the primary certificate in hierarchical order. + # This certificate, as of Synapse 1.0, will need to be a valid + # and verifiable certificate, with a root that is available in + # the root store of other servers you wish to federate to. Any + # required intermediary certificates can be appended after the + # primary certificate in hierarchical order. tls_certificate_path: "%(tls_certificate_path)s" # PEM encoded private key for TLS @@ -183,40 +195,3 @@ class TlsConfig(Config): def read_tls_private_key(self, private_key_path): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - - def generate_files(self, config): - tls_certificate_path = config["tls_certificate_path"] - tls_private_key_path = config["tls_private_key_path"] - - if not self.path_exists(tls_private_key_path): - with open(tls_private_key_path, "wb") as private_key_file: - tls_private_key = crypto.PKey() - tls_private_key.generate_key(crypto.TYPE_RSA, 2048) - private_key_pem = crypto.dump_privatekey( - crypto.FILETYPE_PEM, tls_private_key - ) - private_key_file.write(private_key_pem) - else: - with open(tls_private_key_path) as private_key_file: - private_key_pem = private_key_file.read() - tls_private_key = crypto.load_privatekey( - crypto.FILETYPE_PEM, private_key_pem - ) - - if not self.path_exists(tls_certificate_path): - with open(tls_certificate_path, "wb") as certificate_file: - cert = crypto.X509() - subject = cert.get_subject() - subject.CN = config["server_name"] - - cert.set_serial_number(1000) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) - cert.set_issuer(cert.get_subject()) - cert.set_pubkey(tls_private_key) - - cert.sign(tls_private_key, 'sha256') - - cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - - certificate_file.write(cert_pem) diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py index b5ad99348d..795b4c298d 100644 --- a/tests/config/test_generate.py +++ b/tests/config/test_generate.py @@ -50,8 +50,6 @@ class ConfigGenerationTestCase(unittest.TestCase): "homeserver.yaml", "lemurs.win.log.config", "lemurs.win.signing.key", - "lemurs.win.tls.crt", - "lemurs.win.tls.key", ] ), set(os.listdir(self.dir)), diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py new file mode 100644 index 0000000000..4ccaf35603 --- /dev/null +++ b/tests/config/test_tls.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from synapse.config.tls import TlsConfig + +from tests.unittest import TestCase + + +class TLSConfigTests(TestCase): + + def test_warn_self_signed(self): + """ + Synapse will give a warning when it loads a self-signed certificate. + """ + config_dir = self.mktemp() + os.mkdir(config_dir) + with open(os.path.join(config_dir, "cert.pem"), 'w') as f: + f.write("""-----BEGIN CERTIFICATE----- +MIID6DCCAtACAws9CjANBgkqhkiG9w0BAQUFADCBtzELMAkGA1UEBhMCVFIxDzAN +BgNVBAgMBsOHb3J1bTEUMBIGA1UEBwwLQmHFn21ha8OnxLExEjAQBgNVBAMMCWxv +Y2FsaG9zdDEcMBoGA1UECgwTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECwwb +QXV0b21hdGVkIFRlc3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1 +cml0eUB0d2lzdGVkbWF0cml4LmNvbTAgFw0xNzA3MTIxNDAxNTNaGA8yMTE3MDYx +ODE0MDE1M1owgbcxCzAJBgNVBAYTAlRSMQ8wDQYDVQQIDAbDh29ydW0xFDASBgNV +BAcMC0JhxZ9tYWvDp8SxMRIwEAYDVQQDDAlsb2NhbGhvc3QxHDAaBgNVBAoME1R3 +aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNVBAsMG0F1dG9tYXRlZCBUZXN0aW5nIEF1 +dGhvcml0eTEpMCcGCSqGSIb3DQEJARYac2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5j +b20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDwT6kbqtMUI0sMkx4h +I+L780dA59KfksZCqJGmOsMD6hte9EguasfkZzvCF3dk3NhwCjFSOvKx6rCwiteo +WtYkVfo+rSuVNmt7bEsOUDtuTcaxTzIFB+yHOYwAaoz3zQkyVW0c4pzioiLCGCmf +FLdiDBQGGp74tb+7a0V6kC3vMLFoM3L6QWq5uYRB5+xLzlPJ734ltyvfZHL3Us6p +cUbK+3WTWvb4ER0W2RqArAj6Bc/ERQKIAPFEiZi9bIYTwvBH27OKHRz+KoY/G8zY ++l+WZoJqDhupRAQAuh7O7V/y6bSP+KNxJRie9QkZvw1PSaGSXtGJI3WWdO12/Ulg +epJpAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAJXEq5P9xwvP9aDkXIqzcD0L8sf8 +ewlhlxTQdeqt2Nace0Yk18lIo2oj1t86Y8jNbpAnZJeI813Rr5M7FbHCXoRc/SZG +I8OtG1xGwcok53lyDuuUUDexnK4O5BkjKiVlNPg4HPim5Kuj2hRNFfNt/F2BVIlj +iZupikC5MT1LQaRwidkSNxCku1TfAyueiBwhLnFwTmIGNnhuDCutEVAD9kFmcJN2 +SznugAcPk4doX2+rL+ila+ThqgPzIkwTUHtnmjI0TI6xsDUlXz5S3UyudrE2Qsfz +s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= +-----END CERTIFICATE-----""") + + config = { + "tls_certificate_path": os.path.join(config_dir, "cert.pem"), + "no_tls": True, + "tls_fingerprints": [] + } + + t = TlsConfig() + t.read_config(config) + t.read_certificate_from_disk() + + warnings = self.flushWarnings() + self.assertEqual(len(warnings), 1) + self.assertEqual( + warnings[0]["message"], + ( + "Self-signed TLS certificates will not be accepted by " + "Synapse 1.0. Please either provide a valid certificate, " + "or use Synapse's ACME support to provision one." + ) + ) -- cgit 1.5.1 From cc2d650ef72a7b1767888d2a778036de8a90e44d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 29 Jan 2019 16:49:17 +0000 Subject: Relax requirement for a content-type on .well-known (#4511) --- changelog.d/4511.feature | 1 + synapse/http/federation/matrix_federation_agent.py | 33 +++++++++------------- .../federation/test_matrix_federation_agent.py | 1 - 3 files changed, 14 insertions(+), 21 deletions(-) create mode 100644 changelog.d/4511.feature (limited to 'tests') diff --git a/changelog.d/4511.feature b/changelog.d/4511.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4511.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 07c72c9351..f81fcd4301 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import cgi import json import logging @@ -263,37 +262,31 @@ class MatrixFederationAgent(object): # FIXME: add a cache uri = b"https://%s/.well-known/matrix/server" % (server_name, ) - logger.info("Fetching %s", uri.decode("ascii")) + uri_str = uri.decode("ascii") + logger.info("Fetching %s", uri_str) try: response = yield make_deferred_yieldable( self._well_known_agent.request(b"GET", uri), ) except Exception as e: - logger.info( - "Connection error fetching %s: %s", - uri.decode("ascii"), e, - ) + logger.info("Connection error fetching %s: %s", uri_str, e) defer.returnValue(None) body = yield make_deferred_yieldable(readBody(response)) if response.code != 200: - logger.info( - "Error response %i from %s: %s", - response.code, uri.decode("ascii"), body, - ) + logger.info("Error response %i from %s", response.code, uri_str) defer.returnValue(None) - content_types = response.headers.getRawHeaders(u'content-type') - if content_types is None: - raise Exception("no content-type header on .well-known response") - content_type, _opts = cgi.parse_header(content_types[-1]) - if content_type != 'application/json': - raise Exception("content-type not application/json on .well-known response") - parsed_body = json.loads(body.decode('utf-8')) - logger.info("Response from .well-known: %s", parsed_body) - if not isinstance(parsed_body, dict) or "m.server" not in parsed_body: - raise Exception("invalid .well-known response") + try: + parsed_body = json.loads(body.decode('utf-8')) + logger.info("Response from .well-known: %s", parsed_body) + if not isinstance(parsed_body, dict): + raise Exception("not a dict") + if "m.server" not in parsed_body: + raise Exception("Missing key 'm.server'") + except Exception as e: + raise Exception("invalid .well-known response from %s: %s" % (uri_str, e,)) defer.returnValue(parsed_body["m.server"].encode("ascii")) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index bd80dd0cb6..11ea8ef10c 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -146,7 +146,6 @@ class MatrixFederationAgentTests(TestCase): [b'testserv'], ) # send back a response - request.responseHeaders.setRawHeaders(b'Content-Type', [b'application/json']) request.write(b'{ "m.server": "%s" }' % (target_server,)) request.finish() -- cgit 1.5.1