Federation: make_knock and send_knock implementations

Most of this is explained in the linked MSC (and don't miss the sequence
diagram in the MSC comments), but roughly knocking takes inspiration from
room joins and room invites. This commit is the room join stuff.

First the knocking homeserver POSTs to the make_knock endpoint on
another homeserver. The other homeserver will send back a knock event
that is valid for the knocking user and the room that they are knocking
on. The knocking homeserver will sign the event and send it back, before
the other homeserver takes that event and then sends it into the room on
the knocking homeserver's behalf.

It's worth noting that the accepting/rejecting knocks all happen over
existing room invite/leave flows. A homeserver rescinding its knock as
well is also just sending a leave.

Once the event has been inserted into the room, the homeserver that's in
the room will send back a 200 and an empty JSON dict to confirm
everything went well to the knocker. In a future commit, this dict will
instead be filled with some stripped state events from the room which
the knocking homeserver will pass back to the knocking user.

And yes, the logging statements in this commit are intentional. They're
consistent with the rest of the file :)

7 files changed, 359 insertions, 4 deletions

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 302b2f69bc..ffd1781252 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyrignt 2020 Sorunome
+# Copyrignt 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -539,7 +541,7 @@ class FederationClient(FederationBase):
 
             RuntimeError: if no servers were reachable.
         """
-        valid_memberships = {Membership.JOIN, Membership.LEAVE}
+        valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK}
         if membership not in valid_memberships:
             raise RuntimeError(
                 "make_membership_event called with membership='%s', must be one of %s"
@@ -879,6 +881,46 @@ class FederationClient(FederationBase):
         # content.
         return resp[1]
 
+    async def send_knock(self, destinations: List[str], pdu: EventBase):
+        """Attempts to send a knock event to given a list of servers. Iterates
+        through the list until one attempt succeeds.
+
+        Doing so will cause the remote server to add the event to the graph,
+        and send the event out to the rest of the federation.
+
+        Args:
+            destinations: A list of candidate homeservers which are likely to be
+                participating in the room.
+            pdu: The event to be sent.
+
+        Raises:
+            SynapseError: If the chosen remote server returns a 3xx/4xx code.
+            RuntimeError: If no servers were reachable.
+        """
+
+        async def send_request(destination: str) -> JsonDict:
+            return await self._do_send_knock(destination, pdu)
+
+        return await self._try_destination_list(
+            "send_knock", destinations, send_request
+        )
+
+    async def _do_send_knock(self, destination: str, pdu: EventBase):
+        """Send a knock event to a remote homeserver.
+
+        Args:
+            destination: The homeserver to send to.
+            pdu: The event to send.
+        """
+        time_now = self._clock.time_msec()
+
+        return await self.transport_layer.send_knock_v1(
+            destination=destination,
+            room_id=pdu.room_id,
+            event_id=pdu.event_id,
+            content=pdu.get_pdu_json(time_now),
+        )
+
     def get_public_rooms(
         self,
         remote_server: str,
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 23278e36b7..50c8a64aa8 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -565,6 +565,34 @@ class FederationServer(FederationBase):
         await self.handler.on_send_leave_request(origin, pdu)
         return {}
 
+    async def on_make_knock_request(self, origin: str, room_id: str, user_id: str):
+        origin_host, _ = parse_server_name(origin)
+        await self.check_server_matches_acl(origin_host, room_id)
+        pdu = await self.handler.on_make_knock_request(origin, room_id, user_id)
+
+        room_version = await self.store.get_room_version_id(room_id)
+
+        time_now = self._clock.time_msec()
+        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+
+    async def on_send_knock_request(self, origin: str, content: JsonDict, room_id: str):
+        logger.debug("on_send_knock_request: content: %s", content)
+
+        room_version = await self.store.get_room_version(room_id)
+        pdu = event_from_pdu_json(content, room_version)
+
+        origin_host, _ = parse_server_name(origin)
+        await self.check_server_matches_acl(origin_host, pdu.room_id)
+
+        logger.debug("on_send_knock_request: pdu sigs: %s", pdu.signatures)
+
+        pdu = await self._check_sigs_and_hash(room_version, pdu)
+
+        # Handle the event
+        await self.handler.on_send_knock_request(origin, pdu)
+
+        return {}
+
     async def on_event_auth(
         self, origin: str, room_id: str, event_id: str
     ) -> Tuple[int, Dict[str, Any]]:
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 17a10f622e..4650c009f9 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
+# Copyright 2020 Sorunome
+# Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,6 +28,7 @@ from synapse.api.urls import (
     FEDERATION_V2_PREFIX,
 )
 from synapse.logging.utils import log_function
+from synapse.types import JsonDict
 
 logger = logging.getLogger(__name__)
 
@@ -209,7 +212,7 @@ class TransportLayerClient:
             Fails with ``FederationDeniedError`` if the remote destination
             is not in our federation whitelist
         """
-        valid_memberships = {Membership.JOIN, Membership.LEAVE}
+        valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK}
         if membership not in valid_memberships:
             raise RuntimeError(
                 "make_membership_event called with membership='%s', must be one of %s"
@@ -294,6 +297,31 @@ class TransportLayerClient:
         return response
 
     @log_function
+    async def send_knock_v1(
+        self, destination: str, room_id: str, event_id: str, content: JsonDict,
+    ) -> JsonDict:
+        """
+        Sends an signed knock membership event to a remote server. This is the second
+        step knocking after make_knock.
+
+        Args:
+            destination: The remote homeserver.
+            room_id: The ID of the room to knock on.
+            event_id: The ID of the knock membership event that we're sending.
+            content: The knock membership event that we're sending. Note that this is not the
+                `content` field of the membership event, but the entire signed membership event
+                itself represented as a JSON dict.
+
+        Returns:
+            An empty JSON dictionary.
+        """
+        path = _create_v1_path("/send_knock/%s/%s", room_id, event_id)
+
+        return await self.client.put_json(
+            destination=destination, path=path, data=content
+        )
+
+    @log_function
     async def send_invite_v1(self, destination, room_id, event_id, content):
         path = _create_v1_path("/invite/%s/%s", room_id, event_id)
 
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index a0933fae88..133de2f3a1 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+# Copyright 2020 Sorunome
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -542,6 +543,22 @@ class FederationV2SendLeaveServlet(BaseFederationServlet):
         return 200, content
 
 
+class FederationMakeKnockServlet(BaseFederationServlet):
+    PATH = "/make_knock/(?P<context>[^/]*)/(?P<user_id>[^/]*)"
+
+    async def on_GET(self, origin, content, query, context, user_id):
+        content = await self.handler.on_make_knock_request(origin, context, user_id)
+        return 200, content
+
+
+class FederationV1MakeKnockServlet(BaseFederationServlet):
+    PATH = "/send_knock/(?P<room_id>[^/]*)/(?P<event_id>[^/]*)"
+
+    async def on_PUT(self, origin, content, query, room_id, event_id):
+        content = await self.handler.on_send_knock_request(origin, content, room_id)
+        return 200, content
+
+
 class FederationEventAuthServlet(BaseFederationServlet):
     PATH = "/event_auth/(?P<context>[^/]*)/(?P<event_id>[^/]*)"
 
@@ -1389,11 +1406,13 @@ FEDERATION_SERVLET_CLASSES = (
     FederationQueryServlet,
     FederationMakeJoinServlet,
     FederationMakeLeaveServlet,
+    FederationMakeKnockServlet,
     FederationEventServlet,
     FederationV1SendJoinServlet,
     FederationV2SendJoinServlet,
     FederationV1SendLeaveServlet,
     FederationV2SendLeaveServlet,
+    FederationV1MakeKnockServlet,
     FederationV1InviteServlet,
     FederationV2InviteServlet,
     FederationGetMissingEventsServlet,
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 69bc5ba44d..c89b5d61d4 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+# Copyright 2020 Sorunome
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -1438,6 +1439,61 @@ class FederationHandler(BaseHandler):
 
             run_in_background(self._handle_queued_pdus, room_queue)
 
+    @log_function
+    async def do_knock(
+        self, target_hosts: List[str], room_id: str, knockee: str, content: JsonDict,
+    ) -> Tuple[str, int]:
+        """Sends the knock to the remote server.
+
+        This first triggers a /make_knock request that returns a partial
+        event that we can fill out and sign. This is then sent to the
+        remote server via /send_knock.
+
+        Knock events must be signed by the knockee's server before distributing.
+
+        Args:
+            target_hosts: A list of hosts that we want to try knocking through.
+            room_id: The ID of the room to knock on.
+            knockee: The ID of the user who is knocking.
+            content: The content of the knock event.
+
+        Returns:
+            A tuple of (event ID, stream ID).
+
+        Raises:
+            SynapseError: If the chosen remote server returns a 3xx/4xx code.
+            RuntimeError: If no servers were reachable.
+        """
+        logger.debug("Knocking on room %s on behalf of user %s", room_id, knockee)
+
+        # Ask the remote server to create a valid knock event for us. Once received,
+        # we sign the event
+        origin, event, event_format_version = await self._make_and_verify_event(
+            target_hosts, room_id, knockee, "knock", content,
+        )
+
+        # Record the room ID and its version so that we have a record of the room
+        await self._maybe_store_room_on_outlier_membership(
+            room_id=event.room_id, room_version=event_format_version
+        )
+
+        # Initially try the host that we successfully called /make_knock on
+        try:
+            target_hosts.remove(origin)
+            target_hosts.insert(0, origin)
+        except ValueError:
+            pass
+
+        # Send the signed event back to the room
+        await self.federation_client.send_knock(target_hosts, event)
+
+        # Store the event locally
+        context = await self.state_handler.compute_event_context(event)
+        stream_id = await self.persist_events_and_notify(
+            event.room_id, [(event, context)]
+        )
+        return event.event_id, stream_id
+
     async def _handle_queued_pdus(self, room_queue):
         """Process PDUs which got queued up while we were busy send_joining.
 
@@ -1774,6 +1830,117 @@ class FederationHandler(BaseHandler):
 
         return None
 
+    @log_function
+    async def on_make_knock_request(
+        self, origin: str, room_id: str, user_id: str
+    ) -> EventBase:
+        """We've received a /make_knock/ request, so we create a partial
+        knock event for the room and return that. We do *not* persist or
+        process it until the other server has signed it and sent it back.
+
+        Args:
+            origin: The (verified) server name of the requesting server.
+            room_id: The room to create the knock event in.
+            user_id: The user to create the knock for.
+
+        Returns:
+            The partial knock event.
+        """
+        if get_domain_from_id(user_id) != origin:
+            logger.info(
+                "Get /make_knock request for user %r from different origin %s, ignoring",
+                user_id,
+                origin,
+            )
+            raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
+
+        room_version = await self.store.get_room_version_id(room_id)
+        builder = self.event_builder_factory.new(
+            room_version,
+            {
+                "type": EventTypes.Member,
+                "content": {"membership": Membership.KNOCK},
+                "room_id": room_id,
+                "sender": user_id,
+                "state_key": user_id,
+            },
+        )
+
+        event, context = await self.event_creation_handler.create_new_client_event(
+            builder=builder
+        )
+
+        event_allowed = await self.third_party_event_rules.check_event_allowed(
+            event, context
+        )
+        if not event_allowed:
+            logger.warning("Creation of knock %s forbidden by third-party rules", event)
+            raise SynapseError(
+                403, "This event is not allowed in this context", Codes.FORBIDDEN
+            )
+
+        try:
+            # The remote hasn't signed it yet, obviously. We'll do the full checks
+            # when we get the event back in `on_send_knock_request`
+            await self.auth.check_from_context(
+                room_version, event, context, do_sig_check=False
+            )
+        except AuthError as e:
+            logger.warning("Failed to create new knock %r because %s", event, e)
+            raise e
+
+        return event
+
+    @log_function
+    async def on_send_knock_request(self, origin: str, pdu: EventBase) -> EventContext:
+        """
+        We have received a knock event for a room. Verify that event and send it into the room
+        on the knocking homeserver's behalf.
+
+        Args:
+            origin: The remote homeserver of the knocking user.
+            pdu: The knocking member event that has been signed by the remote homeserver.
+
+        Returns:
+            The context of the event after inserting it into the room graph.
+        """
+        event = pdu
+
+        logger.debug(
+            "on_send_knock_request: Got event: %s, signatures: %s",
+            event.event_id,
+            event.signatures,
+        )
+
+        if get_domain_from_id(event.sender) != origin:
+            logger.info(
+                "Got /send_knock request for user %r from different origin %s",
+                event.sender,
+                origin,
+            )
+            raise SynapseError(403, "User not from origin", Codes.FORBIDDEN)
+
+        event.internal_metadata.outlier = False
+
+        context = await self._handle_new_event(origin, event)
+
+        event_allowed = await self.third_party_event_rules.check_event_allowed(
+            event, context
+        )
+        if not event_allowed:
+            logger.info("Sending of knock %s forbidden by third-party rules", event)
+            raise SynapseError(
+                403, "This event is not allowed in this context", Codes.FORBIDDEN
+            )
+
+        logger.debug(
+            "on_send_knock_request: After _handle_new_event: %s, sigs: %s",
+            event.event_id,
+            event.signatures,
+        )
+
+        return context
+
     async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase]:
         """Returns the state at the event. i.e. not including said event.
         """
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 7cd858b7db..2cce4b136e 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -111,6 +111,20 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
         raise NotImplementedError()
 
     @abc.abstractmethod
+    async def _remote_knock(
+        self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict,
+    ) -> Tuple[str, int]:
+        """Try and knock on a room that this server is not in
+
+        Args:
+            remote_room_hosts: List of servers that can be used to knock via.
+            room_id: Room that we are trying to knock on.
+            user: User who is trying to knock.
+            content: A dict that should be used as the content of the knock event.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
     async def remote_reject_invite(
         self,
         invite_event_id: str,
@@ -552,6 +566,23 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                 if len(latest_event_ids) == 0:
                     latest_event_ids = [invite.event_id]
 
+        elif effective_membership_state == Membership.KNOCK:
+            if not is_host_in_room:
+                # The knock needs to be sent over federation instead
+                remote_room_hosts.append(room_id.split(":", 1)[1])
+
+                content["membership"] = Membership.KNOCK
+
+                profile = self.profile_handler
+                if "displayname" not in content:
+                    content["displayname"] = await profile.get_displayname(target)
+                if "avatar_url" not in content:
+                    content["avatar_url"] = await profile.get_avatar_url(target)
+
+                return await self._remote_knock(
+                    remote_room_hosts, room_id, target, content
+                )
+
         return await self._local_membership_update(
             requester=requester,
             target=target,
@@ -1166,6 +1197,32 @@ class RoomMemberMasterHandler(RoomMemberHandler):
 
         return result_event.event_id, result_event.internal_metadata.stream_ordering
 
+    async def _remote_knock(
+        self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict,
+    ) -> Tuple[str, int]:
+        """Sends a knock to a room. Attempts to do so via one remote out of a given list.
+
+        Args:
+            remote_room_hosts: A list of homeservers to try knocking through.
+            room_id: The ID of the room to knock on.
+            user: The user to knock on behalf of.
+            content: The content of the knock event.
+
+        Returns:
+            A tuple of (event ID, stream ID).
+        """
+        # filter ourselves out of remote_room_hosts
+        remote_room_hosts = [
+            host for host in remote_room_hosts if host != self.hs.hostname
+        ]
+
+        if len(remote_room_hosts) == 0:
+            raise SynapseError(404, "No known servers")
+
+        return await self.federation_handler.do_knock(
+            remote_room_hosts, room_id, user.to_string(), content=content
+        )
+
     async def _user_left_room(self, target: UserID, room_id: str) -> None:
         """Implements RoomMemberHandler._user_left_room
         """
diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py
index f2e88f6a5b..f9cea604ec 100644
--- a/synapse/handlers/room_member_worker.py
+++ b/synapse/handlers/room_member_worker.py
@@ -79,6 +79,20 @@ class RoomMemberWorkerHandler(RoomMemberHandler):
         )
         return ret["event_id"], ret["stream_id"]
 
+    async def _remote_knock(
+        self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict,
+    ) -> Tuple[str, int]:
+        """Sends a knock to a room.
+
+        Implements RoomMemberHandler._remote_knock
+        """
+        return await self._remote_knock(
+            remote_room_hosts=remote_room_hosts,
+            room_id=room_id,
+            user=user,
+            content=content,
+        )
+
     async def _user_left_room(self, target: UserID, room_id: str) -> None:
         """Implements RoomMemberHandler._user_left_room
         """