From 09eff1b3db89ccc9a86d4e9ed160c96c67e9bd55 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 21 Oct 2021 14:50:43 +0200
Subject: fix relative link in docker readme (#11144)

relative links don't work when it's on dockerhub.
---
 changelog.d/11144.misc | 1 +
 docker/README.md       | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11144.misc

diff --git a/changelog.d/11144.misc b/changelog.d/11144.misc
new file mode 100644
index 0000000000..b5db109e2b
--- /dev/null
+++ b/changelog.d/11144.misc
@@ -0,0 +1 @@
+Fix broken link in the docker image README.
diff --git a/docker/README.md b/docker/README.md
index edf917bb11..38d182bf45 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -226,4 +226,5 @@ healthcheck:
 ## Using jemalloc
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.rst).
+You can read about jemalloc by reading the Synapse 
+[README](https://github.com/matrix-org/synapse/blob/HEAD/README.rst#help-synapse-is-slow-and-eats-all-my-ram-cpu).
-- 
cgit 1.5.1


From 0f9adc99ada1f66f4897c8164dcf509a955e5584 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Thu, 21 Oct 2021 09:07:07 -0400
Subject: Add missing type hints to synapse.crypto. (#11146)

And require type hints for this module.
---
 changelog.d/11146.misc            |  1 +
 mypy.ini                          |  3 +++
 synapse/crypto/context_factory.py | 40 +++++++++++++++++++++++++--------------
 synapse/crypto/event_signing.py   |  2 +-
 synapse/crypto/keyring.py         |  8 +++++---
 5 files changed, 36 insertions(+), 18 deletions(-)
 create mode 100644 changelog.d/11146.misc

diff --git a/changelog.d/11146.misc b/changelog.d/11146.misc
new file mode 100644
index 0000000000..6ce1c9f9f5
--- /dev/null
+++ b/changelog.d/11146.misc
@@ -0,0 +1 @@
+Add missing type hints to `synapse.crypto`.
diff --git a/mypy.ini b/mypy.ini
index 14d8bb8eaf..c5f44aea39 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -103,6 +103,9 @@ files =
 [mypy-synapse.api.*]
 disallow_untyped_defs = True
 
+[mypy-synapse.crypto.*]
+disallow_untyped_defs = True
+
 [mypy-synapse.events.*]
 disallow_untyped_defs = True
 
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 2a6110eb10..7855f3498b 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -29,9 +29,12 @@ from twisted.internet.ssl import (
     TLSVersion,
     platformTrust,
 )
+from twisted.protocols.tls import TLSMemoryBIOProtocol
 from twisted.python.failure import Failure
 from twisted.web.iweb import IPolicyForHTTPS
 
+from synapse.config.homeserver import HomeServerConfig
+
 logger = logging.getLogger(__name__)
 
 
@@ -51,7 +54,7 @@ class ServerContextFactory(ContextFactory):
     per https://github.com/matrix-org/synapse/issues/1691
     """
 
-    def __init__(self, config):
+    def __init__(self, config: HomeServerConfig):
         # TODO: once pyOpenSSL exposes TLS_METHOD and SSL_CTX_set_min_proto_version,
         # switch to those (see https://github.com/pyca/cryptography/issues/5379).
         #
@@ -64,7 +67,7 @@ class ServerContextFactory(ContextFactory):
         self.configure_context(self._context, config)
 
     @staticmethod
-    def configure_context(context, config):
+    def configure_context(context: SSL.Context, config: HomeServerConfig) -> None:
         try:
             _ecCurve = crypto.get_elliptic_curve(_defaultCurveName)
             context.set_tmp_ecdh(_ecCurve)
@@ -75,14 +78,15 @@ class ServerContextFactory(ContextFactory):
             SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_NO_TLSv1 | SSL.OP_NO_TLSv1_1
         )
         context.use_certificate_chain_file(config.tls.tls_certificate_file)
+        assert config.tls.tls_private_key is not None
         context.use_privatekey(config.tls.tls_private_key)
 
         # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
         context.set_cipher_list(
-            "ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:ECDH+AES128:!aNULL:!SHA1:!AESCCM"
+            b"ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:ECDH+AES128:!aNULL:!SHA1:!AESCCM"
         )
 
-    def getContext(self):
+    def getContext(self) -> SSL.Context:
         return self._context
 
 
@@ -98,7 +102,7 @@ class FederationPolicyForHTTPS:
     constructs an SSLClientConnectionCreator factory accordingly.
     """
 
-    def __init__(self, config):
+    def __init__(self, config: HomeServerConfig):
         self._config = config
 
         # Check if we're using a custom list of a CA certificates
@@ -131,7 +135,7 @@ class FederationPolicyForHTTPS:
             self._config.tls.federation_certificate_verification_whitelist
         )
 
-    def get_options(self, host: bytes):
+    def get_options(self, host: bytes) -> IOpenSSLClientConnectionCreator:
         # IPolicyForHTTPS.get_options takes bytes, but we want to compare
         # against the str whitelist. The hostnames in the whitelist are already
         # IDNA-encoded like the hosts will be here.
@@ -153,7 +157,9 @@ class FederationPolicyForHTTPS:
 
         return SSLClientConnectionCreator(host, ssl_context, should_verify)
 
-    def creatorForNetloc(self, hostname, port):
+    def creatorForNetloc(
+        self, hostname: bytes, port: int
+    ) -> IOpenSSLClientConnectionCreator:
         """Implements the IPolicyForHTTPS interface so that this can be passed
         directly to agents.
         """
@@ -169,16 +175,18 @@ class RegularPolicyForHTTPS:
     trust root.
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         trust_root = platformTrust()
         self._ssl_context = CertificateOptions(trustRoot=trust_root).getContext()
         self._ssl_context.set_info_callback(_context_info_cb)
 
-    def creatorForNetloc(self, hostname, port):
+    def creatorForNetloc(
+        self, hostname: bytes, port: int
+    ) -> IOpenSSLClientConnectionCreator:
         return SSLClientConnectionCreator(hostname, self._ssl_context, True)
 
 
-def _context_info_cb(ssl_connection, where, ret):
+def _context_info_cb(ssl_connection: SSL.Connection, where: int, ret: int) -> None:
     """The 'information callback' for our openssl context objects.
 
     Note: Once this is set as the info callback on a Context object, the Context should
@@ -204,11 +212,13 @@ class SSLClientConnectionCreator:
     Replaces twisted.internet.ssl.ClientTLSOptions
     """
 
-    def __init__(self, hostname: bytes, ctx, verify_certs: bool):
+    def __init__(self, hostname: bytes, ctx: SSL.Context, verify_certs: bool):
         self._ctx = ctx
         self._verifier = ConnectionVerifier(hostname, verify_certs)
 
-    def clientConnectionForTLS(self, tls_protocol):
+    def clientConnectionForTLS(
+        self, tls_protocol: TLSMemoryBIOProtocol
+    ) -> SSL.Connection:
         context = self._ctx
         connection = SSL.Connection(context, None)
 
@@ -219,7 +229,7 @@ class SSLClientConnectionCreator:
         # ... and we also gut-wrench a '_synapse_tls_verifier' attribute into the
         # tls_protocol so that the SSL context's info callback has something to
         # call to do the cert verification.
-        tls_protocol._synapse_tls_verifier = self._verifier
+        tls_protocol._synapse_tls_verifier = self._verifier  # type: ignore[attr-defined]
         return connection
 
 
@@ -244,7 +254,9 @@ class ConnectionVerifier:
         self._hostnameBytes = hostname
         self._hostnameASCII = self._hostnameBytes.decode("ascii")
 
-    def verify_context_info_cb(self, ssl_connection, where):
+    def verify_context_info_cb(
+        self, ssl_connection: SSL.Connection, where: int
+    ) -> None:
         if where & SSL.SSL_CB_HANDSHAKE_START and not self._is_ip_address:
             ssl_connection.set_tlsext_host_name(self._hostnameBytes)
 
diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py
index 0f2b632e47..7520647d1e 100644
--- a/synapse/crypto/event_signing.py
+++ b/synapse/crypto/event_signing.py
@@ -100,7 +100,7 @@ def compute_content_hash(
 
 
 def compute_event_reference_hash(
-    event, hash_algorithm: Hasher = hashlib.sha256
+    event: EventBase, hash_algorithm: Hasher = hashlib.sha256
 ) -> Tuple[str, bytes]:
     """Computes the event reference hash. This is the hash of the redacted
     event.
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index e1e13a2412..8628e951c4 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -87,7 +87,7 @@ class VerifyJsonRequest:
         server_name: str,
         json_object: JsonDict,
         minimum_valid_until_ms: int,
-    ):
+    ) -> "VerifyJsonRequest":
         """Create a VerifyJsonRequest to verify all signatures on a signed JSON
         object for the given server.
         """
@@ -104,7 +104,7 @@ class VerifyJsonRequest:
         server_name: str,
         event: EventBase,
         minimum_valid_until_ms: int,
-    ):
+    ) -> "VerifyJsonRequest":
         """Create a VerifyJsonRequest to verify all signatures on an event
         object for the given server.
         """
@@ -449,7 +449,9 @@ class StoreKeyFetcher(KeyFetcher):
 
         self.store = hs.get_datastore()
 
-    async def _fetch_keys(self, keys_to_fetch: List[_FetchKeyRequest]):
+    async def _fetch_keys(
+        self, keys_to_fetch: List[_FetchKeyRequest]
+    ) -> Dict[str, Dict[str, FetchKeyResult]]:
         key_ids_to_fetch = (
             (queue_value.server_name, key_id)
             for queue_value in keys_to_fetch
-- 
cgit 1.5.1


From 6408372234eef2d72a13ee838c07199751c56378 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Thu, 21 Oct 2021 17:42:25 +0100
Subject: Improve docstrings for methods related to sending EDUs to application
 services (#11138)

---
 changelog.d/11138.misc         |  1 +
 synapse/handlers/appservice.py | 94 ++++++++++++++++++++++++++++++++++++------
 synapse/handlers/device.py     |  4 ++
 synapse/handlers/presence.py   | 34 ++++++++++++---
 synapse/handlers/receipts.py   |  8 +++-
 synapse/handlers/typing.py     | 12 ++++--
 synapse/notifier.py            | 18 +++++++-
 7 files changed, 148 insertions(+), 23 deletions(-)
 create mode 100644 changelog.d/11138.misc

diff --git a/changelog.d/11138.misc b/changelog.d/11138.misc
new file mode 100644
index 0000000000..79b7776975
--- /dev/null
+++ b/changelog.d/11138.misc
@@ -0,0 +1 @@
+Add docstrings and comments to the application service ephemeral event sending code.
\ No newline at end of file
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 163278708c..36c206dae6 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -185,19 +185,26 @@ class ApplicationServicesHandler:
         new_token: Optional[int],
         users: Optional[Collection[Union[str, UserID]]] = None,
     ) -> None:
-        """This is called by the notifier in the background
-        when a ephemeral event handled by the homeserver.
-
-        This will determine which appservices
-        are interested in the event, and submit them.
+        """
+        This is called by the notifier in the background when an ephemeral event is handled
+        by the homeserver.
 
-        Events will only be pushed to appservices
-        that have opted into ephemeral events
+        This will determine which appservices are interested in the event, and submit them.
 
         Args:
             stream_key: The stream the event came from.
-            new_token: The latest stream token
-            users: The user(s) involved with the event.
+
+                `stream_key` can be "typing_key", "receipt_key" or "presence_key". Any other
+                value for `stream_key` will cause this function to return early.
+
+                Ephemeral events will only be pushed to appservices that have opted into
+                them.
+
+                Appservices will only receive ephemeral events that fall within their
+                registered user and room namespaces.
+
+            new_token: The latest stream token.
+            users: The users that should be informed of the new event, if any.
         """
         if not self.notify_appservices:
             return
@@ -232,21 +239,32 @@ class ApplicationServicesHandler:
             for service in services:
                 # Only handle typing if we have the latest token
                 if stream_key == "typing_key" and new_token is not None:
+                    # Note that we don't persist the token (via set_type_stream_id_for_appservice)
+                    # for typing_key due to performance reasons and due to their highly
+                    # ephemeral nature.
+                    #
+                    # Instead we simply grab the latest typing updates in _handle_typing
+                    # and, if they apply to this application service, send it off.
                     events = await self._handle_typing(service, new_token)
                     if events:
                         self.scheduler.submit_ephemeral_events_for_as(service, events)
-                    # We don't persist the token for typing_key for performance reasons
+
                 elif stream_key == "receipt_key":
                     events = await self._handle_receipts(service)
                     if events:
                         self.scheduler.submit_ephemeral_events_for_as(service, events)
+
+                    # Persist the latest handled stream token for this appservice
                     await self.store.set_type_stream_id_for_appservice(
                         service, "read_receipt", new_token
                     )
+
                 elif stream_key == "presence_key":
                     events = await self._handle_presence(service, users)
                     if events:
                         self.scheduler.submit_ephemeral_events_for_as(service, events)
+
+                    # Persist the latest handled stream token for this appservice
                     await self.store.set_type_stream_id_for_appservice(
                         service, "presence", new_token
                     )
@@ -254,18 +272,54 @@ class ApplicationServicesHandler:
     async def _handle_typing(
         self, service: ApplicationService, new_token: int
     ) -> List[JsonDict]:
+        """
+        Return the typing events since the given stream token that the given application
+        service should receive.
+
+        First fetch all typing events between the given typing stream token (non-inclusive)
+        and the latest typing event stream token (inclusive). Then return only those typing
+        events that the given application service may be interested in.
+
+        Args:
+            service: The application service to check for which events it should receive.
+            new_token: A typing event stream token.
+
+        Returns:
+            A list of JSON dictionaries containing data derived from the typing events that
+            should be sent to the given application service.
+        """
         typing_source = self.event_sources.sources.typing
         # Get the typing events from just before current
         typing, _ = await typing_source.get_new_events_as(
             service=service,
             # For performance reasons, we don't persist the previous
-            # token in the DB and instead fetch the latest typing information
+            # token in the DB and instead fetch the latest typing event
             # for appservices.
+            # TODO: It'd likely be more efficient to simply fetch the
+            #  typing event with the given 'new_token' stream token and
+            #  check if the given service was interested, rather than
+            #  iterating over all typing events and only grabbing the
+            #  latest few.
             from_key=new_token - 1,
         )
         return typing
 
     async def _handle_receipts(self, service: ApplicationService) -> List[JsonDict]:
+        """
+        Return the latest read receipts that the given application service should receive.
+
+        First fetch all read receipts between the last receipt stream token that this
+        application service should have previously received (non-inclusive) and the
+        latest read receipt stream token (inclusive). Then from that set, return only
+        those read receipts that the given application service may be interested in.
+
+        Args:
+            service: The application service to check for which events it should receive.
+
+        Returns:
+            A list of JSON dictionaries containing data derived from the read receipts that
+            should be sent to the given application service.
+        """
         from_key = await self.store.get_type_stream_id_for_appservice(
             service, "read_receipt"
         )
@@ -278,6 +332,22 @@ class ApplicationServicesHandler:
     async def _handle_presence(
         self, service: ApplicationService, users: Collection[Union[str, UserID]]
     ) -> List[JsonDict]:
+        """
+        Return the latest presence updates that the given application service should receive.
+
+        First, filter the given users list to those that the application service is
+        interested in. Then retrieve the latest presence updates since the
+        the last-known previously received presence stream token for the given
+        application service. Return those presence updates.
+
+        Args:
+            service: The application service that ephemeral events are being sent to.
+            users: The users that should receive the presence update.
+
+        Returns:
+            A list of json dictionaries containing data derived from the presence events
+            that should be sent to the given application service.
+        """
         events: List[JsonDict] = []
         presence_source = self.event_sources.sources.presence
         from_key = await self.store.get_type_stream_id_for_appservice(
@@ -290,9 +360,9 @@ class ApplicationServicesHandler:
             interested = await service.is_interested_in_presence(user, self.store)
             if not interested:
                 continue
+
             presence_events, _ = await presence_source.get_new_events(
                 user=user,
-                service=service,
                 from_key=from_key,
             )
             time_now = self.clock.time_msec()
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 6eafbea25d..68b446eb66 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -454,6 +454,10 @@ class DeviceHandler(DeviceWorkerHandler):
     ) -> None:
         """Notify that a user's device(s) has changed. Pokes the notifier, and
         remote servers if the user is local.
+
+        Args:
+            user_id: The Matrix ID of the user who's device list has been updated.
+            device_ids: The device IDs that have changed.
         """
         if not device_ids:
             # No changes to notify about, so this is a no-op.
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index b5968e047b..fdab50da37 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -52,7 +52,6 @@ import synapse.metrics
 from synapse.api.constants import EventTypes, Membership, PresenceState
 from synapse.api.errors import SynapseError
 from synapse.api.presence import UserPresenceState
-from synapse.appservice import ApplicationService
 from synapse.events.presence_router import PresenceRouter
 from synapse.logging.context import run_in_background
 from synapse.logging.utils import log_function
@@ -1483,11 +1482,37 @@ def should_notify(old_state: UserPresenceState, new_state: UserPresenceState) ->
 def format_user_presence_state(
     state: UserPresenceState, now: int, include_user_id: bool = True
 ) -> JsonDict:
-    """Convert UserPresenceState to a format that can be sent down to clients
+    """Convert UserPresenceState to a JSON format that can be sent down to clients
     and to other servers.
 
-    The "user_id" is optional so that this function can be used to format presence
-    updates for client /sync responses and for federation /send requests.
+    Args:
+        state: The user presence state to format.
+        now: The current timestamp since the epoch in ms.
+        include_user_id: Whether to include `user_id` in the returned dictionary.
+            As this function can be used both to format presence updates for client /sync
+            responses and for federation /send requests, only the latter needs the include
+            the `user_id` field.
+
+    Returns:
+        A JSON dictionary with the following keys:
+            * presence: The presence state as a str.
+            * user_id: Optional. Included if `include_user_id` is truthy. The canonical
+                Matrix ID of the user.
+            * last_active_ago: Optional. Included if `last_active_ts` is set on `state`.
+                The timestamp that the user was last active.
+            * status_msg: Optional. Included if `status_msg` is set on `state`. The user's
+                status.
+            * currently_active: Optional. Included only if `state.state` is "online".
+
+        Example:
+
+        {
+            "presence": "online",
+            "user_id": "@alice:example.com",
+            "last_active_ago": 16783813918,
+            "status_msg": "Hello world!",
+            "currently_active": True
+        }
     """
     content: JsonDict = {"presence": state.state}
     if include_user_id:
@@ -1526,7 +1551,6 @@ class PresenceEventSource(EventSource[int, UserPresenceState]):
         is_guest: bool = False,
         explicit_room_id: Optional[str] = None,
         include_offline: bool = True,
-        service: Optional[ApplicationService] = None,
     ) -> Tuple[List[UserPresenceState], int]:
         # The process for getting presence events are:
         #  1. Get the rooms the user is in.
diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py
index 374e961e3b..4911a11535 100644
--- a/synapse/handlers/receipts.py
+++ b/synapse/handlers/receipts.py
@@ -241,12 +241,18 @@ class ReceiptEventSource(EventSource[int, JsonDict]):
     async def get_new_events_as(
         self, from_key: int, service: ApplicationService
     ) -> Tuple[List[JsonDict], int]:
-        """Returns a set of new receipt events that an appservice
+        """Returns a set of new read receipt events that an appservice
         may be interested in.
 
         Args:
             from_key: the stream position at which events should be fetched from
             service: The appservice which may be interested
+
+        Returns:
+            A two-tuple containing the following:
+                * A list of json dictionaries derived from read receipts that the
+                  appservice may be interested in.
+                * The current read receipt stream token.
         """
         from_key = int(from_key)
         to_key = self.get_current_key()
diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py
index d10e9b8ec4..c411d69924 100644
--- a/synapse/handlers/typing.py
+++ b/synapse/handlers/typing.py
@@ -465,17 +465,23 @@ class TypingNotificationEventSource(EventSource[int, JsonDict]):
         may be interested in.
 
         Args:
-            from_key: the stream position at which events should be fetched from
-            service: The appservice which may be interested
+            from_key: the stream position at which events should be fetched from.
+            service: The appservice which may be interested.
+
+        Returns:
+            A two-tuple containing the following:
+                * A list of json dictionaries derived from typing events that the
+                  appservice may be interested in.
+                * The latest known room serial.
         """
         with Measure(self.clock, "typing.get_new_events_as"):
-            from_key = int(from_key)
             handler = self.get_typing_handler()
 
             events = []
             for room_id in handler._room_serials.keys():
                 if handler._room_serials[room_id] <= from_key:
                     continue
+
                 if not await service.matches_user_in_member_list(
                     room_id, handler.store
                 ):
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1a9f84ba45..1acd899fab 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -379,7 +379,14 @@ class Notifier:
         stream_key: str,
         new_token: Union[int, RoomStreamToken],
         users: Optional[Collection[Union[str, UserID]]] = None,
-    ):
+    ) -> None:
+        """Notify application services of ephemeral event activity.
+
+        Args:
+            stream_key: The stream the event came from.
+            new_token: The value of the new stream token.
+            users: The users that should be informed of the new event, if any.
+        """
         try:
             stream_token = None
             if isinstance(new_token, int):
@@ -402,10 +409,17 @@ class Notifier:
         new_token: Union[int, RoomStreamToken],
         users: Optional[Collection[Union[str, UserID]]] = None,
         rooms: Optional[Collection[str]] = None,
-    ):
+    ) -> None:
         """Used to inform listeners that something has happened event wise.
 
         Will wake up all listeners for the given users and rooms.
+
+        Args:
+            stream_key: The stream the event came from.
+            new_token: The value of the new stream token.
+            users: The users that should be informed of the new event.
+            rooms: A collection of room IDs for which each joined member will be
+                informed of the new event.
         """
         users = users or []
         rooms = rooms or []
-- 
cgit 1.5.1


From 2d91b6256e53a9e60027880b0407bd77cb653ad1 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 21 Oct 2021 17:48:59 +0100
Subject: Fix adding excluded users to the private room sharing tables when
 joining a room (#11143)

* We only need to fetch users in private rooms

* Filter out `user_id` at the top

* Discard excluded users in the top loop

We weren't doing this in the "First, if they're our user" branch so this
is a bugfix.

* The caller must check that `user_id` is included

This is in the docstring. There are two call sites:
- one in `_handle_room_publicity_change`, which explicitly checks before calling;
- and another in `_handle_room_membership_event`, which returns early if
  the user is excluded.

So this change is safe.

* Test joining a private room with an excluded user

* Tweak an existing test

* Changelog

* test docstring

* lint
---
 changelog.d/11143.misc                |  1 +
 synapse/handlers/user_directory.py    | 28 +++++++--------
 tests/handlers/test_user_directory.py | 67 +++++++++++++++++++++++++++--------
 3 files changed, 67 insertions(+), 29 deletions(-)
 create mode 100644 changelog.d/11143.misc

diff --git a/changelog.d/11143.misc b/changelog.d/11143.misc
new file mode 100644
index 0000000000..496e44a9c0
--- /dev/null
+++ b/changelog.d/11143.misc
@@ -0,0 +1 @@
+Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room.
\ No newline at end of file
diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py
index 991fee7e58..a0eb45446f 100644
--- a/synapse/handlers/user_directory.py
+++ b/synapse/handlers/user_directory.py
@@ -373,31 +373,29 @@ class UserDirectoryHandler(StateDeltasHandler):
         is_public = await self.store.is_room_world_readable_or_publicly_joinable(
             room_id
         )
-        other_users_in_room = await self.store.get_users_in_room(room_id)
-
         if is_public:
             await self.store.add_users_in_public_rooms(room_id, (user_id,))
         else:
+            users_in_room = await self.store.get_users_in_room(room_id)
+            other_users_in_room = [
+                other
+                for other in users_in_room
+                if other != user_id
+                and (
+                    not self.is_mine_id(other)
+                    or await self.store.should_include_local_user_in_dir(other)
+                )
+            ]
             to_insert = set()
 
             # First, if they're our user then we need to update for every user
             if self.is_mine_id(user_id):
-                if await self.store.should_include_local_user_in_dir(user_id):
-                    for other_user_id in other_users_in_room:
-                        if user_id == other_user_id:
-                            continue
-
-                        to_insert.add((user_id, other_user_id))
+                for other_user_id in other_users_in_room:
+                    to_insert.add((user_id, other_user_id))
 
             # Next we need to update for every local user in the room
             for other_user_id in other_users_in_room:
-                if user_id == other_user_id:
-                    continue
-
-                include_other_user = self.is_mine_id(
-                    other_user_id
-                ) and await self.store.should_include_local_user_in_dir(other_user_id)
-                if include_other_user:
+                if self.is_mine_id(other_user_id):
                     to_insert.add((other_user_id, user_id))
 
             if to_insert:
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index b9ad92b977..70c621b825 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -646,22 +646,20 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
         u2_token = self.login(u2, "pass")
         u3 = self.register_user("user3", "pass")
 
-        # We do not add users to the directory until they join a room.
+        # u1 can't see u2 until they share a private room, or u1 is in a public room.
         s = self.get_success(self.handler.search_users(u1, "user2", 10))
         self.assertEqual(len(s["results"]), 0)
 
+        # Get u1 and u2 into a private room.
         room = self.helper.create_room_as(u1, is_public=False, tok=u1_token)
         self.helper.invite(room, src=u1, targ=u2, tok=u1_token)
         self.helper.join(room, user=u2, tok=u2_token)
 
         # Check we have populated the database correctly.
-        shares_private = self.get_success(
-            self.user_dir_helper.get_users_who_share_private_rooms()
-        )
-        public_users = self.get_success(
-            self.user_dir_helper.get_users_in_public_rooms()
+        users, public_users, shares_private = self.get_success(
+            self.user_dir_helper.get_tables()
         )
-
+        self.assertEqual(users, {u1, u2, u3})
         self.assertEqual(shares_private, {(u1, u2, room), (u2, u1, room)})
         self.assertEqual(public_users, set())
 
@@ -680,14 +678,11 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
         # User 2 then leaves.
         self.helper.leave(room, user=u2, tok=u2_token)
 
-        # Check we have removed the values.
-        shares_private = self.get_success(
-            self.user_dir_helper.get_users_who_share_private_rooms()
-        )
-        public_users = self.get_success(
-            self.user_dir_helper.get_users_in_public_rooms()
+        # Check this is reflected in the DB.
+        users, public_users, shares_private = self.get_success(
+            self.user_dir_helper.get_tables()
         )
-
+        self.assertEqual(users, {u1, u2, u3})
         self.assertEqual(shares_private, set())
         self.assertEqual(public_users, set())
 
@@ -698,6 +693,50 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
         s = self.get_success(self.handler.search_users(u1, "user3", 10))
         self.assertEqual(len(s["results"]), 0)
 
+    def test_joining_private_room_with_excluded_user(self) -> None:
+        """
+        When a user excluded from the user directory, E say, joins a private
+        room, E will not appear in the `users_who_share_private_rooms` table.
+
+        When a normal user, U say, joins a private room containing E, then
+        U will appear in the `users_who_share_private_rooms` table, but E will
+        not.
+        """
+        # Setup a support and two normal users.
+        alice = self.register_user("alice", "pass")
+        alice_token = self.login(alice, "pass")
+        bob = self.register_user("bob", "pass")
+        bob_token = self.login(bob, "pass")
+        support = "@support1:test"
+        self.get_success(
+            self.store.register_user(
+                user_id=support, password_hash=None, user_type=UserTypes.SUPPORT
+            )
+        )
+
+        # Alice makes a room. Inject the support user into the room.
+        room = self.helper.create_room_as(alice, is_public=False, tok=alice_token)
+        self.get_success(inject_member_event(self.hs, room, support, "join"))
+        # Check the DB state. The support user should not be in the directory.
+        users, in_public, in_private = self.get_success(
+            self.user_dir_helper.get_tables()
+        )
+        self.assertEqual(users, {alice, bob})
+        self.assertEqual(in_public, set())
+        self.assertEqual(in_private, set())
+
+        # Then invite Bob, who accepts.
+        self.helper.invite(room, alice, bob, tok=alice_token)
+        self.helper.join(room, bob, tok=bob_token)
+
+        # Check the DB state. The support user should not be in the directory.
+        users, in_public, in_private = self.get_success(
+            self.user_dir_helper.get_tables()
+        )
+        self.assertEqual(users, {alice, bob})
+        self.assertEqual(in_public, set())
+        self.assertEqual(in_private, {(alice, bob, room), (bob, alice, room)})
+
     def test_spam_checker(self) -> None:
         """
         A user which fails the spam checks will not appear in search results.
-- 
cgit 1.5.1


From ba00e20234eadae66f105f8bda64e39beed9a92d Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Thu, 21 Oct 2021 14:39:16 -0400
Subject: Add a thread relation type per MSC3440. (#11088)

Adds experimental support for MSC3440's `io.element.thread` relation
type (and the aggregation for it).
---
 changelog.d/11088.feature                   |  1 +
 synapse/api/constants.py                    |  1 +
 synapse/config/experimental.py              |  2 +
 synapse/events/utils.py                     | 17 +++++++++
 synapse/rest/client/relations.py            |  3 +-
 synapse/storage/databases/main/events.py    |  4 ++
 synapse/storage/databases/main/relations.py | 59 ++++++++++++++++++++++++++++-
 tests/rest/client/test_relations.py         | 40 ++++++++++++++++---
 8 files changed, 119 insertions(+), 8 deletions(-)
 create mode 100644 changelog.d/11088.feature

diff --git a/changelog.d/11088.feature b/changelog.d/11088.feature
new file mode 100644
index 0000000000..76b0d28084
--- /dev/null
+++ b/changelog.d/11088.feature
@@ -0,0 +1 @@
+Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index a31f037748..a33ac34161 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -176,6 +176,7 @@ class RelationTypes:
     ANNOTATION = "m.annotation"
     REPLACE = "m.replace"
     REFERENCE = "m.reference"
+    THREAD = "io.element.thread"
 
 
 class LimitBlockingTypes:
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index b013a3918c..8b098ad48d 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -26,6 +26,8 @@ class ExperimentalConfig(Config):
 
         # Whether to enable experimental MSC1849 (aka relations) support
         self.msc1849_enabled = config.get("experimental_msc1849_support_enabled", True)
+        # MSC3440 (thread relation)
+        self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False)
 
         # MSC3026 (busy presence state)
         self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 3f3eba86a8..6fa631aa1d 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -386,6 +386,7 @@ class EventClientSerializer:
     def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
         self._msc1849_enabled = hs.config.experimental.msc1849_enabled
+        self._msc3440_enabled = hs.config.experimental.msc3440_enabled
 
     async def serialize_event(
         self,
@@ -462,6 +463,22 @@ class EventClientSerializer:
                     "sender": edit.sender,
                 }
 
+            # If this event is the start of a thread, include a summary of the replies.
+            if self._msc3440_enabled:
+                (
+                    thread_count,
+                    latest_thread_event,
+                ) = await self.store.get_thread_summary(event_id)
+                if latest_thread_event:
+                    r = serialized_event["unsigned"].setdefault("m.relations", {})
+                    r[RelationTypes.THREAD] = {
+                        # Don't bundle aggregations as this could recurse forever.
+                        "latest_event": await self.serialize_event(
+                            latest_thread_event, time_now, bundle_aggregations=False
+                        ),
+                        "count": thread_count,
+                    }
+
         return serialized_event
 
     async def serialize_events(
diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py
index d695c18be2..58f6699073 100644
--- a/synapse/rest/client/relations.py
+++ b/synapse/rest/client/relations.py
@@ -128,9 +128,10 @@ class RelationSendServlet(RestServlet):
 
         content["m.relates_to"] = {
             "event_id": parent_id,
-            "key": aggregation_key,
             "rel_type": relation_type,
         }
+        if aggregation_key is not None:
+            content["m.relates_to"]["key"] = aggregation_key
 
         event_dict = {
             "type": event_type,
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 37439f8562..8d9086ecf0 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -1710,6 +1710,7 @@ class PersistEventsStore:
             RelationTypes.ANNOTATION,
             RelationTypes.REFERENCE,
             RelationTypes.REPLACE,
+            RelationTypes.THREAD,
         ):
             # Unknown relation type
             return
@@ -1740,6 +1741,9 @@ class PersistEventsStore:
         if rel_type == RelationTypes.REPLACE:
             txn.call_after(self.store.get_applicable_edit.invalidate, (parent_id,))
 
+        if rel_type == RelationTypes.THREAD:
+            txn.call_after(self.store.get_thread_summary.invalidate, (parent_id,))
+
     def _handle_insertion_event(self, txn: LoggingTransaction, event: EventBase):
         """Handles keeping track of insertion events and edges/connections.
         Part of MSC2716.
diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py
index 2bbf6d6a95..40760fbd1b 100644
--- a/synapse/storage/databases/main/relations.py
+++ b/synapse/storage/databases/main/relations.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import Optional
+from typing import Optional, Tuple
 
 import attr
 
@@ -269,6 +269,63 @@ class RelationsWorkerStore(SQLBaseStore):
 
         return await self.get_event(edit_id, allow_none=True)
 
+    @cached()
+    async def get_thread_summary(
+        self, event_id: str
+    ) -> Tuple[int, Optional[EventBase]]:
+        """Get the number of threaded replies, the senders of those replies, and
+        the latest reply (if any) for the given event.
+
+        Args:
+            event_id: The original event ID
+
+        Returns:
+            The number of items in the thread and the most recent response, if any.
+        """
+
+        def _get_thread_summary_txn(txn) -> Tuple[int, Optional[str]]:
+            # Fetch the count of threaded events and the latest event ID.
+            # TODO Should this only allow m.room.message events.
+            sql = """
+                SELECT event_id
+                FROM event_relations
+                INNER JOIN events USING (event_id)
+                WHERE
+                    relates_to_id = ?
+                    AND relation_type = ?
+                ORDER BY topological_ordering DESC, stream_ordering DESC
+                LIMIT 1
+            """
+
+            txn.execute(sql, (event_id, RelationTypes.THREAD))
+            row = txn.fetchone()
+            if row is None:
+                return 0, None
+
+            latest_event_id = row[0]
+
+            sql = """
+                SELECT COALESCE(COUNT(event_id), 0)
+                FROM event_relations
+                WHERE
+                    relates_to_id = ?
+                    AND relation_type = ?
+            """
+            txn.execute(sql, (event_id, RelationTypes.THREAD))
+            count = txn.fetchone()[0]
+
+            return count, latest_event_id
+
+        count, latest_event_id = await self.db_pool.runInteraction(
+            "get_thread_summary", _get_thread_summary_txn
+        )
+
+        latest_event = None
+        if latest_event_id:
+            latest_event = await self.get_event(latest_event_id, allow_none=True)
+
+        return count, latest_event
+
     async def has_user_annotated_event(
         self, parent_id: str, event_type: str, aggregation_key: str, sender: str
     ) -> bool:
diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py
index 3c7d49f0b4..78c2fb86b9 100644
--- a/tests/rest/client/test_relations.py
+++ b/tests/rest/client/test_relations.py
@@ -101,10 +101,10 @@ class RelationsTestCase(unittest.HomeserverTestCase):
 
     def test_basic_paginate_relations(self):
         """Tests that calling pagination API correctly the latest relations."""
-        channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction")
+        channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction", "a")
         self.assertEquals(200, channel.code, channel.json_body)
 
-        channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction")
+        channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction", "b")
         self.assertEquals(200, channel.code, channel.json_body)
         annotation_id = channel.json_body["event_id"]
 
@@ -141,8 +141,10 @@ class RelationsTestCase(unittest.HomeserverTestCase):
         """
 
         expected_event_ids = []
-        for _ in range(10):
-            channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction")
+        for idx in range(10):
+            channel = self._send_relation(
+                RelationTypes.ANNOTATION, "m.reaction", chr(ord("a") + idx)
+            )
             self.assertEquals(200, channel.code, channel.json_body)
             expected_event_ids.append(channel.json_body["event_id"])
 
@@ -386,8 +388,9 @@ class RelationsTestCase(unittest.HomeserverTestCase):
         )
         self.assertEquals(400, channel.code, channel.json_body)
 
+    @unittest.override_config({"experimental_features": {"msc3440_enabled": True}})
     def test_aggregation_get_event(self):
-        """Test that annotations and references get correctly bundled when
+        """Test that annotations, references, and threads get correctly bundled when
         getting the parent event.
         """
 
@@ -410,6 +413,13 @@ class RelationsTestCase(unittest.HomeserverTestCase):
         self.assertEquals(200, channel.code, channel.json_body)
         reply_2 = channel.json_body["event_id"]
 
+        channel = self._send_relation(RelationTypes.THREAD, "m.room.test")
+        self.assertEquals(200, channel.code, channel.json_body)
+
+        channel = self._send_relation(RelationTypes.THREAD, "m.room.test")
+        self.assertEquals(200, channel.code, channel.json_body)
+        thread_2 = channel.json_body["event_id"]
+
         channel = self.make_request(
             "GET",
             "/rooms/%s/event/%s" % (self.room, self.parent_id),
@@ -429,6 +439,25 @@ class RelationsTestCase(unittest.HomeserverTestCase):
                 RelationTypes.REFERENCE: {
                     "chunk": [{"event_id": reply_1}, {"event_id": reply_2}]
                 },
+                RelationTypes.THREAD: {
+                    "count": 2,
+                    "latest_event": {
+                        "age": 100,
+                        "content": {
+                            "m.relates_to": {
+                                "event_id": self.parent_id,
+                                "rel_type": RelationTypes.THREAD,
+                            }
+                        },
+                        "event_id": thread_2,
+                        "origin_server_ts": 1600,
+                        "room_id": self.room,
+                        "sender": self.user_id,
+                        "type": "m.room.test",
+                        "unsigned": {"age": 100},
+                        "user_id": self.user_id,
+                    },
+                },
             },
         )
 
@@ -559,7 +588,6 @@ class RelationsTestCase(unittest.HomeserverTestCase):
             {
                 "m.relates_to": {
                     "event_id": self.parent_id,
-                    "key": None,
                     "rel_type": "m.reference",
                 }
             },
-- 
cgit 1.5.1


From b0f03aeb6a02a5c152f769cea4992079609481d1 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Fri, 22 Oct 2021 10:00:28 +0200
Subject: Add more information what happens when a user is deactivated (#11083)

---
 changelog.d/11083.doc            |  1 +
 docs/admin_api/user_admin_api.md | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 changelog.d/11083.doc

diff --git a/changelog.d/11083.doc b/changelog.d/11083.doc
new file mode 100644
index 0000000000..245dd3758d
--- /dev/null
+++ b/changelog.d/11083.doc
@@ -0,0 +1 @@
+Add more information about what happens when a user is deactivated.
\ No newline at end of file
diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
index 60dc913915..534f8400ba 100644
--- a/docs/admin_api/user_admin_api.md
+++ b/docs/admin_api/user_admin_api.md
@@ -341,6 +341,7 @@ The following actions are performed when deactivating an user:
 - Remove all 3PIDs from the homeserver
 - Delete all devices and E2EE keys
 - Delete all access tokens
+- Delete all pushers
 - Delete the password hash
 - Removal from all rooms the user is a member of
 - Remove the user from the user directory
@@ -354,6 +355,15 @@ is set to `true`:
 - Remove the user's avatar URL
 - Mark the user as erased
 
+The following actions are **NOT** performed. The list may be incomplete.
+
+- Remove mappings of SSO IDs
+- [Delete media uploaded](#delete-media-uploaded-by-a-user) by user (included avatar images)
+- Delete sent and received messages
+- Delete E2E cross-signing keys
+- Remove the user's creation (registration) timestamp
+- [Remove rate limit overrides](#override-ratelimiting-for-users)
+- Remove from monthly active users
 
 ## Reset password
 
-- 
cgit 1.5.1


From b9ce53e8785d6f0dba6a3efcd708e4a185c32465 Mon Sep 17 00:00:00 2001
From: Jason Robinson <jasonr@element.io>
Date: Fri, 22 Oct 2021 13:00:52 +0300
Subject: Fix synapse.config module "read" command (#11145)

`synapse.config.__main__` has the possibility to read a config item. This can be used to conveniently also validate the config is valid before trying to start Synapse.

 The "read" command broke in https://github.com/matrix-org/synapse/pull/10916 as it now requires passing in "server.server_name" for example.

 Also made the read command optional so one can just call this with just the confirm file reference and get a "Config parses OK" if things are ok.

Signed-off-by: Jason Robinson <jasonr@matrix.org>
Co-authored-by: Brendan Abolivier <babolivier@matrix.org>
---
 changelog.d/11145.bugfix      |  1 +
 synapse/config/__main__.py    | 46 ++++++++++++++++++++--------
 tests/config/test___main__.py | 31 +++++++++++++++++++
 tests/config/test_load.py     | 70 ++++++++++---------------------------------
 tests/config/utils.py         | 58 +++++++++++++++++++++++++++++++++++
 5 files changed, 138 insertions(+), 68 deletions(-)
 create mode 100644 changelog.d/11145.bugfix
 create mode 100644 tests/config/test___main__.py
 create mode 100644 tests/config/utils.py

diff --git a/changelog.d/11145.bugfix b/changelog.d/11145.bugfix
new file mode 100644
index 0000000000..f369feac42
--- /dev/null
+++ b/changelog.d/11145.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.45.0 breaking the configuration file parsing script.
diff --git a/synapse/config/__main__.py b/synapse/config/__main__.py
index b5b6735a8f..c555f5f914 100644
--- a/synapse/config/__main__.py
+++ b/synapse/config/__main__.py
@@ -1,4 +1,5 @@
 # Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,25 +12,44 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import sys
+
 from synapse.config._base import ConfigError
+from synapse.config.homeserver import HomeServerConfig
 
-if __name__ == "__main__":
-    import sys
 
-    from synapse.config.homeserver import HomeServerConfig
+def main(args):
+    action = args[1] if len(args) > 1 and args[1] == "read" else None
+    # If we're reading a key in the config file, then `args[1]` will be `read`  and `args[2]`
+    # will be the key to read.
+    # We'll want to rework this code if we want to support more actions than just `read`.
+    load_config_args = args[3:] if action else args[1:]
 
-    action = sys.argv[1]
+    try:
+        config = HomeServerConfig.load_config("", load_config_args)
+    except ConfigError as e:
+        sys.stderr.write("\n" + str(e) + "\n")
+        sys.exit(1)
+
+    print("Config parses OK!")
 
     if action == "read":
-        key = sys.argv[2]
+        key = args[2]
+        key_parts = key.split(".")
+
+        value = config
         try:
-            config = HomeServerConfig.load_config("", sys.argv[3:])
-        except ConfigError as e:
-            sys.stderr.write("\n" + str(e) + "\n")
+            while len(key_parts):
+                value = getattr(value, key_parts[0])
+                key_parts.pop(0)
+
+            print(f"\n{key}: {value}")
+        except AttributeError:
+            print(
+                f"\nNo '{key}' key could be found in the provided configuration file."
+            )
             sys.exit(1)
 
-        print(getattr(config, key))
-        sys.exit(0)
-    else:
-        sys.stderr.write("Unknown command %r\n" % (action,))
-        sys.exit(1)
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/tests/config/test___main__.py b/tests/config/test___main__.py
new file mode 100644
index 0000000000..b1c73d3612
--- /dev/null
+++ b/tests/config/test___main__.py
@@ -0,0 +1,31 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.config.__main__ import main
+
+from tests.config.utils import ConfigFileTestCase
+
+
+class ConfigMainFileTestCase(ConfigFileTestCase):
+    def test_executes_without_an_action(self):
+        self.generate_config()
+        main(["", "-c", self.config_file])
+
+    def test_read__error_if_key_not_found(self):
+        self.generate_config()
+        with self.assertRaises(SystemExit):
+            main(["", "read", "foo.bar.hello", "-c", self.config_file])
+
+    def test_read__passes_if_key_found(self):
+        self.generate_config()
+        main(["", "read", "server.server_name", "-c", self.config_file])
diff --git a/tests/config/test_load.py b/tests/config/test_load.py
index 59635de205..765258c47a 100644
--- a/tests/config/test_load.py
+++ b/tests/config/test_load.py
@@ -1,4 +1,5 @@
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2021 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,43 +12,30 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os.path
-import shutil
-import tempfile
-from contextlib import redirect_stdout
-from io import StringIO
-
 import yaml
 
 from synapse.config import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 
-from tests import unittest
-
-
-class ConfigLoadingTestCase(unittest.TestCase):
-    def setUp(self):
-        self.dir = tempfile.mkdtemp()
-        self.file = os.path.join(self.dir, "homeserver.yaml")
+from tests.config.utils import ConfigFileTestCase
 
-    def tearDown(self):
-        shutil.rmtree(self.dir)
 
+class ConfigLoadingFileTestCase(ConfigFileTestCase):
     def test_load_fails_if_server_name_missing(self):
         self.generate_config_and_remove_lines_containing("server_name")
         with self.assertRaises(ConfigError):
-            HomeServerConfig.load_config("", ["-c", self.file])
+            HomeServerConfig.load_config("", ["-c", self.config_file])
         with self.assertRaises(ConfigError):
-            HomeServerConfig.load_or_generate_config("", ["-c", self.file])
+            HomeServerConfig.load_or_generate_config("", ["-c", self.config_file])
 
     def test_generates_and_loads_macaroon_secret_key(self):
         self.generate_config()
 
-        with open(self.file) as f:
+        with open(self.config_file) as f:
             raw = yaml.safe_load(f)
         self.assertIn("macaroon_secret_key", raw)
 
-        config = HomeServerConfig.load_config("", ["-c", self.file])
+        config = HomeServerConfig.load_config("", ["-c", self.config_file])
         self.assertTrue(
             hasattr(config.key, "macaroon_secret_key"),
             "Want config to have attr macaroon_secret_key",
@@ -58,7 +46,7 @@ class ConfigLoadingTestCase(unittest.TestCase):
                 "was: %r" % (config.key.macaroon_secret_key,)
             )
 
-        config = HomeServerConfig.load_or_generate_config("", ["-c", self.file])
+        config = HomeServerConfig.load_or_generate_config("", ["-c", self.config_file])
         self.assertTrue(
             hasattr(config.key, "macaroon_secret_key"),
             "Want config to have attr macaroon_secret_key",
@@ -71,9 +59,9 @@ class ConfigLoadingTestCase(unittest.TestCase):
 
     def test_load_succeeds_if_macaroon_secret_key_missing(self):
         self.generate_config_and_remove_lines_containing("macaroon")
-        config1 = HomeServerConfig.load_config("", ["-c", self.file])
-        config2 = HomeServerConfig.load_config("", ["-c", self.file])
-        config3 = HomeServerConfig.load_or_generate_config("", ["-c", self.file])
+        config1 = HomeServerConfig.load_config("", ["-c", self.config_file])
+        config2 = HomeServerConfig.load_config("", ["-c", self.config_file])
+        config3 = HomeServerConfig.load_or_generate_config("", ["-c", self.config_file])
         self.assertEqual(
             config1.key.macaroon_secret_key, config2.key.macaroon_secret_key
         )
@@ -87,15 +75,15 @@ class ConfigLoadingTestCase(unittest.TestCase):
             ["enable_registration: true", "disable_registration: true"]
         )
         # Check that disable_registration clobbers enable_registration.
-        config = HomeServerConfig.load_config("", ["-c", self.file])
+        config = HomeServerConfig.load_config("", ["-c", self.config_file])
         self.assertFalse(config.registration.enable_registration)
 
-        config = HomeServerConfig.load_or_generate_config("", ["-c", self.file])
+        config = HomeServerConfig.load_or_generate_config("", ["-c", self.config_file])
         self.assertFalse(config.registration.enable_registration)
 
         # Check that either config value is clobbered by the command line.
         config = HomeServerConfig.load_or_generate_config(
-            "", ["-c", self.file, "--enable-registration"]
+            "", ["-c", self.config_file, "--enable-registration"]
         )
         self.assertTrue(config.registration.enable_registration)
 
@@ -104,33 +92,5 @@ class ConfigLoadingTestCase(unittest.TestCase):
         self.add_lines_to_config(["enable_metrics: true"])
 
         # The default Metrics Flags are off by default.
-        config = HomeServerConfig.load_config("", ["-c", self.file])
+        config = HomeServerConfig.load_config("", ["-c", self.config_file])
         self.assertFalse(config.metrics.metrics_flags.known_servers)
-
-    def generate_config(self):
-        with redirect_stdout(StringIO()):
-            HomeServerConfig.load_or_generate_config(
-                "",
-                [
-                    "--generate-config",
-                    "-c",
-                    self.file,
-                    "--report-stats=yes",
-                    "-H",
-                    "lemurs.win",
-                ],
-            )
-
-    def generate_config_and_remove_lines_containing(self, needle):
-        self.generate_config()
-
-        with open(self.file) as f:
-            contents = f.readlines()
-        contents = [line for line in contents if needle not in line]
-        with open(self.file, "w") as f:
-            f.write("".join(contents))
-
-    def add_lines_to_config(self, lines):
-        with open(self.file, "a") as f:
-            for line in lines:
-                f.write(line + "\n")
diff --git a/tests/config/utils.py b/tests/config/utils.py
new file mode 100644
index 0000000000..94c18a052b
--- /dev/null
+++ b/tests/config/utils.py
@@ -0,0 +1,58 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import shutil
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from io import StringIO
+
+from synapse.config.homeserver import HomeServerConfig
+
+
+class ConfigFileTestCase(unittest.TestCase):
+    def setUp(self):
+        self.dir = tempfile.mkdtemp()
+        self.config_file = os.path.join(self.dir, "homeserver.yaml")
+
+    def tearDown(self):
+        shutil.rmtree(self.dir)
+
+    def generate_config(self):
+        with redirect_stdout(StringIO()):
+            HomeServerConfig.load_or_generate_config(
+                "",
+                [
+                    "--generate-config",
+                    "-c",
+                    self.config_file,
+                    "--report-stats=yes",
+                    "-H",
+                    "lemurs.win",
+                ],
+            )
+
+    def generate_config_and_remove_lines_containing(self, needle):
+        self.generate_config()
+
+        with open(self.config_file) as f:
+            contents = f.readlines()
+        contents = [line for line in contents if needle not in line]
+        with open(self.config_file, "w") as f:
+            f.write("".join(contents))
+
+    def add_lines_to_config(self, lines):
+        with open(self.config_file, "a") as f:
+            for line in lines:
+                f.write(line + "\n")
-- 
cgit 1.5.1


From 2b82ec425fccb0ef626242779f7ccd4d77a0685c Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Fri, 22 Oct 2021 18:15:41 +0100
Subject: Add type hints for most `HomeServer` parameters (#11095)

---
 changelog.d/11095.misc                             |  1 +
 synapse/app/_base.py                               |  8 +++----
 synapse/app/admin_cmd.py                           |  4 ++--
 synapse/app/generic_worker.py                      |  4 ++--
 synapse/app/homeserver.py                          |  2 +-
 synapse/app/phone_stats_home.py                    |  8 +++++--
 synapse/appservice/api.py                          |  3 ++-
 synapse/config/logger.py                           |  9 ++++++-
 synapse/federation/federation_base.py              |  7 +++++-
 synapse/federation/federation_server.py            |  9 +++----
 synapse/http/matrixfederationclient.py             |  8 +++++--
 synapse/http/server.py                             | 19 +++++++++------
 synapse/replication/http/__init__.py               |  9 +++++--
 synapse/replication/http/_base.py                  |  8 ++++---
 synapse/replication/http/account_data.py           | 14 +++++++----
 synapse/replication/http/devices.py                |  8 +++++--
 synapse/replication/http/federation.py             | 16 ++++++++-----
 synapse/replication/http/login.py                  |  8 +++++--
 synapse/replication/http/membership.py             |  6 ++---
 synapse/replication/http/presence.py               |  2 +-
 synapse/replication/http/push.py                   |  2 +-
 synapse/replication/http/register.py               | 10 +++++---
 synapse/replication/http/send_event.py             |  8 +++++--
 synapse/replication/http/streams.py                |  8 +++++--
 synapse/replication/slave/storage/_base.py         |  7 ++++--
 synapse/replication/slave/storage/client_ips.py    |  7 +++++-
 synapse/replication/slave/storage/devices.py       |  7 +++++-
 synapse/replication/slave/storage/events.py        |  6 ++++-
 synapse/replication/slave/storage/filtering.py     |  7 +++++-
 synapse/replication/slave/storage/groups.py        |  7 +++++-
 synapse/replication/tcp/external_cache.py          |  9 ++++++-
 synapse/replication/tcp/handler.py                 |  6 ++++-
 synapse/replication/tcp/resource.py                |  8 +++++--
 synapse/replication/tcp/streams/_base.py           | 20 ++++++++--------
 synapse/rest/admin/devices.py                      |  2 +-
 synapse/server.py                                  | 11 ++++++---
 synapse/storage/database.py                        |  6 ++++-
 synapse/storage/databases/__init__.py              | 28 +++++++++++++++++-----
 synapse/storage/databases/main/__init__.py         |  7 ++++--
 synapse/storage/databases/main/account_data.py     |  7 ++++--
 synapse/storage/databases/main/cache.py            |  7 ++++--
 synapse/storage/databases/main/deviceinbox.py      |  9 ++++---
 synapse/storage/databases/main/devices.py          | 21 ++++++++++++----
 synapse/storage/databases/main/event_federation.py |  9 ++++---
 .../storage/databases/main/event_push_actions.py   |  9 ++++---
 .../storage/databases/main/events_bg_updates.py    |  7 ++++--
 synapse/storage/databases/main/media_repository.py |  9 ++++---
 synapse/storage/databases/main/metrics.py          |  7 ++++--
 .../storage/databases/main/monthly_active_users.py |  9 ++++---
 synapse/storage/databases/main/push_rule.py        |  7 ++++--
 synapse/storage/databases/main/receipts.py         |  7 ++++--
 synapse/storage/databases/main/room.py             | 11 +++++----
 synapse/storage/databases/main/roommember.py       |  7 +++---
 synapse/storage/databases/main/search.py           |  9 ++++---
 synapse/storage/databases/main/state.py            | 11 +++++----
 synapse/storage/databases/main/stats.py            |  7 ++++--
 synapse/storage/databases/main/transactions.py     |  7 ++++--
 synapse/storage/persist_events.py                  |  6 ++++-
 58 files changed, 342 insertions(+), 143 deletions(-)
 create mode 100644 changelog.d/11095.misc

diff --git a/changelog.d/11095.misc b/changelog.d/11095.misc
new file mode 100644
index 0000000000..786e90b595
--- /dev/null
+++ b/changelog.d/11095.misc
@@ -0,0 +1 @@
+Add type hints to most `HomeServer` parameters.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index bb4d53d778..2ca2e051e4 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -294,7 +294,7 @@ def listen_ssl(
     return r
 
 
-def refresh_certificate(hs):
+def refresh_certificate(hs: "HomeServer"):
     """
     Refresh the TLS certificates that Synapse is using by re-reading them from
     disk and updating the TLS context factories to use them.
@@ -419,11 +419,11 @@ async def start(hs: "HomeServer"):
         atexit.register(gc.freeze)
 
 
-def setup_sentry(hs):
+def setup_sentry(hs: "HomeServer"):
     """Enable sentry integration, if enabled in configuration
 
     Args:
-        hs (synapse.server.HomeServer)
+        hs
     """
 
     if not hs.config.metrics.sentry_enabled:
@@ -449,7 +449,7 @@ def setup_sentry(hs):
         scope.set_tag("worker_name", name)
 
 
-def setup_sdnotify(hs):
+def setup_sdnotify(hs: "HomeServer"):
     """Adds process state hooks to tell systemd what we are up to."""
 
     # Tell systemd our state, if we're using it. This will silently fail if
diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py
index b156b93bf3..2fc848596d 100644
--- a/synapse/app/admin_cmd.py
+++ b/synapse/app/admin_cmd.py
@@ -68,11 +68,11 @@ class AdminCmdServer(HomeServer):
     DATASTORE_CLASS = AdminCmdSlavedStore
 
 
-async def export_data_command(hs, args):
+async def export_data_command(hs: HomeServer, args):
     """Export data for a user.
 
     Args:
-        hs (HomeServer)
+        hs
         args (argparse.Namespace)
     """
 
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index 7489f31d9a..51eadf122d 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -131,10 +131,10 @@ class KeyUploadServlet(RestServlet):
 
     PATTERNS = client_patterns("/keys/upload(/(?P<device_id>[^/]+))?$")
 
-    def __init__(self, hs):
+    def __init__(self, hs: HomeServer):
         """
         Args:
-            hs (synapse.server.HomeServer): server
+            hs: server
         """
         super().__init__()
         self.auth = hs.get_auth()
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 422f03cc04..93e2299266 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -412,7 +412,7 @@ def format_config_error(e: ConfigError) -> Iterator[str]:
         e = e.__cause__
 
 
-def run(hs):
+def run(hs: HomeServer):
     PROFILE_SYNAPSE = False
     if PROFILE_SYNAPSE:
 
diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py
index fcd01e833c..126450e17a 100644
--- a/synapse/app/phone_stats_home.py
+++ b/synapse/app/phone_stats_home.py
@@ -15,11 +15,15 @@ import logging
 import math
 import resource
 import sys
+from typing import TYPE_CHECKING
 
 from prometheus_client import Gauge
 
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger("synapse.app.homeserver")
 
 # Contains the list of processes we will be monitoring
@@ -41,7 +45,7 @@ registered_reserved_users_mau_gauge = Gauge(
 
 
 @wrap_as_background_process("phone_stats_home")
-async def phone_stats_home(hs, stats, stats_process=_stats_process):
+async def phone_stats_home(hs: "HomeServer", stats, stats_process=_stats_process):
     logger.info("Gathering stats for reporting")
     now = int(hs.get_clock().time())
     uptime = int(now - hs.start_time)
@@ -142,7 +146,7 @@ async def phone_stats_home(hs, stats, stats_process=_stats_process):
         logger.warning("Error reporting stats: %s", e)
 
 
-def start_phone_stats_home(hs):
+def start_phone_stats_home(hs: "HomeServer"):
     """
     Start the background tasks which report phone home stats.
     """
diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py
index 935f24263c..d08f6bbd7f 100644
--- a/synapse/appservice/api.py
+++ b/synapse/appservice/api.py
@@ -27,6 +27,7 @@ from synapse.util.caches.response_cache import ResponseCache
 
 if TYPE_CHECKING:
     from synapse.appservice import ApplicationService
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 
@@ -84,7 +85,7 @@ class ApplicationServiceApi(SimpleHttpClient):
     pushing.
     """
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.clock = hs.get_clock()
 
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index 0a08231e5a..5252e61a99 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -18,6 +18,7 @@ import os
 import sys
 import threading
 from string import Template
+from typing import TYPE_CHECKING
 
 import yaml
 from zope.interface import implementer
@@ -38,6 +39,9 @@ from synapse.util.versionstring import get_version_string
 
 from ._base import Config, ConfigError
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 DEFAULT_LOG_CONFIG = Template(
     """\
 # Log configuration for Synapse.
@@ -306,7 +310,10 @@ def _reload_logging_config(log_config_path):
 
 
 def setup_logging(
-    hs, config, use_worker_options=False, logBeginner: LogBeginner = globalLogBeginner
+    hs: "HomeServer",
+    config,
+    use_worker_options=False,
+    logBeginner: LogBeginner = globalLogBeginner,
 ) -> None:
     """
     Set up the logging subsystem.
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index 0cd424e12a..f56344a3b9 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 import logging
 from collections import namedtuple
+from typing import TYPE_CHECKING
 
 from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
@@ -25,11 +26,15 @@ from synapse.events.utils import prune_event, validate_canonicaljson
 from synapse.http.servlet import assert_params_in_dict
 from synapse.types import JsonDict, get_domain_from_id
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
 logger = logging.getLogger(__name__)
 
 
 class FederationBase:
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.hs = hs
 
         self.server_name = hs.hostname
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index d8c0b86f23..0d66034f44 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -467,7 +467,7 @@ class FederationServer(FederationBase):
 
     async def on_room_state_request(
         self, origin: str, room_id: str, event_id: Optional[str]
-    ) -> Tuple[int, Dict[str, Any]]:
+    ) -> Tuple[int, JsonDict]:
         origin_host, _ = parse_server_name(origin)
         await self.check_server_matches_acl(origin_host, room_id)
 
@@ -481,7 +481,7 @@ class FederationServer(FederationBase):
         # - but that's non-trivial to get right, and anyway somewhat defeats
         # the point of the linearizer.
         with (await self._server_linearizer.queue((origin, room_id))):
-            resp = dict(
+            resp: JsonDict = dict(
                 await self._state_resp_cache.wrap(
                     (room_id, event_id),
                     self._on_context_state_request_compute,
@@ -1061,11 +1061,12 @@ class FederationServer(FederationBase):
 
                 origin, event = next
 
-            lock = await self.store.try_acquire_lock(
+            new_lock = await self.store.try_acquire_lock(
                 _INBOUND_EVENT_HANDLING_LOCK_NAME, room_id
             )
-            if not lock:
+            if not new_lock:
                 return
+            lock = new_lock
 
     def __str__(self) -> str:
         return "<ReplicationLayer(%s)>" % self.server_name
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 4f59224686..203d723d41 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -21,6 +21,7 @@ import typing
 import urllib.parse
 from io import BytesIO, StringIO
 from typing import (
+    TYPE_CHECKING,
     Callable,
     Dict,
     Generic,
@@ -73,6 +74,9 @@ from synapse.util import json_decoder
 from synapse.util.async_helpers import timeout_deferred
 from synapse.util.metrics import Measure
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 outgoing_requests_counter = Counter(
@@ -319,7 +323,7 @@ class MatrixFederationHttpClient:
             requests.
     """
 
-    def __init__(self, hs, tls_client_options_factory):
+    def __init__(self, hs: "HomeServer", tls_client_options_factory):
         self.hs = hs
         self.signing_key = hs.signing_key
         self.server_name = hs.hostname
@@ -711,7 +715,7 @@ class MatrixFederationHttpClient:
         Returns:
             A list of headers to be added as "Authorization:" headers
         """
-        request = {
+        request: JsonDict = {
             "method": method.decode("ascii"),
             "uri": url_bytes.decode("ascii"),
             "origin": self.server_name,
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 897ba5e453..1af0d9a31d 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -22,6 +22,7 @@ import urllib
 from http import HTTPStatus
 from inspect import isawaitable
 from typing import (
+    TYPE_CHECKING,
     Any,
     Awaitable,
     Callable,
@@ -61,6 +62,9 @@ from synapse.util import json_encoder
 from synapse.util.caches import intern_dict
 from synapse.util.iterutils import chunk_seq
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 HTML_ERROR_TEMPLATE = """<!DOCTYPE html>
@@ -343,6 +347,11 @@ class DirectServeJsonResource(_AsyncResource):
         return_json_error(f, request)
 
 
+_PathEntry = collections.namedtuple(
+    "_PathEntry", ["pattern", "callback", "servlet_classname"]
+)
+
+
 class JsonResource(DirectServeJsonResource):
     """This implements the HttpServer interface and provides JSON support for
     Resources.
@@ -359,14 +368,10 @@ class JsonResource(DirectServeJsonResource):
 
     isLeaf = True
 
-    _PathEntry = collections.namedtuple(
-        "_PathEntry", ["pattern", "callback", "servlet_classname"]
-    )
-
-    def __init__(self, hs, canonical_json=True, extract_context=False):
+    def __init__(self, hs: "HomeServer", canonical_json=True, extract_context=False):
         super().__init__(canonical_json, extract_context)
         self.clock = hs.get_clock()
-        self.path_regexs = {}
+        self.path_regexs: Dict[bytes, List[_PathEntry]] = {}
         self.hs = hs
 
     def register_paths(self, method, path_patterns, callback, servlet_classname):
@@ -391,7 +396,7 @@ class JsonResource(DirectServeJsonResource):
         for path_pattern in path_patterns:
             logger.debug("Registering for %s %s", method, path_pattern.pattern)
             self.path_regexs.setdefault(method, []).append(
-                self._PathEntry(path_pattern, callback, servlet_classname)
+                _PathEntry(path_pattern, callback, servlet_classname)
             )
 
     def _get_handler_for_request(
diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py
index ba8114ac9e..1457d9d59b 100644
--- a/synapse/replication/http/__init__.py
+++ b/synapse/replication/http/__init__.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TYPE_CHECKING
+
 from synapse.http.server import JsonResource
 from synapse.replication.http import (
     account_data,
@@ -26,16 +28,19 @@ from synapse.replication.http import (
     streams,
 )
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 REPLICATION_PREFIX = "/_synapse/replication"
 
 
 class ReplicationRestResource(JsonResource):
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         # We enable extracting jaeger contexts here as these are internal APIs.
         super().__init__(hs, canonical_json=False, extract_context=True)
         self.register_servlets(hs)
 
-    def register_servlets(self, hs):
+    def register_servlets(self, hs: "HomeServer"):
         send_event.register_servlets(hs, self)
         federation.register_servlets(hs, self)
         presence.register_servlets(hs, self)
diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py
index e047ec74d8..585332b244 100644
--- a/synapse/replication/http/_base.py
+++ b/synapse/replication/http/_base.py
@@ -17,7 +17,7 @@ import logging
 import re
 import urllib
 from inspect import signature
-from typing import TYPE_CHECKING, Dict, List, Tuple
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Tuple
 
 from prometheus_client import Counter, Gauge
 
@@ -156,7 +156,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):
         pass
 
     @classmethod
-    def make_client(cls, hs):
+    def make_client(cls, hs: "HomeServer"):
         """Create a client that makes requests.
 
         Returns a callable that accepts the same parameters as
@@ -208,7 +208,9 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):
                     url_args.append(txn_id)
 
                 if cls.METHOD == "POST":
-                    request_func = client.post_json_get_json
+                    request_func: Callable[
+                        ..., Awaitable[Any]
+                    ] = client.post_json_get_json
                 elif cls.METHOD == "PUT":
                     request_func = client.put_json
                 elif cls.METHOD == "GET":
diff --git a/synapse/replication/http/account_data.py b/synapse/replication/http/account_data.py
index 70e951af63..5f0f225aa9 100644
--- a/synapse/replication/http/account_data.py
+++ b/synapse/replication/http/account_data.py
@@ -13,10 +13,14 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -37,7 +41,7 @@ class ReplicationUserAccountDataRestServlet(ReplicationEndpoint):
     PATH_ARGS = ("user_id", "account_data_type")
     CACHE = False
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.handler = hs.get_account_data_handler()
@@ -78,7 +82,7 @@ class ReplicationRoomAccountDataRestServlet(ReplicationEndpoint):
     PATH_ARGS = ("user_id", "room_id", "account_data_type")
     CACHE = False
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.handler = hs.get_account_data_handler()
@@ -119,7 +123,7 @@ class ReplicationAddTagRestServlet(ReplicationEndpoint):
     PATH_ARGS = ("user_id", "room_id", "tag")
     CACHE = False
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.handler = hs.get_account_data_handler()
@@ -162,7 +166,7 @@ class ReplicationRemoveTagRestServlet(ReplicationEndpoint):
     )
     CACHE = False
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.handler = hs.get_account_data_handler()
@@ -183,7 +187,7 @@ class ReplicationRemoveTagRestServlet(ReplicationEndpoint):
         return 200, {"max_stream_id": max_stream_id}
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationUserAccountDataRestServlet(hs).register(http_server)
     ReplicationRoomAccountDataRestServlet(hs).register(http_server)
     ReplicationAddTagRestServlet(hs).register(http_server)
diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py
index 5a5818ef61..42dffb39cb 100644
--- a/synapse/replication/http/devices.py
+++ b/synapse/replication/http/devices.py
@@ -13,9 +13,13 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.replication.http._base import ReplicationEndpoint
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -51,7 +55,7 @@ class ReplicationUserDevicesResyncRestServlet(ReplicationEndpoint):
     PATH_ARGS = ("user_id",)
     CACHE = False
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.device_list_updater = hs.get_device_handler().device_list_updater
@@ -68,5 +72,5 @@ class ReplicationUserDevicesResyncRestServlet(ReplicationEndpoint):
         return 200, user_devices
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationUserDevicesResyncRestServlet(hs).register(http_server)
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index a0b3145f4e..5ed535c90d 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.events import make_event_from_dict
@@ -21,6 +22,9 @@ from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
 from synapse.util.metrics import Measure
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +60,7 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
     NAME = "fed_send_events"
     PATH_ARGS = ()
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.store = hs.get_datastore()
@@ -151,7 +155,7 @@ class ReplicationFederationSendEduRestServlet(ReplicationEndpoint):
     NAME = "fed_send_edu"
     PATH_ARGS = ("edu_type",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.store = hs.get_datastore()
@@ -194,7 +198,7 @@ class ReplicationGetQueryRestServlet(ReplicationEndpoint):
     # This is a query, so let's not bother caching
     CACHE = False
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.store = hs.get_datastore()
@@ -238,7 +242,7 @@ class ReplicationCleanRoomRestServlet(ReplicationEndpoint):
     NAME = "fed_cleanup_room"
     PATH_ARGS = ("room_id",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.store = hs.get_datastore()
@@ -273,7 +277,7 @@ class ReplicationStoreRoomOnOutlierMembershipRestServlet(ReplicationEndpoint):
     NAME = "store_room_on_outlier_membership"
     PATH_ARGS = ("room_id",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.store = hs.get_datastore()
@@ -289,7 +293,7 @@ class ReplicationStoreRoomOnOutlierMembershipRestServlet(ReplicationEndpoint):
         return 200, {}
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationFederationSendEventsRestServlet(hs).register(http_server)
     ReplicationFederationSendEduRestServlet(hs).register(http_server)
     ReplicationGetQueryRestServlet(hs).register(http_server)
diff --git a/synapse/replication/http/login.py b/synapse/replication/http/login.py
index 550bd5c95f..0db419ea57 100644
--- a/synapse/replication/http/login.py
+++ b/synapse/replication/http/login.py
@@ -13,10 +13,14 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -30,7 +34,7 @@ class RegisterDeviceReplicationServlet(ReplicationEndpoint):
     NAME = "device_check_registered"
     PATH_ARGS = ("user_id",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.registration_handler = hs.get_registration_handler()
 
@@ -82,5 +86,5 @@ class RegisterDeviceReplicationServlet(ReplicationEndpoint):
         return 200, res
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     RegisterDeviceReplicationServlet(hs).register(http_server)
diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py
index 34206c5060..7371c240b2 100644
--- a/synapse/replication/http/membership.py
+++ b/synapse/replication/http/membership.py
@@ -45,7 +45,7 @@ class ReplicationRemoteJoinRestServlet(ReplicationEndpoint):
     NAME = "remote_join"
     PATH_ARGS = ("room_id", "user_id")
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.federation_handler = hs.get_federation_handler()
@@ -320,7 +320,7 @@ class ReplicationUserJoinedLeftRoomRestServlet(ReplicationEndpoint):
     PATH_ARGS = ("room_id", "user_id", "change")
     CACHE = False  # No point caching as should return instantly.
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.registeration_handler = hs.get_registration_handler()
@@ -360,7 +360,7 @@ class ReplicationUserJoinedLeftRoomRestServlet(ReplicationEndpoint):
         return 200, {}
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationRemoteJoinRestServlet(hs).register(http_server)
     ReplicationRemoteRejectInviteRestServlet(hs).register(http_server)
     ReplicationUserJoinedLeftRoomRestServlet(hs).register(http_server)
diff --git a/synapse/replication/http/presence.py b/synapse/replication/http/presence.py
index bb00247953..63143085d5 100644
--- a/synapse/replication/http/presence.py
+++ b/synapse/replication/http/presence.py
@@ -117,6 +117,6 @@ class ReplicationPresenceSetState(ReplicationEndpoint):
         )
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationBumpPresenceActiveTime(hs).register(http_server)
     ReplicationPresenceSetState(hs).register(http_server)
diff --git a/synapse/replication/http/push.py b/synapse/replication/http/push.py
index 139427cb1f..6c8db3061e 100644
--- a/synapse/replication/http/push.py
+++ b/synapse/replication/http/push.py
@@ -67,5 +67,5 @@ class ReplicationRemovePusherRestServlet(ReplicationEndpoint):
         return 200, {}
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationRemovePusherRestServlet(hs).register(http_server)
diff --git a/synapse/replication/http/register.py b/synapse/replication/http/register.py
index d6dd7242eb..7adfbb666f 100644
--- a/synapse/replication/http/register.py
+++ b/synapse/replication/http/register.py
@@ -13,10 +13,14 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -26,7 +30,7 @@ class ReplicationRegisterServlet(ReplicationEndpoint):
     NAME = "register_user"
     PATH_ARGS = ("user_id",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.store = hs.get_datastore()
         self.registration_handler = hs.get_registration_handler()
@@ -100,7 +104,7 @@ class ReplicationPostRegisterActionsServlet(ReplicationEndpoint):
     NAME = "post_register"
     PATH_ARGS = ("user_id",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.store = hs.get_datastore()
         self.registration_handler = hs.get_registration_handler()
@@ -130,6 +134,6 @@ class ReplicationPostRegisterActionsServlet(ReplicationEndpoint):
         return 200, {}
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationRegisterServlet(hs).register(http_server)
     ReplicationPostRegisterActionsServlet(hs).register(http_server)
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index fae5ffa451..9f6851d059 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.events import make_event_from_dict
@@ -22,6 +23,9 @@ from synapse.replication.http._base import ReplicationEndpoint
 from synapse.types import Requester, UserID
 from synapse.util.metrics import Measure
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -57,7 +61,7 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
     NAME = "send_event"
     PATH_ARGS = ("event_id",)
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self.event_creation_handler = hs.get_event_creation_handler()
@@ -135,5 +139,5 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
         )
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationSendEventRestServlet(hs).register(http_server)
diff --git a/synapse/replication/http/streams.py b/synapse/replication/http/streams.py
index 9afa147d00..3223bc2432 100644
--- a/synapse/replication/http/streams.py
+++ b/synapse/replication/http/streams.py
@@ -13,11 +13,15 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import parse_integer
 from synapse.replication.http._base import ReplicationEndpoint
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -46,7 +50,7 @@ class ReplicationGetStreamUpdates(ReplicationEndpoint):
     PATH_ARGS = ("stream_name",)
     METHOD = "GET"
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
 
         self._instance_name = hs.get_instance_name()
@@ -74,5 +78,5 @@ class ReplicationGetStreamUpdates(ReplicationEndpoint):
         )
 
 
-def register_servlets(hs, http_server):
+def register_servlets(hs: "HomeServer", http_server):
     ReplicationGetStreamUpdates(hs).register(http_server)
diff --git a/synapse/replication/slave/storage/_base.py b/synapse/replication/slave/storage/_base.py
index e460dd85cd..7ecb446e7c 100644
--- a/synapse/replication/slave/storage/_base.py
+++ b/synapse/replication/slave/storage/_base.py
@@ -13,18 +13,21 @@
 # limitations under the License.
 
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.util.id_generators import MultiWriterIdGenerator
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class BaseSlavedStore(CacheInvalidationWorkerStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
         if isinstance(self.database_engine, PostgresEngine):
             self._cache_id_gen: Optional[
diff --git a/synapse/replication/slave/storage/client_ips.py b/synapse/replication/slave/storage/client_ips.py
index 436d39c320..61cd7e5228 100644
--- a/synapse/replication/slave/storage/client_ips.py
+++ b/synapse/replication/slave/storage/client_ips.py
@@ -12,15 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TYPE_CHECKING
+
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.client_ips import LAST_SEEN_GRANULARITY
 from synapse.util.caches.lrucache import LruCache
 
 from ._base import BaseSlavedStore
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class SlavedClientIpStore(BaseSlavedStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.client_ip_last_seen: LruCache[tuple, int] = LruCache(
diff --git a/synapse/replication/slave/storage/devices.py b/synapse/replication/slave/storage/devices.py
index 26bdead565..0a58296089 100644
--- a/synapse/replication/slave/storage/devices.py
+++ b/synapse/replication/slave/storage/devices.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TYPE_CHECKING
+
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
 from synapse.replication.tcp.streams._base import DeviceListsStream, UserSignatureStream
@@ -20,9 +22,12 @@ from synapse.storage.databases.main.devices import DeviceWorkerStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyWorkerStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class SlavedDeviceStore(EndToEndKeyWorkerStore, DeviceWorkerStore, BaseSlavedStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.hs = hs
diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index d4d3f8c448..63ed50caa5 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+from typing import TYPE_CHECKING
 
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.event_federation import EventFederationWorkerStore
@@ -30,6 +31,9 @@ from synapse.util.caches.stream_change_cache import StreamChangeCache
 
 from ._base import BaseSlavedStore
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,7 +58,7 @@ class SlavedEventStore(
     RelationsWorkerStore,
     BaseSlavedStore,
 ):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         events_max = self._stream_id_gen.get_current_token()
diff --git a/synapse/replication/slave/storage/filtering.py b/synapse/replication/slave/storage/filtering.py
index 37875bc973..90284c202d 100644
--- a/synapse/replication/slave/storage/filtering.py
+++ b/synapse/replication/slave/storage/filtering.py
@@ -12,14 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TYPE_CHECKING
+
 from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.filtering import FilteringStore
 
 from ._base import BaseSlavedStore
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class SlavedFilteringStore(BaseSlavedStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
     # Filters are immutable so this cache doesn't need to be expired
diff --git a/synapse/replication/slave/storage/groups.py b/synapse/replication/slave/storage/groups.py
index e9bdc38470..497e16c69e 100644
--- a/synapse/replication/slave/storage/groups.py
+++ b/synapse/replication/slave/storage/groups.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import TYPE_CHECKING
+
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
 from synapse.replication.tcp.streams import GroupServerStream
@@ -19,9 +21,12 @@ from synapse.storage.database import DatabasePool
 from synapse.storage.databases.main.group_server import GroupServerWorkerStore
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 
 class SlavedGroupServerStore(GroupServerWorkerStore, BaseSlavedStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.hs = hs
diff --git a/synapse/replication/tcp/external_cache.py b/synapse/replication/tcp/external_cache.py
index b402f82810..aaf91e5e02 100644
--- a/synapse/replication/tcp/external_cache.py
+++ b/synapse/replication/tcp/external_cache.py
@@ -21,6 +21,8 @@ from synapse.logging.context import make_deferred_yieldable
 from synapse.util import json_decoder, json_encoder
 
 if TYPE_CHECKING:
+    from txredisapi import RedisProtocol
+
     from synapse.server import HomeServer
 
 set_counter = Counter(
@@ -59,7 +61,12 @@ class ExternalCache:
     """
 
     def __init__(self, hs: "HomeServer"):
-        self._redis_connection = hs.get_outbound_redis_connection()
+        if hs.config.redis.redis_enabled:
+            self._redis_connection: Optional[
+                "RedisProtocol"
+            ] = hs.get_outbound_redis_connection()
+        else:
+            self._redis_connection = None
 
     def _get_redis_key(self, cache_name: str, key: str) -> str:
         return "cache_v1:%s:%s" % (cache_name, key)
diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py
index 6aa9318027..06fd06fdf3 100644
--- a/synapse/replication/tcp/handler.py
+++ b/synapse/replication/tcp/handler.py
@@ -294,7 +294,7 @@ class ReplicationCommandHandler:
             # This shouldn't be possible
             raise Exception("Unrecognised command %s in stream queue", cmd.NAME)
 
-    def start_replication(self, hs):
+    def start_replication(self, hs: "HomeServer"):
         """Helper method to start a replication connection to the remote server
         using TCP.
         """
@@ -321,6 +321,8 @@ class ReplicationCommandHandler:
                 hs.config.redis.redis_host,  # type: ignore[arg-type]
                 hs.config.redis.redis_port,
                 self._factory,
+                timeout=30,
+                bindAddress=None,
             )
         else:
             client_name = hs.get_instance_name()
@@ -331,6 +333,8 @@ class ReplicationCommandHandler:
                 host,  # type: ignore[arg-type]
                 port,
                 self._factory,
+                timeout=30,
+                bindAddress=None,
             )
 
     def get_streams(self) -> Dict[str, Stream]:
diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py
index 80f9b23bfd..55326877fd 100644
--- a/synapse/replication/tcp/resource.py
+++ b/synapse/replication/tcp/resource.py
@@ -16,6 +16,7 @@
 
 import logging
 import random
+from typing import TYPE_CHECKING
 
 from prometheus_client import Counter
 
@@ -27,6 +28,9 @@ from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol
 from synapse.replication.tcp.streams import EventsStream
 from synapse.util.metrics import Measure
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 stream_updates_counter = Counter(
     "synapse_replication_tcp_resource_stream_updates", "", ["stream_name"]
 )
@@ -37,7 +41,7 @@ logger = logging.getLogger(__name__)
 class ReplicationStreamProtocolFactory(Factory):
     """Factory for new replication connections."""
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.command_handler = hs.get_tcp_replication()
         self.clock = hs.get_clock()
         self.server_name = hs.config.server.server_name
@@ -65,7 +69,7 @@ class ReplicationStreamer:
     data is available it will propagate to all connected clients.
     """
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
         self.notifier = hs.get_notifier()
diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py
index 9b905aba9d..c8b188ae4e 100644
--- a/synapse/replication/tcp/streams/_base.py
+++ b/synapse/replication/tcp/streams/_base.py
@@ -241,7 +241,7 @@ class BackfillStream(Stream):
     NAME = "backfill"
     ROW_TYPE = BackfillStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -363,7 +363,7 @@ class ReceiptsStream(Stream):
     NAME = "receipts"
     ROW_TYPE = ReceiptsStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -380,7 +380,7 @@ class PushRulesStream(Stream):
     NAME = "push_rules"
     ROW_TYPE = PushRulesStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
 
         super().__init__(
@@ -405,7 +405,7 @@ class PushersStream(Stream):
     NAME = "pushers"
     ROW_TYPE = PushersStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
 
         super().__init__(
@@ -438,7 +438,7 @@ class CachesStream(Stream):
     NAME = "caches"
     ROW_TYPE = CachesStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -459,7 +459,7 @@ class DeviceListsStream(Stream):
     NAME = "device_lists"
     ROW_TYPE = DeviceListsStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -476,7 +476,7 @@ class ToDeviceStream(Stream):
     NAME = "to_device"
     ROW_TYPE = ToDeviceStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -495,7 +495,7 @@ class TagAccountDataStream(Stream):
     NAME = "tag_account_data"
     ROW_TYPE = TagAccountDataStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -582,7 +582,7 @@ class GroupServerStream(Stream):
     NAME = "groups"
     ROW_TYPE = GroupsStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
@@ -599,7 +599,7 @@ class UserSignatureStream(Stream):
     NAME = "user_signature"
     ROW_TYPE = UserSignatureStreamRow
 
-    def __init__(self, hs):
+    def __init__(self, hs: "HomeServer"):
         store = hs.get_datastore()
         super().__init__(
             hs.get_instance_name(),
diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py
index a6fa03c90f..80fbf32f17 100644
--- a/synapse/rest/admin/devices.py
+++ b/synapse/rest/admin/devices.py
@@ -110,7 +110,7 @@ class DevicesRestServlet(RestServlet):
     def __init__(self, hs: "HomeServer"):
         """
         Args:
-            hs (synapse.server.HomeServer): server
+            hs: server
         """
         self.hs = hs
         self.auth = hs.get_auth()
diff --git a/synapse/server.py b/synapse/server.py
index a64c846d1c..0fbf36ba99 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -800,9 +800,14 @@ class HomeServer(metaclass=abc.ABCMeta):
         return ExternalCache(self)
 
     @cache_in_self
-    def get_outbound_redis_connection(self) -> Optional["RedisProtocol"]:
-        if not self.config.redis.redis_enabled:
-            return None
+    def get_outbound_redis_connection(self) -> "RedisProtocol":
+        """
+        The Redis connection used for replication.
+
+        Raises:
+            AssertionError: if Redis is not enabled in the homeserver config.
+        """
+        assert self.config.redis.redis_enabled
 
         # We only want to import redis module if we're using it, as we have
         # `txredisapi` as an optional dependency.
diff --git a/synapse/storage/database.py b/synapse/storage/database.py
index f5a8f90a0f..fa4e89d35c 100644
--- a/synapse/storage/database.py
+++ b/synapse/storage/database.py
@@ -19,6 +19,7 @@ from collections import defaultdict
 from sys import intern
 from time import monotonic as monotonic_time
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
     Collection,
@@ -52,6 +53,9 @@ from synapse.storage.background_updates import BackgroundUpdater
 from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
 from synapse.storage.types import Connection, Cursor
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 # python 3 does not have a maximum int value
 MAX_TXN_ID = 2 ** 63 - 1
 
@@ -392,7 +396,7 @@ class DatabasePool:
 
     def __init__(
         self,
-        hs,
+        hs: "HomeServer",
         database_config: DatabaseConnectionConfig,
         engine: BaseDatabaseEngine,
     ):
diff --git a/synapse/storage/databases/__init__.py b/synapse/storage/databases/__init__.py
index 20b755056b..cfe887b7f7 100644
--- a/synapse/storage/databases/__init__.py
+++ b/synapse/storage/databases/__init__.py
@@ -13,33 +13,49 @@
 # limitations under the License.
 
 import logging
+from typing import TYPE_CHECKING, Generic, List, Optional, Type, TypeVar
 
+from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import DatabasePool, make_conn
 from synapse.storage.databases.main.events import PersistEventsStore
 from synapse.storage.databases.state import StateGroupDataStore
 from synapse.storage.engines import create_engine
 from synapse.storage.prepare_database import prepare_database
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
-class Databases:
+DataStoreT = TypeVar("DataStoreT", bound=SQLBaseStore, covariant=True)
+
+
+class Databases(Generic[DataStoreT]):
     """The various databases.
 
     These are low level interfaces to physical databases.
 
     Attributes:
-        main (DataStore)
+        databases
+        main
+        state
+        persist_events
     """
 
-    def __init__(self, main_store_class, hs):
+    databases: List[DatabasePool]
+    main: DataStoreT
+    state: StateGroupDataStore
+    persist_events: Optional[PersistEventsStore]
+
+    def __init__(self, main_store_class: Type[DataStoreT], hs: "HomeServer"):
         # Note we pass in the main store class here as workers use a different main
         # store.
 
         self.databases = []
-        main = None
-        state = None
-        persist_events = None
+        main: Optional[DataStoreT] = None
+        state: Optional[StateGroupDataStore] = None
+        persist_events: Optional[PersistEventsStore] = None
 
         for database_config in hs.config.database.databases:
             db_name = database_config.name
diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py
index 5c21402dea..259cae5b37 100644
--- a/synapse/storage/databases/main/__init__.py
+++ b/synapse/storage/databases/main/__init__.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import logging
-from typing import List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple
 
 from synapse.config.homeserver import HomeServerConfig
 from synapse.storage.database import DatabasePool
@@ -75,6 +75,9 @@ from .ui_auth import UIAuthStore
 from .user_directory import UserDirectoryStore
 from .user_erasure_store import UserErasureStore
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -126,7 +129,7 @@ class DataStore(
     LockStore,
     SessionStore,
 ):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         self.hs = hs
         self._clock = hs.get_clock()
         self.database_engine = database.engine
diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py
index 70ca3e09f7..f8bec266ac 100644
--- a/synapse/storage/databases/main/account_data.py
+++ b/synapse/storage/databases/main/account_data.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 import logging
-from typing import Dict, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
 
 from synapse.api.constants import AccountDataTypes
 from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
@@ -28,6 +28,9 @@ from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -36,7 +39,7 @@ class AccountDataWorkerStore(SQLBaseStore):
     `get_max_account_data_stream_id` which can be called in the initializer.
     """
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         self._instance_name = hs.get_instance_name()
 
         if isinstance(database.engine, PostgresEngine):
diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py
index c57ae5ef15..36e8422fc6 100644
--- a/synapse/storage/databases/main/cache.py
+++ b/synapse/storage/databases/main/cache.py
@@ -15,7 +15,7 @@
 
 import itertools
 import logging
-from typing import Any, Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple
 
 from synapse.api.constants import EventTypes
 from synapse.replication.tcp.streams import BackfillStream, CachesStream
@@ -29,6 +29,9 @@ from synapse.storage.database import DatabasePool
 from synapse.storage.engines import PostgresEngine
 from synapse.util.iterutils import batch_iter
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -38,7 +41,7 @@ CURRENT_STATE_CACHE_NAME = "cs_cache_fake"
 
 
 class CacheInvalidationWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self._instance_name = hs.get_instance_name()
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 3154906d45..8143168107 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple
 
 from synapse.logging import issue9533_logger
 from synapse.logging.opentracing import log_kv, set_tag, trace
@@ -26,11 +26,14 @@ from synapse.util import json_encoder
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class DeviceInboxWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self._instance_name = hs.get_instance_name()
@@ -553,7 +556,7 @@ class DeviceInboxWorkerStore(SQLBaseStore):
 class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
     DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_index_update(
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index 6464520386..a01bf2c5b7 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -15,7 +15,17 @@
 # limitations under the License.
 import abc
 import logging
-from typing import Any, Collection, Dict, Iterable, List, Optional, Set, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Collection,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+)
 
 from synapse.api.errors import Codes, StoreError
 from synapse.logging.opentracing import (
@@ -38,6 +48,9 @@ from synapse.util.caches.lrucache import LruCache
 from synapse.util.iterutils import batch_iter
 from synapse.util.stringutils import shortstr
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES = (
@@ -48,7 +61,7 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes"
 
 
 class DeviceWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         if hs.config.worker.run_background_tasks:
@@ -915,7 +928,7 @@ class DeviceWorkerStore(SQLBaseStore):
 
 
 class DeviceBackgroundUpdateStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_index_update(
@@ -1047,7 +1060,7 @@ class DeviceBackgroundUpdateStore(SQLBaseStore):
 
 
 class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         # Map of (user_id, device_id) -> bool. If there is an entry that implies
diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py
index ba9f71a230..ef5d1ef01e 100644
--- a/synapse/storage/databases/main/event_federation.py
+++ b/synapse/storage/databases/main/event_federation.py
@@ -14,7 +14,7 @@
 import itertools
 import logging
 from queue import Empty, PriorityQueue
-from typing import Collection, Dict, Iterable, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Optional, Set, Tuple
 
 from prometheus_client import Counter, Gauge
 
@@ -34,6 +34,9 @@ from synapse.util.caches.descriptors import cached
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.iterutils import batch_iter
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 oldest_pdu_in_federation_staging = Gauge(
     "synapse_federation_server_oldest_inbound_pdu_in_staging",
     "The age in seconds since we received the oldest pdu in the federation staging area",
@@ -59,7 +62,7 @@ class _NoChainCoverIndex(Exception):
 
 
 class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         if hs.config.worker.run_background_tasks:
@@ -1511,7 +1514,7 @@ class EventFederationStore(EventFederationWorkerStore):
 
     EVENT_AUTH_STATE_ONLY = "event_auth_state_only"
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_update_handler(
diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py
index 97b3e92d3f..d957e770dc 100644
--- a/synapse/storage/databases/main/event_push_actions.py
+++ b/synapse/storage/databases/main/event_push_actions.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
 
 import attr
 
@@ -23,6 +23,9 @@ from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -64,7 +67,7 @@ def _deserialize_action(actions, is_highlight):
 
 
 class EventPushActionsWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         # These get correctly set by _find_stream_orderings_for_times_txn
@@ -892,7 +895,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
 class EventPushActionsStore(EventPushActionsWorkerStore):
     EPA_HIGHLIGHT_INDEX = "epa_highlight_index"
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_index_update(
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
index 1afc59fafb..fc49112063 100644
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 
 import attr
 
@@ -26,6 +26,9 @@ from synapse.storage.databases.main.events import PersistEventsStore
 from synapse.storage.types import Cursor
 from synapse.types import JsonDict
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -76,7 +79,7 @@ class _CalculateChainCover:
 
 
 class EventsBackgroundUpdatesStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_update_handler(
diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py
index 2fa945d171..717487be28 100644
--- a/synapse/storage/databases/main/media_repository.py
+++ b/synapse/storage/databases/main/media_repository.py
@@ -13,11 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from enum import Enum
-from typing import Any, Dict, Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple
 
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import DatabasePool
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 BG_UPDATE_REMOVE_MEDIA_REPO_INDEX_WITHOUT_METHOD = (
     "media_repository_drop_index_wo_method"
 )
@@ -43,7 +46,7 @@ class MediaSortOrder(Enum):
 
 
 class MediaRepositoryBackgroundUpdateStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.db_pool.updates.register_background_index_update(
@@ -123,7 +126,7 @@ class MediaRepositoryBackgroundUpdateStore(SQLBaseStore):
 class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):
     """Persistence for attachments and avatars"""
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
         self.server_name = hs.hostname
 
diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py
index dac3d14da8..d901933ae4 100644
--- a/synapse/storage/databases/main/metrics.py
+++ b/synapse/storage/databases/main/metrics.py
@@ -14,7 +14,7 @@
 import calendar
 import logging
 import time
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
 
 from synapse.metrics import GaugeBucketCollector
 from synapse.metrics.background_process_metrics import wrap_as_background_process
@@ -24,6 +24,9 @@ from synapse.storage.databases.main.event_push_actions import (
     EventPushActionsWorkerStore,
 )
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 # Collect metrics on the number of forward extremities that exist.
@@ -52,7 +55,7 @@ class ServerMetricsStore(EventPushActionsWorkerStore, SQLBaseStore):
     stats and prometheus metrics.
     """
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         # Read the extrems every 60 minutes
diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py
index a14ac03d4b..b5284e4f67 100644
--- a/synapse/storage/databases/main/monthly_active_users.py
+++ b/synapse/storage/databases/main/monthly_active_users.py
@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Dict, List, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.storage._base import SQLBaseStore
 from synapse.storage.database import DatabasePool, make_in_list_sql_clause
 from synapse.util.caches.descriptors import cached
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 # Number of msec of granularity to store the monthly_active_user timestamp
@@ -27,7 +30,7 @@ LAST_SEEN_GRANULARITY = 60 * 60 * 1000
 
 
 class MonthlyActiveUsersWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
         self._clock = hs.get_clock()
         self.hs = hs
@@ -209,7 +212,7 @@ class MonthlyActiveUsersWorkerStore(SQLBaseStore):
 
 
 class MonthlyActiveUsersStore(MonthlyActiveUsersWorkerStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self._mau_stats_only = hs.config.server.mau_stats_only
diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py
index fc720f5947..fa782023d4 100644
--- a/synapse/storage/databases/main/push_rule.py
+++ b/synapse/storage/databases/main/push_rule.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 import abc
 import logging
-from typing import Dict, List, Tuple, Union
+from typing import TYPE_CHECKING, Dict, List, Tuple, Union
 
 from synapse.api.errors import NotFoundError, StoreError
 from synapse.push.baserules import list_with_base_rules
@@ -33,6 +33,9 @@ from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -75,7 +78,7 @@ class PushRulesWorkerStore(
     `get_max_push_rules_stream_id` which can be called in the initializer.
     """
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         if hs.config.worker.worker_app is None:
diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py
index 01a4281301..c99f8aebdb 100644
--- a/synapse/storage/databases/main/receipts.py
+++ b/synapse/storage/databases/main/receipts.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 import logging
-from typing import Any, Dict, Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple
 
 from twisted.internet import defer
 
@@ -29,11 +29,14 @@ from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
 class ReceiptsWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         self._instance_name = hs.get_instance_name()
 
         if isinstance(database.engine, PostgresEngine):
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 835d7889cb..f879bbe7c7 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -17,7 +17,7 @@ import collections
 import logging
 from abc import abstractmethod
 from enum import Enum
-from typing import Any, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from synapse.api.constants import EventContentFields, EventTypes, JoinRules
 from synapse.api.errors import StoreError
@@ -32,6 +32,9 @@ from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached
 from synapse.util.stringutils import MXC_REGEX
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -69,7 +72,7 @@ class RoomSortOrder(Enum):
 
 
 class RoomWorkerStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.config = hs.config
@@ -1026,7 +1029,7 @@ _REPLACE_ROOM_DEPTH_SQL_COMMANDS = (
 
 
 class RoomBackgroundUpdateStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.config = hs.config
@@ -1411,7 +1414,7 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
 
 
 class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.config = hs.config
diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py
index ddb162a4fc..4b288bb2e7 100644
--- a/synapse/storage/databases/main/roommember.py
+++ b/synapse/storage/databases/main/roommember.py
@@ -53,6 +53,7 @@ from synapse.util.caches.descriptors import _CacheContext, cached, cachedList
 from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
+    from synapse.server import HomeServer
     from synapse.state import _StateCacheEntry
 
 logger = logging.getLogger(__name__)
@@ -63,7 +64,7 @@ _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership"
 
 
 class RoomMemberWorkerStore(EventsWorkerStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         # Used by `_get_joined_hosts` to ensure only one thing mutates the cache
@@ -982,7 +983,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
 
 
 class RoomMemberBackgroundUpdateStore(SQLBaseStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
         self.db_pool.updates.register_background_update_handler(
             _MEMBERSHIP_PROFILE_UPDATE_NAME, self._background_add_membership_profile
@@ -1132,7 +1133,7 @@ class RoomMemberBackgroundUpdateStore(SQLBaseStore):
 
 
 class RoomMemberStore(RoomMemberWorkerStore, RoomMemberBackgroundUpdateStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
     async def forget(self, user_id: str, room_id: str) -> None:
diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py
index c85383c975..7fe233767f 100644
--- a/synapse/storage/databases/main/search.py
+++ b/synapse/storage/databases/main/search.py
@@ -15,7 +15,7 @@
 import logging
 import re
 from collections import namedtuple
-from typing import Collection, Iterable, List, Optional, Set
+from typing import TYPE_CHECKING, Collection, Iterable, List, Optional, Set
 
 from synapse.api.errors import SynapseError
 from synapse.events import EventBase
@@ -24,6 +24,9 @@ from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 SearchEntry = namedtuple(
@@ -102,7 +105,7 @@ class SearchBackgroundUpdateStore(SearchWorkerStore):
     EVENT_SEARCH_USE_GIST_POSTGRES_NAME = "event_search_postgres_gist"
     EVENT_SEARCH_USE_GIN_POSTGRES_NAME = "event_search_postgres_gin"
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         if not hs.config.server.enable_search:
@@ -355,7 +358,7 @@ class SearchBackgroundUpdateStore(SearchWorkerStore):
 
 
 class SearchStore(SearchBackgroundUpdateStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
     async def search_msgs(self, room_ids, search_term, keys):
diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py
index a8e8dd4577..fa2c3b1feb 100644
--- a/synapse/storage/databases/main/state.py
+++ b/synapse/storage/databases/main/state.py
@@ -15,7 +15,7 @@
 import collections.abc
 import logging
 from collections import namedtuple
-from typing import Iterable, Optional, Set
+from typing import TYPE_CHECKING, Iterable, Optional, Set
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError
@@ -30,6 +30,9 @@ from synapse.types import StateMap
 from synapse.util.caches import intern_string
 from synapse.util.caches.descriptors import cached, cachedList
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 
@@ -53,7 +56,7 @@ class _GetStateGroupDelta(
 class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
     """The parts of StateGroupStore that can be called from workers."""
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
     async def get_room_version(self, room_id: str) -> RoomVersion:
@@ -346,7 +349,7 @@ class MainStateBackgroundUpdateStore(RoomMemberWorkerStore):
     EVENT_STATE_GROUP_INDEX_UPDATE_NAME = "event_to_state_groups_sg_index"
     DELETE_CURRENT_STATE_UPDATE_NAME = "delete_old_current_state_events"
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.server_name = hs.hostname
@@ -533,5 +536,5 @@ class StateStore(StateGroupWorkerStore, MainStateBackgroundUpdateStore):
       * `state_groups_state`: Maps state group to state events.
     """
 
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py
index e20033bb28..5d7b59d861 100644
--- a/synapse/storage/databases/main/stats.py
+++ b/synapse/storage/databases/main/stats.py
@@ -16,7 +16,7 @@
 import logging
 from enum import Enum
 from itertools import chain
-from typing import Any, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from typing_extensions import Counter
 
@@ -29,6 +29,9 @@ from synapse.storage.databases.main.state_deltas import StateDeltasStore
 from synapse.types import JsonDict
 from synapse.util.caches.descriptors import cached
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 # these fields track absolutes (e.g. total number of rooms on the server)
@@ -93,7 +96,7 @@ class UserSortOrder(Enum):
 
 
 class StatsStore(StateDeltasStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         self.server_name = hs.hostname
diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py
index 860146cd1b..d7dc1f73ac 100644
--- a/synapse/storage/databases/main/transactions.py
+++ b/synapse/storage/databases/main/transactions.py
@@ -14,7 +14,7 @@
 
 import logging
 from collections import namedtuple
-from typing import Iterable, List, Optional, Tuple
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple
 
 import attr
 from canonicaljson import encode_canonical_json
@@ -26,6 +26,9 @@ from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
 from synapse.types import JsonDict
 from synapse.util.caches.descriptors import cached
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 db_binary_type = memoryview
 
 logger = logging.getLogger(__name__)
@@ -57,7 +60,7 @@ class DestinationRetryTimings:
 
 
 class TransactionWorkerStore(CacheInvalidationWorkerStore):
-    def __init__(self, database: DatabasePool, db_conn, hs):
+    def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
 
         if hs.config.worker.run_background_tasks:
diff --git a/synapse/storage/persist_events.py b/synapse/storage/persist_events.py
index 0e8270746d..402f134d89 100644
--- a/synapse/storage/persist_events.py
+++ b/synapse/storage/persist_events.py
@@ -18,6 +18,7 @@ import itertools
 import logging
 from collections import deque
 from typing import (
+    TYPE_CHECKING,
     Any,
     Awaitable,
     Callable,
@@ -56,6 +57,9 @@ from synapse.types import (
 from synapse.util.async_helpers import ObservableDeferred, yieldable_gather_results
 from synapse.util.metrics import Measure
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 logger = logging.getLogger(__name__)
 
 # The number of times we are recalculating the current state
@@ -272,7 +276,7 @@ class EventsPersistenceStorage:
     current state and forward extremity changes.
     """
 
-    def __init__(self, hs, stores: Databases):
+    def __init__(self, hs: "HomeServer", stores: Databases):
         # We ultimately want to split out the state store from the main store,
         # so we use separate variables here even though they point to the same
         # store for now.
-- 
cgit 1.5.1


From 12d79ff1b6287b148e84d3b2c14dfbbfc6e11361 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Mon, 18 Oct 2021 16:44:27 +0100
Subject: Fix Shellcheck SC2164: exit in case cd fails.

Use `cd ... || exit` in case cd fails.

https://github.com/koalaman/shellcheck/wiki/SC2164

Signed-off-by: Dan Callahan <danc@element.io>
---
 demo/start.sh                         | 8 ++++----
 scripts-dev/check_line_terminators.sh | 2 +-
 scripts-dev/config-lint.sh            | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/demo/start.sh b/demo/start.sh
index bc4854091b..fc5d08a63b 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -4,7 +4,7 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 CWD=$(pwd)
 
-cd "$DIR/.."
+cd "$DIR/.." || exit
 
 mkdir -p demo/etc
 
@@ -18,7 +18,7 @@ for port in 8080 8081 8082; do
 
     https_port=$((port + 400))
     mkdir -p demo/$port
-    pushd demo/$port
+    pushd demo/$port || exit
 
     #rm $DIR/etc/$port.config
     python3 -m synapse.app.homeserver \
@@ -152,7 +152,7 @@ for port in 8080 8081 8082; do
         --config-path "$DIR/etc/$port.config" \
         -D \
 
-    popd
+    popd || exit
 done
 
-cd "$CWD"
+cd "$CWD" || exit
diff --git a/scripts-dev/check_line_terminators.sh b/scripts-dev/check_line_terminators.sh
index c983956231..762f848cab 100755
--- a/scripts-dev/check_line_terminators.sh
+++ b/scripts-dev/check_line_terminators.sh
@@ -25,7 +25,7 @@
 # terminators are found, 0 otherwise.
 
 # cd to the root of the repository
-cd `dirname $0`/..
+cd `dirname $0`/.. || exit
 
 # Find and print files with non-unix line terminators
 if find . -path './.git/*' -prune -o -type f -print0 | xargs -0 grep -I -l $'\r$'; then
diff --git a/scripts-dev/config-lint.sh b/scripts-dev/config-lint.sh
index 8c6323e59a..6b160c8ad2 100755
--- a/scripts-dev/config-lint.sh
+++ b/scripts-dev/config-lint.sh
@@ -3,7 +3,7 @@
 # Exits with 0 if there are no problems, or another code otherwise.
 
 # cd to the root of the repository
-cd `dirname $0`/..
+cd `dirname $0`/.. || exit
 
 # Restore backup of sample config upon script exit
 trap "mv docs/sample_config.yaml.bak docs/sample_config.yaml" EXIT
-- 
cgit 1.5.1


From 64adbb7b5425bb618077e1ab85faede1301de024 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Mon, 18 Oct 2021 16:55:18 +0100
Subject: Fix Shellcheck SC2046: Quote to prevent word split

Quote this to prevent word splitting

https://www.shellcheck.net/wiki/SC2046

Signed-off-by: Dan Callahan <danc@element.io>
---
 .ci/scripts/test_export_data_command.sh       | 2 +-
 .ci/scripts/test_synapse_port_db.sh           | 2 +-
 debian/test/provision.sh                      | 2 +-
 demo/start.sh                                 | 2 +-
 scripts-dev/check_line_terminators.sh         | 2 +-
 scripts-dev/config-lint.sh                    | 2 +-
 scripts-dev/docker_update_debian_changelog.sh | 2 +-
 scripts-dev/generate_sample_config            | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.ci/scripts/test_export_data_command.sh b/.ci/scripts/test_export_data_command.sh
index 75f5811d10..2180072c06 100755
--- a/.ci/scripts/test_export_data_command.sh
+++ b/.ci/scripts/test_export_data_command.sh
@@ -3,7 +3,7 @@
 # Test for the export-data admin command against sqlite and postgres
 
 set -xe
-cd `dirname $0`/../..
+cd "`dirname $0`/../.."
 
 echo "--- Install dependencies"
 
diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh
index 50115b3079..7bcbe63654 100755
--- a/.ci/scripts/test_synapse_port_db.sh
+++ b/.ci/scripts/test_synapse_port_db.sh
@@ -7,7 +7,7 @@
 
 
 set -xe
-cd `dirname $0`/../..
+cd "`dirname $0`/../.."
 
 echo "--- Install dependencies"
 
diff --git a/debian/test/provision.sh b/debian/test/provision.sh
index a5c7f59712..9dc8ab671c 100644
--- a/debian/test/provision.sh
+++ b/debian/test/provision.sh
@@ -10,7 +10,7 @@ set -e
 apt-get update
 apt-get install -y lsb-release
 
-deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
+deb=`ls "/debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb" | sort | tail -n1`
 
 debconf-set-selections <<EOF
 matrix-synapse matrix-synapse/report-stats boolean false
diff --git a/demo/start.sh b/demo/start.sh
index fc5d08a63b..e426094994 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -8,7 +8,7 @@ cd "$DIR/.." || exit
 
 mkdir -p demo/etc
 
-export PYTHONPATH=$(readlink -f $(pwd))
+export PYTHONPATH=$(readlink -f "$(pwd)")
 
 
 echo $PYTHONPATH
diff --git a/scripts-dev/check_line_terminators.sh b/scripts-dev/check_line_terminators.sh
index 762f848cab..068f157ea1 100755
--- a/scripts-dev/check_line_terminators.sh
+++ b/scripts-dev/check_line_terminators.sh
@@ -25,7 +25,7 @@
 # terminators are found, 0 otherwise.
 
 # cd to the root of the repository
-cd `dirname $0`/.. || exit
+cd "`dirname $0`/.." || exit
 
 # Find and print files with non-unix line terminators
 if find . -path './.git/*' -prune -o -type f -print0 | xargs -0 grep -I -l $'\r$'; then
diff --git a/scripts-dev/config-lint.sh b/scripts-dev/config-lint.sh
index 6b160c8ad2..15c8c2f93a 100755
--- a/scripts-dev/config-lint.sh
+++ b/scripts-dev/config-lint.sh
@@ -3,7 +3,7 @@
 # Exits with 0 if there are no problems, or another code otherwise.
 
 # cd to the root of the repository
-cd `dirname $0`/.. || exit
+cd "`dirname $0`/.." || exit
 
 # Restore backup of sample config upon script exit
 trap "mv docs/sample_config.yaml.bak docs/sample_config.yaml" EXIT
diff --git a/scripts-dev/docker_update_debian_changelog.sh b/scripts-dev/docker_update_debian_changelog.sh
index 89821bba72..14c155aee5 100755
--- a/scripts-dev/docker_update_debian_changelog.sh
+++ b/scripts-dev/docker_update_debian_changelog.sh
@@ -60,5 +60,5 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y devscripts
 
 # Update the Debian changelog.
 ver=${1}
-dch -M -v $(sed -Ee 's/(rc|a|b|c)/~\1/' <<<$ver) "New synapse release $ver."
+dch -M -v "$(sed -Ee 's/(rc|a|b|c)/~\1/' <<<$ver)" "New synapse release $ver."
 dch -M -r -D stable ""
diff --git a/scripts-dev/generate_sample_config b/scripts-dev/generate_sample_config
index 02739894b5..06e4c8c669 100755
--- a/scripts-dev/generate_sample_config
+++ b/scripts-dev/generate_sample_config
@@ -4,7 +4,7 @@
 
 set -e
 
-cd `dirname $0`/..
+cd "`dirname $0`/.."
 
 SAMPLE_CONFIG="docs/sample_config.yaml"
 SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
-- 
cgit 1.5.1


From 5eb481cd5b825220f6b8283a553cce936bd6d8c4 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Mon, 18 Oct 2021 17:00:05 +0100
Subject: Fix Shellcheck SC2115: Ensure never expands to /*

Use "${var:?}" to ensure this never expands to /* .

https://github.com/koalaman/shellcheck/wiki/SC2115

Signed-off-by: Dan Callahan <danc@element.io>
---
 demo/clean.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/demo/clean.sh b/demo/clean.sh
index 6b809f6e83..99482e87d3 100755
--- a/demo/clean.sh
+++ b/demo/clean.sh
@@ -12,8 +12,8 @@ if [ -f $PID_FILE ]; then
 fi
 
 for port in 8080 8081 8082; do
-    rm -rf $DIR/$port
+    rm -rf ${DIR:?}/$port
     rm -rf $DIR/media_store.$port
 done
 
-rm -rf $DIR/etc
+rm -rf ${DIR:?}/etc
-- 
cgit 1.5.1


From 898e3be4c9582551093e0918bba2398cd08fb0e0 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Mon, 18 Oct 2021 21:49:09 +0100
Subject: Fix Shellcheck SC2064: Use single quotes on traps

Use single quotes, otherwise this expands now rather than when signalled.

https://github.com/koalaman/shellcheck/wiki/SC2064

Signed-off-by: Dan Callahan <danc@element.io>
---
 debian/build_virtualenv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index 801ecb9086..e006b48849 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -57,7 +57,7 @@ case "$DEB_BUILD_OPTIONS" in
         # Copy tests to a temporary directory so that we can put them on the
         # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
         tmpdir=`mktemp -d`
-        trap "rm -r $tmpdir" EXIT
+        trap 'rm -r $tmpdir' EXIT
 
         cp -r tests "$tmpdir"
 
-- 
cgit 1.5.1


From 6c736fa47202bcffb4f056d6e5b2f12424e9f758 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Thu, 21 Oct 2021 22:58:48 +0100
Subject: Fix Shellcheck SC2154: variable possibly undefined

var is referenced but not assigned.

https://github.com/koalaman/shellcheck/wiki/SC2154

Signed-off-by: Dan Callahan <danc@element.io>
---
 docker/build_debian.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index 801ff45471..1d0be42cfc 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -5,7 +5,7 @@
 set -ex
 
 # Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< $distro`
+DIST=`cut -d ':' -f2 <<< ${distro:?}`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build
-- 
cgit 1.5.1


From 6a9d84a6761bd1afb8223512cec0c9809c4db9b5 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Thu, 21 Oct 2021 23:06:41 +0100
Subject: Fix Shellcheck SC2166: test -a is not well defined

Prefer [ p ] && [ q ] as [ p -a q ] is not well defined.

https://github.com/koalaman/shellcheck/wiki/SC2166

Signed-off-by: Dan Callahan <danc@element.io>
---
 scripts-dev/check-newsfragment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index 393a548d58..56600de0ad 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -45,7 +45,7 @@ matched=0
 for f in `git diff --name-only FETCH_HEAD... -- changelog.d`; do
     # check that any modified newsfiles on this branch end with a full stop.
     lastchar=`tr -d '\n' < $f | tail -c 1`
-    if [ $lastchar != '.' -a $lastchar != '!' ]; then
+    if [ $lastchar != '.' ] && [ $lastchar != '!' ]; then
         echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2
         echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2
         exit 1
-- 
cgit 1.5.1


From dfa61431330075f21589a4799fa2d52a42eb6ccc Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Thu, 21 Oct 2021 23:10:14 +0100
Subject: Fix Shellcheck SC2155: Declare + export separately

Declare and assign separately to avoid masking return values.

https://github.com/koalaman/shellcheck/wiki/SC2155

Signed-off-by: Dan Callahan <danc@element.io>
---
 demo/start.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/demo/start.sh b/demo/start.sh
index e426094994..28ea1b1cb6 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -8,7 +8,8 @@ cd "$DIR/.." || exit
 
 mkdir -p demo/etc
 
-export PYTHONPATH=$(readlink -f "$(pwd)")
+PYTHONPATH=$(readlink -f "$(pwd)")
+export PYTHONPATH
 
 
 echo $PYTHONPATH
-- 
cgit 1.5.1


From 99e698d6ed4ac307fa0be1e5dcce4bba0b2d7069 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Thu, 21 Oct 2021 23:38:29 +0100
Subject: Fix Shellcheck SC2089 and SC2090: Quotes in vars

SC2089: Quotes/backslashes will be treated literally. Use an array.

https://github.com/koalaman/shellcheck/wiki/SC2089

SC2090: Quotes/backslashes in this variable will not be respected.

https://github.com/koalaman/shellcheck/wiki/SC2090

Putting literal JSON in a variable mistakenly triggers these warnings.
Instead of adding ignore directives, this can be avoided by inlining the
JSON data into the curl invocation.

Since the variable is only used in this one location, inlining is fine.

Signed-off-by: Dan Callahan <danc@element.io>
---
 contrib/purge_api/purge_history.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/contrib/purge_api/purge_history.sh b/contrib/purge_api/purge_history.sh
index 9d5324ea1c..de58dcdbb7 100644
--- a/contrib/purge_api/purge_history.sh
+++ b/contrib/purge_api/purge_history.sh
@@ -84,7 +84,9 @@ AUTH="Authorization: Bearer $TOKEN"
 ###################################################################################################
 # finally start pruning the room:
 ###################################################################################################
-POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
+# this will really delete local events, so the messages in the room really
+# disappear unless they are restored by remote federation. This is because
+# we pass {"delete_local_events":true} to the curl invocation below.
 
 for ROOM in "${ROOMS_ARRAY[@]}"; do
     echo "########################################### $(date) ################# "
@@ -104,7 +106,7 @@ for ROOM in "${ROOMS_ARRAY[@]}"; do
       SLEEP=2
       set -x
       # call purge
-      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
+      OUT=$(curl --header "$AUTH" -s -d '{"delete_local_events":true}' POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
       PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
       if [ "$PURGE_ID" == "" ]; then
         # probably the history purge is already in progress for $ROOM
-- 
cgit 1.5.1


From 7cf83c0acaaecd53c7cedda73fbff4704b256eac Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Thu, 21 Oct 2021 23:47:20 +0100
Subject: Fix Shellcheck SC1001: Meaningless char escapes

This \o will be a regular 'o' in this context.

https://github.com/koalaman/shellcheck/wiki/SC1001

Signed-off-by: Dan Callahan <danc@element.io>
---
 demo/start.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/demo/start.sh b/demo/start.sh
index 28ea1b1cb6..245e9d777b 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -70,7 +70,7 @@ for port in 8080 8081 8082; do
         echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
 
         # Generate tls keys
-        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
+        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost:$https_port.tls.key -out $DIR/etc/localhost:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
 
         # Ignore keys from the trusted keys server
         echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
-- 
cgit 1.5.1


From bab2bc844c4e81461c51a1066b2b0322326938d1 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Thu, 21 Oct 2021 23:50:21 +0100
Subject: Fix Shellcheck SC1091: Can't follow file

Not following: (error message here)

https://github.com/koalaman/shellcheck/wiki/SC1091

Signed-off-by: Dan Callahan <danc@element.io>
---
 debian/matrix-synapse-py3.config   | 1 +
 debian/matrix-synapse-py3.postinst | 1 +
 2 files changed, 2 insertions(+)

diff --git a/debian/matrix-synapse-py3.config b/debian/matrix-synapse-py3.config
index 37a781b3e8..3b2f469e15 100755
--- a/debian/matrix-synapse-py3.config
+++ b/debian/matrix-synapse-py3.config
@@ -2,6 +2,7 @@
 
 set -e
 
+# shellcheck disable=SC1091
 . /usr/share/debconf/confmodule
 
 # try to update the debconf db according to whatever is in the config files
diff --git a/debian/matrix-synapse-py3.postinst b/debian/matrix-synapse-py3.postinst
index c0dd7e5534..a8dde1e082 100644
--- a/debian/matrix-synapse-py3.postinst
+++ b/debian/matrix-synapse-py3.postinst
@@ -1,5 +1,6 @@
 #!/bin/sh -e
 
+# shellcheck disable=SC1091
 . /usr/share/debconf/confmodule
 
 CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml"
-- 
cgit 1.5.1


From 9d0f9d51d5da7ebcd3178fefc1e68c3c5494d673 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Fri, 22 Oct 2021 21:44:25 +0100
Subject: Fix Shellcheck SC2016: Single quotes don't expand

Expressions don't expand in single quotes, use double quotes for that.

https://github.com/koalaman/shellcheck/wiki/SC2016

This specifically warned about the '$aregis...' part of the sed script.
Which is a relatively obscure use of sed.

Splitting this into two commands makes its intent more obvious and
avoids contravening Shellcheck's lints.

Signed-off-by: Dan Callahan <danc@element.io>
---
 debian/test/provision.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/debian/test/provision.sh b/debian/test/provision.sh
index 9dc8ab671c..931f300e97 100644
--- a/debian/test/provision.sh
+++ b/debian/test/provision.sh
@@ -19,5 +19,6 @@ EOF
 
 dpkg -i "$deb"
 
-sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
+sed -i -e 's/port: 8448$/port: 18448/; s/port: 8008$/port: 18008' /etc/matrix-synapse/homeserver.yaml
+echo 'registration_shared_secret: secret' >> /etc/matrix-synapse/homeserver.yaml
 systemctl restart matrix-synapse
-- 
cgit 1.5.1


From 31096132c313e2a72d66e3eb13612e37350800ea Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Fri, 22 Oct 2021 21:58:48 +0100
Subject: Fix Shellcheck SC2012: Use find instead of ls

Use find instead of ls to better handle non-alphanumeric filenames.

https://github.com/koalaman/shellcheck/wiki/SC2012

Signed-off-by: Dan Callahan <danc@element.io>
---
 debian/test/provision.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/test/provision.sh b/debian/test/provision.sh
index 931f300e97..8567b146ad 100644
--- a/debian/test/provision.sh
+++ b/debian/test/provision.sh
@@ -10,7 +10,7 @@ set -e
 apt-get update
 apt-get install -y lsb-release
 
-deb=`ls "/debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb" | sort | tail -n1`
+deb=`find /debs -name "matrix-synapse-py3_*+$(lsb_release -cs)*.deb" | sort | tail -n1`
 
 debconf-set-selections <<EOF
 matrix-synapse matrix-synapse/report-stats boolean false
-- 
cgit 1.5.1


From 13f084eb58c379b2fc073680818d5931e94626f4 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Fri, 22 Oct 2021 22:46:06 +0100
Subject: Fix Shellcheck SC2086: Quote to prevent splitting

Double quote to prevent globbing and word splitting.

https://github.com/koalaman/shellcheck/wiki/SC2086

Signed-off-by: Dan Callahan <danc@element.io>
---
 .ci/scripts/test_export_data_command.sh       |  2 +-
 .ci/scripts/test_synapse_port_db.sh           |  2 +-
 debian/build_virtualenv                       |  2 +-
 demo/clean.sh                                 |  8 ++---
 demo/start.sh                                 | 48 +++++++++++++--------------
 demo/stop.sh                                  |  2 +-
 docker/build_debian.sh                        |  2 +-
 scripts-dev/check-newsfragment                |  4 +--
 scripts-dev/check_line_terminators.sh         |  2 +-
 scripts-dev/complement.sh                     |  6 ++--
 scripts-dev/config-lint.sh                    |  2 +-
 scripts-dev/docker_update_debian_changelog.sh |  2 +-
 scripts-dev/generate_sample_config            |  2 +-
 13 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/.ci/scripts/test_export_data_command.sh b/.ci/scripts/test_export_data_command.sh
index 2180072c06..468a9d61ff 100755
--- a/.ci/scripts/test_export_data_command.sh
+++ b/.ci/scripts/test_export_data_command.sh
@@ -3,7 +3,7 @@
 # Test for the export-data admin command against sqlite and postgres
 
 set -xe
-cd "`dirname $0`/../.."
+cd "`dirname "$0"`/../.."
 
 echo "--- Install dependencies"
 
diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh
index 7bcbe63654..3926a17eab 100755
--- a/.ci/scripts/test_synapse_port_db.sh
+++ b/.ci/scripts/test_synapse_port_db.sh
@@ -7,7 +7,7 @@
 
 
 set -xe
-cd "`dirname $0`/../.."
+cd "`dirname "$0"`/../.."
 
 echo "--- Install dependencies"
 
diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index e006b48849..575c2e016d 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -98,7 +98,7 @@ esac
         --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.
-PYPKG=`basename $SNAKE`
+PYPKG=`basename "$SNAKE"`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
 
 
diff --git a/demo/clean.sh b/demo/clean.sh
index 99482e87d3..e9b440d90d 100755
--- a/demo/clean.sh
+++ b/demo/clean.sh
@@ -6,14 +6,14 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 PID_FILE="$DIR/servers.pid"
 
-if [ -f $PID_FILE ]; then
+if [ -f "$PID_FILE" ]; then
     echo "servers.pid exists!"
     exit 1
 fi
 
 for port in 8080 8081 8082; do
-    rm -rf ${DIR:?}/$port
-    rm -rf $DIR/media_store.$port
+    rm -rf "${DIR:?}/$port"
+    rm -rf "$DIR/media_store.$port"
 done
 
-rm -rf ${DIR:?}/etc
+rm -rf "${DIR:?}/etc"
diff --git a/demo/start.sh b/demo/start.sh
index 245e9d777b..03c3b375cf 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -12,7 +12,7 @@ PYTHONPATH=$(readlink -f "$(pwd)")
 export PYTHONPATH
 
 
-echo $PYTHONPATH
+echo "$PYTHONPATH"
 
 for port in 8080 8081 8082; do
     echo "Starting server on port $port... "
@@ -28,12 +28,12 @@ for port in 8080 8081 8082; do
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
-        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
+    if ! grep -F "Customisation made by demo/start.sh" -q "$DIR/etc/$port.config"; then
+        printf '\n\n# Customisation made by demo/start.sh\n' >> "$DIR/etc/$port.config"
 
-        echo "public_baseurl: http://localhost:$port/" >> $DIR/etc/$port.config
+        echo "public_baseurl: http://localhost:$port/" >> "$DIR/etc/$port.config"
 
-        echo 'enable_registration: true' >> $DIR/etc/$port.config
+        echo 'enable_registration: true' >> "$DIR/etc/$port.config"
 
         # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
         # accidentaly bork me with your fancy settings.
@@ -57,26 +57,26 @@ for port in 8080 8081 8082; do
 		        compress: false
 		PORTLISTENERS
 		)
-        echo "${listeners}" >> $DIR/etc/$port.config
+        echo "${listeners}" >> "$DIR/etc/$port.config"
 
         # Disable tls for the servers
-        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
-        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
-        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
-        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
+        printf '\n\n# Disable tls on the servers.' >> "$DIR/etc/$port.config"
+        echo '# DO NOT USE IN PRODUCTION' >> "$DIR/etc/$port.config"
+        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> "$DIR/etc/$port.config"
+        echo 'federation_verify_certificates: false' >> "$DIR/etc/$port.config"
 
         # Set tls paths
-        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
-        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
+        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> "$DIR/etc/$port.config"
+        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> "$DIR/etc/$port.config"
 
         # Generate tls keys
-        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost:$https_port.tls.key -out $DIR/etc/localhost:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
+        openssl req -x509 -newkey rsa:4096 -keyout "$DIR/etc/localhost:$https_port.tls.key" -out "$DIR/etc/localhost:$https_port.tls.crt" -days 365 -nodes -subj "/O=matrix"
 
         # Ignore keys from the trusted keys server
-        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
-        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
-        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
-        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
+        echo '# Ignore keys from the trusted keys server' >> "$DIR/etc/$port.config"
+        echo 'trusted_key_servers:' >> "$DIR/etc/$port.config"
+        echo '  - server_name: "matrix.org"' >> "$DIR/etc/$port.config"
+        echo '    accept_keys_insecurely: true' >> "$DIR/etc/$port.config"
 
         # Reduce the blacklist
         blacklist=$(cat <<-BLACK
@@ -91,12 +91,12 @@ for port in 8080 8081 8082; do
 		  - 'fc00::/7'
 		BLACK
 		)
-        echo "${blacklist}" >> $DIR/etc/$port.config
+        echo "${blacklist}" >> "$DIR/etc/$port.config"
     fi
 
     # Check script parameters
     if [ $# -eq 1 ]; then
-        if [ $1 = "--no-rate-limit" ]; then
+        if [ "$1" = "--no-rate-limit" ]; then
 
             # Disable any rate limiting
             ratelimiting=$(cat <<-RC
@@ -138,15 +138,15 @@ for port in 8080 8081 8082; do
 			    burst_count: 1000
 			RC
 			)
-            echo "${ratelimiting}" >> $DIR/etc/$port.config
+            echo "${ratelimiting}" >> "$DIR/etc/$port.config"
         fi
     fi
 
-    if ! grep -F "full_twisted_stacktraces" -q  $DIR/etc/$port.config; then
-        echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config
+    if ! grep -F "full_twisted_stacktraces" -q  "$DIR/etc/$port.config"; then
+        echo "full_twisted_stacktraces: true" >> "$DIR/etc/$port.config"
     fi
-    if ! grep -F "report_stats" -q  $DIR/etc/$port.config ; then
-        echo "report_stats: false" >> $DIR/etc/$port.config
+    if ! grep -F "report_stats" -q  "$DIR/etc/$port.config" ; then
+        echo "report_stats: false" >> "$DIR/etc/$port.config"
     fi
 
     python3 -m synapse.app.homeserver \
diff --git a/demo/stop.sh b/demo/stop.sh
index f9dddc5914..c97e4b8d00 100755
--- a/demo/stop.sh
+++ b/demo/stop.sh
@@ -8,7 +8,7 @@ for pid_file in $FILES; do
     pid=$(cat "$pid_file")
     if [[ $pid ]]; then
         echo "Killing $pid_file with $pid"
-        kill $pid
+        kill "$pid"
     fi
 done
 
diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index 1d0be42cfc..32ad07a0cc 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -5,7 +5,7 @@
 set -ex
 
 # Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< ${distro:?}`
+DIST=`cut -d ':' -f2 <<< "${distro:?}"`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index 56600de0ad..5fc68e3213 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -44,8 +44,8 @@ echo
 matched=0
 for f in `git diff --name-only FETCH_HEAD... -- changelog.d`; do
     # check that any modified newsfiles on this branch end with a full stop.
-    lastchar=`tr -d '\n' < $f | tail -c 1`
-    if [ $lastchar != '.' ] && [ $lastchar != '!' ]; then
+    lastchar=`tr -d '\n' < "$f" | tail -c 1`
+    if [ "$lastchar" != '.' ] && [ "$lastchar" != '!' ]; then
         echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2
         echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2
         exit 1
diff --git a/scripts-dev/check_line_terminators.sh b/scripts-dev/check_line_terminators.sh
index 068f157ea1..b48fb88b07 100755
--- a/scripts-dev/check_line_terminators.sh
+++ b/scripts-dev/check_line_terminators.sh
@@ -25,7 +25,7 @@
 # terminators are found, 0 otherwise.
 
 # cd to the root of the repository
-cd "`dirname $0`/.." || exit
+cd "`dirname "$0"`/.." || exit
 
 # Find and print files with non-unix line terminators
 if find . -path './.git/*' -prune -o -type f -print0 | xargs -0 grep -I -l $'\r$'; then
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 89af7a4fde..7d38b39e90 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -24,7 +24,7 @@
 set -e
 
 # Change to the repository root
-cd "$(dirname $0)/.."
+cd "$(dirname "$0")/.."
 
 # Check for a user-specified Complement checkout
 if [[ -z "$COMPLEMENT_DIR" ]]; then
@@ -61,8 +61,8 @@ cd "$COMPLEMENT_DIR"
 EXTRA_COMPLEMENT_ARGS=""
 if [[ -n "$1" ]]; then
   # A test name regex has been set, supply it to Complement
-  EXTRA_COMPLEMENT_ARGS+="-run $1 "
+  EXTRA_COMPLEMENT_ARGS=(-run "$1")
 fi
 
 # Run the tests!
-go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403,msc2716 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
+go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403,msc2716 -count=1 "${EXTRA_COMPLEMENT_ARGS[@]}" ./tests/...
diff --git a/scripts-dev/config-lint.sh b/scripts-dev/config-lint.sh
index 15c8c2f93a..7fb6ab68ef 100755
--- a/scripts-dev/config-lint.sh
+++ b/scripts-dev/config-lint.sh
@@ -3,7 +3,7 @@
 # Exits with 0 if there are no problems, or another code otherwise.
 
 # cd to the root of the repository
-cd "`dirname $0`/.." || exit
+cd "`dirname "$0"`/.." || exit
 
 # Restore backup of sample config upon script exit
 trap "mv docs/sample_config.yaml.bak docs/sample_config.yaml" EXIT
diff --git a/scripts-dev/docker_update_debian_changelog.sh b/scripts-dev/docker_update_debian_changelog.sh
index 14c155aee5..729f8fc467 100755
--- a/scripts-dev/docker_update_debian_changelog.sh
+++ b/scripts-dev/docker_update_debian_changelog.sh
@@ -60,5 +60,5 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y devscripts
 
 # Update the Debian changelog.
 ver=${1}
-dch -M -v "$(sed -Ee 's/(rc|a|b|c)/~\1/' <<<$ver)" "New synapse release $ver."
+dch -M -v "$(sed -Ee 's/(rc|a|b|c)/~\1/' <<<"$ver")" "New synapse release $ver."
 dch -M -r -D stable ""
diff --git a/scripts-dev/generate_sample_config b/scripts-dev/generate_sample_config
index 06e4c8c669..69d0b632df 100755
--- a/scripts-dev/generate_sample_config
+++ b/scripts-dev/generate_sample_config
@@ -4,7 +4,7 @@
 
 set -e
 
-cd "`dirname $0`/.."
+cd "`dirname "$0"`/.."
 
 SAMPLE_CONFIG="docs/sample_config.yaml"
 SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
-- 
cgit 1.5.1


From b5e910521b8f712b0ff0d61be33414efa4b3b6ba Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Fri, 22 Oct 2021 23:00:04 +0100
Subject: Fix Shellcheck SC2129: Consider using {..} >> file

Consider using { cmd1; cmd2; } >> file instead of individual redirects.

https://github.com/koalaman/shellcheck/wiki/SC2129

Signed-off-by: Dan Callahan <danc@element.io>
---
 demo/start.sh | 123 ++++++++++++++++++++++++++++++----------------------------
 1 file changed, 63 insertions(+), 60 deletions(-)

diff --git a/demo/start.sh b/demo/start.sh
index 03c3b375cf..8ffb14e30a 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -29,69 +29,72 @@ for port in 8080 8081 8082; do
         --report-stats no
 
     if ! grep -F "Customisation made by demo/start.sh" -q "$DIR/etc/$port.config"; then
-        printf '\n\n# Customisation made by demo/start.sh\n' >> "$DIR/etc/$port.config"
-
-        echo "public_baseurl: http://localhost:$port/" >> "$DIR/etc/$port.config"
-
-        echo 'enable_registration: true' >> "$DIR/etc/$port.config"
-
-        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
-        # accidentaly bork me with your fancy settings.
-		listeners=$(cat <<-PORTLISTENERS
-		# Configure server to listen on both $https_port and $port
-		# This overides some of the default settings above
-		listeners:
-		  - port: $https_port
-		    type: http
-		    tls: true
-		    resources:
-		      - names: [client, federation]
-
-		  - port: $port
-		    tls: false
-		    bind_addresses: ['::1', '127.0.0.1']
-		    type: http
-		    x_forwarded: true
-		    resources:
-		      - names: [client, federation]
-		        compress: false
-		PORTLISTENERS
-		)
-        echo "${listeners}" >> "$DIR/etc/$port.config"
-
-        # Disable tls for the servers
-        printf '\n\n# Disable tls on the servers.' >> "$DIR/etc/$port.config"
-        echo '# DO NOT USE IN PRODUCTION' >> "$DIR/etc/$port.config"
-        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> "$DIR/etc/$port.config"
-        echo 'federation_verify_certificates: false' >> "$DIR/etc/$port.config"
-
-        # Set tls paths
-        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> "$DIR/etc/$port.config"
-        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> "$DIR/etc/$port.config"
-
         # Generate tls keys
         openssl req -x509 -newkey rsa:4096 -keyout "$DIR/etc/localhost:$https_port.tls.key" -out "$DIR/etc/localhost:$https_port.tls.crt" -days 365 -nodes -subj "/O=matrix"
 
-        # Ignore keys from the trusted keys server
-        echo '# Ignore keys from the trusted keys server' >> "$DIR/etc/$port.config"
-        echo 'trusted_key_servers:' >> "$DIR/etc/$port.config"
-        echo '  - server_name: "matrix.org"' >> "$DIR/etc/$port.config"
-        echo '    accept_keys_insecurely: true' >> "$DIR/etc/$port.config"
-
-        # Reduce the blacklist
-        blacklist=$(cat <<-BLACK
-		# Set the blacklist so that it doesn't include 127.0.0.1, ::1
-		federation_ip_range_blacklist:
-		  - '10.0.0.0/8'
-		  - '172.16.0.0/12'
-		  - '192.168.0.0/16'
-		  - '100.64.0.0/10'
-		  - '169.254.0.0/16'
-		  - 'fe80::/64'
-		  - 'fc00::/7'
-		BLACK
-		)
-        echo "${blacklist}" >> "$DIR/etc/$port.config"
+        # Regenerate configuration
+        {
+            printf '\n\n# Customisation made by demo/start.sh\n'
+            echo "public_baseurl: http://localhost:$port/"
+            echo 'enable_registration: true'
+
+			# Warning, this heredoc depends on the interaction of tabs and spaces.
+			# Please don't accidentaly bork me with your fancy settings.
+			listeners=$(cat <<-PORTLISTENERS
+			# Configure server to listen on both $https_port and $port
+			# This overides some of the default settings above
+			listeners:
+			  - port: $https_port
+			    type: http
+			    tls: true
+			    resources:
+			      - names: [client, federation]
+
+			  - port: $port
+			    tls: false
+			    bind_addresses: ['::1', '127.0.0.1']
+			    type: http
+			    x_forwarded: true
+			    resources:
+			      - names: [client, federation]
+			        compress: false
+			PORTLISTENERS
+			)
+
+            echo "${listeners}"
+
+            # Disable tls for the servers
+            printf '\n\n# Disable tls on the servers.'
+            echo '# DO NOT USE IN PRODUCTION'
+            echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true'
+            echo 'federation_verify_certificates: false'
+
+            # Set tls paths
+            echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\""
+            echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\""
+
+            # Ignore keys from the trusted keys server
+            echo '# Ignore keys from the trusted keys server'
+            echo 'trusted_key_servers:'
+            echo '  - server_name: "matrix.org"'
+            echo '    accept_keys_insecurely: true'
+
+			# Reduce the blacklist
+			blacklist=$(cat <<-BLACK
+			# Set the blacklist so that it doesn't include 127.0.0.1, ::1
+			federation_ip_range_blacklist:
+			  - '10.0.0.0/8'
+			  - '172.16.0.0/12'
+			  - '192.168.0.0/16'
+			  - '100.64.0.0/10'
+			  - '169.254.0.0/16'
+			  - 'fe80::/64'
+			  - 'fc00::/7'
+			BLACK
+			)
+
+            echo "${blacklist}"
+        } >> "$DIR/etc/$port.config"
     fi
 
     # Check script parameters
-- 
cgit 1.5.1


From d7141e0b8b945047b37b8ca093ed0bfd63724a25 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Fri, 22 Oct 2021 23:07:23 +0100
Subject: Fix Shellcheck SC2006: Use $(...) notation

Use $(...) notation instead of legacy backticked `...`.

https://github.com/koalaman/shellcheck/wiki/SC2006

Signed-off-by: Dan Callahan <danc@element.io>
---
 .ci/scripts/test_export_data_command.sh | 2 +-
 .ci/scripts/test_synapse_port_db.sh     | 2 +-
 debian/build_virtualenv                 | 8 ++++----
 debian/test/provision.sh                | 2 +-
 docker/build_debian.sh                  | 4 ++--
 scripts-dev/check-newsfragment          | 4 ++--
 scripts-dev/check_line_terminators.sh   | 2 +-
 scripts-dev/config-lint.sh              | 2 +-
 scripts-dev/generate_sample_config      | 2 +-
 scripts-dev/next_github_number.sh       | 2 +-
 10 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/.ci/scripts/test_export_data_command.sh b/.ci/scripts/test_export_data_command.sh
index 468a9d61ff..ab96387a0a 100755
--- a/.ci/scripts/test_export_data_command.sh
+++ b/.ci/scripts/test_export_data_command.sh
@@ -3,7 +3,7 @@
 # Test for the export-data admin command against sqlite and postgres
 
 set -xe
-cd "`dirname "$0"`/../.."
+cd "$(dirname "$0")/../.."
 
 echo "--- Install dependencies"
 
diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh
index 3926a17eab..797904e64c 100755
--- a/.ci/scripts/test_synapse_port_db.sh
+++ b/.ci/scripts/test_synapse_port_db.sh
@@ -7,7 +7,7 @@
 
 
 set -xe
-cd "`dirname "$0"`/../.."
+cd "$(dirname "$0")/../.."
 
 echo "--- Install dependencies"
 
diff --git a/debian/build_virtualenv b/debian/build_virtualenv
index 575c2e016d..3097371d59 100755
--- a/debian/build_virtualenv
+++ b/debian/build_virtualenv
@@ -15,7 +15,7 @@ export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
 # python won't look in the right directory. At least this way, the error will
 # be a *bit* more obvious.
 #
-SNAKE=`readlink -e /usr/bin/python3`
+SNAKE=$(readlink -e /usr/bin/python3)
 
 # try to set the CFLAGS so any compiled C extensions are compiled with the most
 # generic as possible x64 instructions, so that compiling it on a new Intel chip
@@ -24,7 +24,7 @@ SNAKE=`readlink -e /usr/bin/python3`
 # TODO: add similar things for non-amd64, or figure out a more generic way to
 # do this.
 
-case `dpkg-architecture -q DEB_HOST_ARCH` in
+case $(dpkg-architecture -q DEB_HOST_ARCH) in
     amd64)
         export CFLAGS=-march=x86-64
         ;;
@@ -56,7 +56,7 @@ case "$DEB_BUILD_OPTIONS" in
     *)
         # Copy tests to a temporary directory so that we can put them on the
         # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-        tmpdir=`mktemp -d`
+        tmpdir=$(mktemp -d)
         trap 'rm -r $tmpdir' EXIT
 
         cp -r tests "$tmpdir"
@@ -98,7 +98,7 @@ esac
         --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.
-PYPKG=`basename "$SNAKE"`
+PYPKG=$(basename "$SNAKE")
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
 
 
diff --git a/debian/test/provision.sh b/debian/test/provision.sh
index 8567b146ad..55d7b8e03a 100644
--- a/debian/test/provision.sh
+++ b/debian/test/provision.sh
@@ -10,7 +10,7 @@ set -e
 apt-get update
 apt-get install -y lsb-release
 
-deb=`find /debs -name "matrix-synapse-py3_*+$(lsb_release -cs)*.deb" | sort | tail -n1`
+deb=$(find /debs -name "matrix-synapse-py3_*+$(lsb_release -cs)*.deb" | sort | tail -n1)
 
 debconf-set-selections <<EOF
 matrix-synapse matrix-synapse/report-stats boolean false
diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index 32ad07a0cc..9eae38af91 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -5,7 +5,7 @@
 set -ex
 
 # Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< "${distro:?}"`
+DIST=$(cut -d ':' -f2 <<< "${distro:?}")
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build
@@ -17,7 +17,7 @@ cd /synapse/build
 # Section to determine which "component" it should go into (see
 # https://manpages.debian.org/stretch/reprepro/reprepro.1.en.html#GUESSING)
 
-DEB_VERSION=`dpkg-parsechangelog -SVersion`
+DEB_VERSION=$(dpkg-parsechangelog -SVersion)
 case $DEB_VERSION in
     *~rc*|*~a*|*~b*|*~c*)
         sed -ie '/^Section:/c\Section: prerelease' debian/control
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index 5fc68e3213..af4de345df 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -42,9 +42,9 @@ echo "--------------------------"
 echo
 
 matched=0
-for f in `git diff --name-only FETCH_HEAD... -- changelog.d`; do
+for f in $(git diff --name-only FETCH_HEAD... -- changelog.d); do
     # check that any modified newsfiles on this branch end with a full stop.
-    lastchar=`tr -d '\n' < "$f" | tail -c 1`
+    lastchar=$(tr -d '\n' < "$f" | tail -c 1)
     if [ "$lastchar" != '.' ] && [ "$lastchar" != '!' ]; then
         echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2
         echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2
diff --git a/scripts-dev/check_line_terminators.sh b/scripts-dev/check_line_terminators.sh
index b48fb88b07..fffa24e01e 100755
--- a/scripts-dev/check_line_terminators.sh
+++ b/scripts-dev/check_line_terminators.sh
@@ -25,7 +25,7 @@
 # terminators are found, 0 otherwise.
 
 # cd to the root of the repository
-cd "`dirname "$0"`/.." || exit
+cd "$(dirname "$0")/.." || exit
 
 # Find and print files with non-unix line terminators
 if find . -path './.git/*' -prune -o -type f -print0 | xargs -0 grep -I -l $'\r$'; then
diff --git a/scripts-dev/config-lint.sh b/scripts-dev/config-lint.sh
index 7fb6ab68ef..6ce030b819 100755
--- a/scripts-dev/config-lint.sh
+++ b/scripts-dev/config-lint.sh
@@ -3,7 +3,7 @@
 # Exits with 0 if there are no problems, or another code otherwise.
 
 # cd to the root of the repository
-cd "`dirname "$0"`/.." || exit
+cd "$(dirname "$0")/.." || exit
 
 # Restore backup of sample config upon script exit
 trap "mv docs/sample_config.yaml.bak docs/sample_config.yaml" EXIT
diff --git a/scripts-dev/generate_sample_config b/scripts-dev/generate_sample_config
index 69d0b632df..4cd1d1d5b8 100755
--- a/scripts-dev/generate_sample_config
+++ b/scripts-dev/generate_sample_config
@@ -4,7 +4,7 @@
 
 set -e
 
-cd "`dirname "$0"`/.."
+cd "$(dirname "$0")/.."
 
 SAMPLE_CONFIG="docs/sample_config.yaml"
 SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
diff --git a/scripts-dev/next_github_number.sh b/scripts-dev/next_github_number.sh
index 00e9b14569..5ecd515127 100755
--- a/scripts-dev/next_github_number.sh
+++ b/scripts-dev/next_github_number.sh
@@ -4,6 +4,6 @@ set -e
 
 # Fetch the current GitHub issue number, add one to it -- presto! The likely
 # next PR number.
-CURRENT_NUMBER=`curl -s "https://api.github.com/repos/matrix-org/synapse/issues?state=all&per_page=1" | jq -r ".[0].number"`
+CURRENT_NUMBER=$(curl -s "https://api.github.com/repos/matrix-org/synapse/issues?state=all&per_page=1" | jq -r ".[0].number")
 CURRENT_NUMBER=$((CURRENT_NUMBER+1))
 echo $CURRENT_NUMBER
-- 
cgit 1.5.1


From 1afc6ecae1f4fdb63e47596fbe88bf93b461c8f9 Mon Sep 17 00:00:00 2001
From: Dan Callahan <danc@element.io>
Date: Fri, 22 Oct 2021 23:21:40 +0100
Subject: Changelog

Signed-off-by: Dan Callahan <danc@element.io>
---
 changelog.d/11166.misc | 1 +
 debian/changelog       | 6 ++++++
 2 files changed, 7 insertions(+)
 create mode 100644 changelog.d/11166.misc

diff --git a/changelog.d/11166.misc b/changelog.d/11166.misc
new file mode 100644
index 0000000000..79342e43d9
--- /dev/null
+++ b/changelog.d/11166.misc
@@ -0,0 +1 @@
+Update scripts to pass Shellcheck lints.
diff --git a/debian/changelog b/debian/changelog
index 1ee81f2a34..f68c993094 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.45.1+nmu1) UNRELEASED; urgency=medium
+
+  * Update scripts to pass Shellcheck lints.
+
+ -- root <root@cae79a6e79d7>  Fri, 22 Oct 2021 22:20:31 +0000
+
 matrix-synapse-py3 (1.45.1) stable; urgency=medium
 
   * New synapse release 1.45.1.
-- 
cgit 1.5.1


From 85a09f8b8ba7c8023c0d28a526d32111fc704197 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Mon, 25 Oct 2021 13:01:04 +0100
Subject: Fix module API's `get_user_ip_and_agents` function when run on
 workers (#11112)

---
 changelog.d/11112.bugfix                     |   1 +
 synapse/module_api/__init__.py               |   6 +-
 synapse/storage/databases/main/client_ips.py | 124 ++++++++++++++++++---------
 3 files changed, 91 insertions(+), 40 deletions(-)
 create mode 100644 changelog.d/11112.bugfix

diff --git a/changelog.d/11112.bugfix b/changelog.d/11112.bugfix
new file mode 100644
index 0000000000..c8e22da8cf
--- /dev/null
+++ b/changelog.d/11112.bugfix
@@ -0,0 +1 @@
+Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time.
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index ab7ef8f950..d37252b6b3 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -46,6 +46,7 @@ from synapse.http.site import SynapseRequest
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.rest.client.login import LoginResponse
+from synapse.storage import DataStore
 from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.storage.databases.main.roommember import ProfileInfo
 from synapse.storage.state import StateFilter
@@ -61,6 +62,7 @@ from synapse.util import Clock
 from synapse.util.caches.descriptors import cached
 
 if TYPE_CHECKING:
+    from synapse.app.generic_worker import GenericWorkerSlavedStore
     from synapse.server import HomeServer
 
 """
@@ -111,7 +113,9 @@ class ModuleApi:
     def __init__(self, hs: "HomeServer", auth_handler):
         self._hs = hs
 
-        self._store = hs.get_datastore()
+        # TODO: Fix this type hint once the types for the data stores have been ironed
+        #       out.
+        self._store: Union[DataStore, "GenericWorkerSlavedStore"] = hs.get_datastore()
         self._auth = hs.get_auth()
         self._auth_handler = auth_handler
         self._server_name = hs.hostname
diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py
index b81d9218ce..1dc7f0ebe3 100644
--- a/synapse/storage/databases/main/client_ips.py
+++ b/synapse/storage/databases/main/client_ips.py
@@ -478,6 +478,58 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore):
 
         return {(d["user_id"], d["device_id"]): d for d in res}
 
+    async def get_user_ip_and_agents(
+        self, user: UserID, since_ts: int = 0
+    ) -> List[LastConnectionInfo]:
+        """Fetch the IPs and user agents for a user since the given timestamp.
+
+        The result might be slightly out of date as client IPs are inserted in batches.
+
+        Args:
+            user: The user for which to fetch IP addresses and user agents.
+            since_ts: The timestamp after which to fetch IP addresses and user agents,
+                in milliseconds.
+
+        Returns:
+            A list of dictionaries, each containing:
+             * `access_token`: The access token used.
+             * `ip`: The IP address used.
+             * `user_agent`: The last user agent seen for this access token and IP
+               address combination.
+             * `last_seen`: The timestamp at which this access token and IP address
+               combination was last seen, in milliseconds.
+
+            Only the latest user agent for each access token and IP address combination
+            is available.
+        """
+        user_id = user.to_string()
+
+        def get_recent(txn: LoggingTransaction) -> List[Tuple[str, str, str, int]]:
+            txn.execute(
+                """
+                SELECT access_token, ip, user_agent, last_seen FROM user_ips
+                WHERE last_seen >= ? AND user_id = ?
+                ORDER BY last_seen
+                DESC
+                """,
+                (since_ts, user_id),
+            )
+            return cast(List[Tuple[str, str, str, int]], txn.fetchall())
+
+        rows = await self.db_pool.runInteraction(
+            desc="get_user_ip_and_agents", func=get_recent
+        )
+
+        return [
+            {
+                "access_token": access_token,
+                "ip": ip,
+                "user_agent": user_agent,
+                "last_seen": last_seen,
+            }
+            for access_token, ip, user_agent, last_seen in rows
+        ]
+
 
 class ClientIpStore(ClientIpWorkerStore, MonthlyActiveUsersStore):
     def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"):
@@ -622,49 +674,43 @@ class ClientIpStore(ClientIpWorkerStore, MonthlyActiveUsersStore):
     async def get_user_ip_and_agents(
         self, user: UserID, since_ts: int = 0
     ) -> List[LastConnectionInfo]:
+        """Fetch the IPs and user agents for a user since the given timestamp.
+
+        Args:
+            user: The user for which to fetch IP addresses and user agents.
+            since_ts: The timestamp after which to fetch IP addresses and user agents,
+                in milliseconds.
+
+        Returns:
+            A list of dictionaries, each containing:
+             * `access_token`: The access token used.
+             * `ip`: The IP address used.
+             * `user_agent`: The last user agent seen for this access token and IP
+               address combination.
+             * `last_seen`: The timestamp at which this access token and IP address
+               combination was last seen, in milliseconds.
+
+            Only the latest user agent for each access token and IP address combination
+            is available.
         """
-        Fetch IP/User Agent connection since a given timestamp.
-        """
-        user_id = user.to_string()
-        results: Dict[Tuple[str, str], Tuple[str, int]] = {}
+        results: Dict[Tuple[str, str], LastConnectionInfo] = {
+            (connection["access_token"], connection["ip"]): connection
+            for connection in await super().get_user_ip_and_agents(user, since_ts)
+        }
 
+        # Overlay data that is pending insertion on top of the results from the
+        # database.
+        user_id = user.to_string()
         for key in self._batch_row_update:
-            (
-                uid,
-                access_token,
-                ip,
-            ) = key
+            uid, access_token, ip = key
             if uid == user_id:
                 user_agent, _, last_seen = self._batch_row_update[key]
                 if last_seen >= since_ts:
-                    results[(access_token, ip)] = (user_agent, last_seen)
-
-        def get_recent(txn: LoggingTransaction) -> List[Tuple[str, str, str, int]]:
-            txn.execute(
-                """
-                SELECT access_token, ip, user_agent, last_seen FROM user_ips
-                WHERE last_seen >= ? AND user_id = ?
-                ORDER BY last_seen
-                DESC
-                """,
-                (since_ts, user_id),
-            )
-            return cast(List[Tuple[str, str, str, int]], txn.fetchall())
-
-        rows = await self.db_pool.runInteraction(
-            desc="get_user_ip_and_agents", func=get_recent
-        )
+                    results[(access_token, ip)] = {
+                        "access_token": access_token,
+                        "ip": ip,
+                        "user_agent": user_agent,
+                        "last_seen": last_seen,
+                    }
 
-        results.update(
-            ((access_token, ip), (user_agent, last_seen))
-            for access_token, ip, user_agent, last_seen in rows
-        )
-        return [
-            {
-                "access_token": access_token,
-                "ip": ip,
-                "user_agent": user_agent,
-                "last_seen": last_seen,
-            }
-            for (access_token, ip), (user_agent, last_seen) in results.items()
-        ]
+        return list(results.values())
-- 
cgit 1.5.1


From da957a60e8958b08a52bd1404a89cf9bbcd033e0 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 25 Oct 2021 16:21:09 +0200
Subject: Ensure that we correctly auth events returned by `send_join` (#11012)

This is the final piece of the jigsaw for #9595. As with other changes before this one (eg #10771), we need to make sure that we auth the auth events in the right order, and actually check that their predecessors haven't been rejected.

To do this I've reused the existing code we use when persisting outliers elsewhere.

I've removed the code for attempting to fetch missing auth_events - the events should have been present in the send_join response, so the likely reason they are missing is that we couldn't verify them, so requesting them again is unlikely to help. Instead, we simply drop any state which relies on those auth events, as we do at a backwards-extremity. See also matrix-org/complement#216 for a test for this.
---
 changelog.d/11012.bugfix             |   1 +
 synapse/handlers/federation_event.py | 146 ++++++++++++++---------------------
 2 files changed, 61 insertions(+), 86 deletions(-)
 create mode 100644 changelog.d/11012.bugfix

diff --git a/changelog.d/11012.bugfix b/changelog.d/11012.bugfix
new file mode 100644
index 0000000000..13b8e5983b
--- /dev/null
+++ b/changelog.d/11012.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state.
diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index 3431a80ab4..9584d5bd46 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -361,6 +361,7 @@ class FederationEventHandler:
         # need to.
         await self._event_creation_handler.cache_joined_hosts_for_event(event, context)
 
+        await self._check_for_soft_fail(event, None, origin=origin)
         await self._run_push_actions_and_persist_event(event, context)
         return event, context
 
@@ -402,29 +403,28 @@ class FederationEventHandler:
         """Persists the events returned by a send_join
 
         Checks the auth chain is valid (and passes auth checks) for the
-        state and event. Then persists the auth chain and state atomically.
-        Persists the event separately. Notifies about the persisted events
-        where appropriate.
-
-        Will attempt to fetch missing auth events.
+        state and event. Then persists all of the events.
+        Notifies about the persisted events where appropriate.
 
         Args:
             origin: Where the events came from
-            room_id,
+            room_id:
             auth_events
             state
             event
             room_version: The room version we expect this room to have, and
                 will raise if it doesn't match the version in the create event.
+
+        Returns:
+            The stream ID after which all events have been persisted.
+
+        Raises:
+            SynapseError if the response is in some way invalid.
         """
-        events_to_context = {}
         for e in itertools.chain(auth_events, state):
             e.internal_metadata.outlier = True
-            events_to_context[e.event_id] = EventContext.for_outlier()
 
-        event_map = {
-            e.event_id: e for e in itertools.chain(auth_events, state, [event])
-        }
+        event_map = {e.event_id: e for e in itertools.chain(auth_events, state)}
 
         create_event = None
         for e in auth_events:
@@ -444,64 +444,36 @@ class FederationEventHandler:
         if room_version.identifier != room_version_id:
             raise SynapseError(400, "Room version mismatch")
 
-        missing_auth_events = set()
-        for e in itertools.chain(auth_events, state, [event]):
-            for e_id in e.auth_event_ids():
-                if e_id not in event_map:
-                    missing_auth_events.add(e_id)
-
-        for e_id in missing_auth_events:
-            m_ev = await self._federation_client.get_pdu(
-                [origin],
-                e_id,
-                room_version=room_version,
-                outlier=True,
-                timeout=10000,
-            )
-            if m_ev and m_ev.event_id == e_id:
-                event_map[e_id] = m_ev
-            else:
-                logger.info("Failed to find auth event %r", e_id)
-
-        for e in itertools.chain(auth_events, state, [event]):
-            auth_for_e = [
-                event_map[e_id] for e_id in e.auth_event_ids() if e_id in event_map
-            ]
-            if create_event:
-                auth_for_e.append(create_event)
-
-            try:
-                validate_event_for_room_version(room_version, e)
-                check_auth_rules_for_event(room_version, e, auth_for_e)
-            except SynapseError as err:
-                # we may get SynapseErrors here as well as AuthErrors. For
-                # instance, there are a couple of (ancient) events in some
-                # rooms whose senders do not have the correct sigil; these
-                # cause SynapseErrors in auth.check. We don't want to give up
-                # the attempt to federate altogether in such cases.
-
-                logger.warning("Rejecting %s because %s", e.event_id, err.msg)
-
-                if e == event:
-                    raise
-                events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR
-
-        if auth_events or state:
-            await self.persist_events_and_notify(
-                room_id,
-                [
-                    (e, events_to_context[e.event_id])
-                    for e in itertools.chain(auth_events, state)
-                ],
+        # filter out any events we have already seen
+        seen_remotes = await self._store.have_seen_events(room_id, event_map.keys())
+        for s in seen_remotes:
+            event_map.pop(s, None)
+
+        # persist the auth chain and state events.
+        #
+        # any invalid events here will be marked as rejected, and we'll carry on.
+        #
+        # any events whose auth events are missing (ie, not in the send_join response,
+        # and not already in our db) will just be ignored. This is correct behaviour,
+        # because the reason that auth_events are missing might be due to us being
+        # unable to validate their signatures. The fact that we can't validate their
+        # signatures right now doesn't mean that we will *never* be able to, so it
+        # is premature to reject them.
+        #
+        await self._auth_and_persist_outliers(room_id, event_map.values())
+
+        # and now persist the join event itself.
+        logger.info("Peristing join-via-remote %s", event)
+        with nested_logging_context(suffix=event.event_id):
+            context = await self._state_handler.compute_event_context(
+                event, old_state=state
             )
 
-        new_event_context = await self._state_handler.compute_event_context(
-            event, old_state=state
-        )
+            context = await self._check_event_auth(origin, event, context)
+            if context.rejected:
+                raise SynapseError(400, "Join event was rejected")
 
-        return await self.persist_events_and_notify(
-            room_id, [(event, new_event_context)]
-        )
+            return await self.persist_events_and_notify(room_id, [(event, context)])
 
     @log_function
     async def backfill(
@@ -974,9 +946,15 @@ class FederationEventHandler:
     ) -> None:
         """Called when we have a new non-outlier event.
 
-        This is called when we have a new event to add to the room DAG - either directly
-        via a /send request, retrieved via get_missing_events after a /send request, or
-        backfilled after a client request.
+        This is called when we have a new event to add to the room DAG. This can be
+        due to:
+           * events received directly via a /send request
+           * events retrieved via get_missing_events after a /send request
+           * events backfilled after a client request.
+
+        It's not currently used for events received from incoming send_{join,knock,leave}
+        requests (which go via on_send_membership_event), nor for joins created by a
+        remote join dance (which go via process_remote_join).
 
         We need to do auth checks and put it through the StateHandler.
 
@@ -1012,11 +990,19 @@ class FederationEventHandler:
             logger.exception("Unexpected AuthError from _check_event_auth")
             raise FederationError("ERROR", e.code, e.msg, affected=event.event_id)
 
+        if not backfilled and not context.rejected:
+            # For new (non-backfilled and non-outlier) events we check if the event
+            # passes auth based on the current state. If it doesn't then we
+            # "soft-fail" the event.
+            await self._check_for_soft_fail(event, state, origin=origin)
+
         await self._run_push_actions_and_persist_event(event, context, backfilled)
 
-        if backfilled:
+        if backfilled or context.rejected:
             return
 
+        await self._maybe_kick_guest_users(event)
+
         # For encrypted messages we check that we know about the sending device,
         # if we don't then we mark the device cache for that user as stale.
         if event.type == EventTypes.Encrypted:
@@ -1317,14 +1303,14 @@ class FederationEventHandler:
                 for auth_event_id in event.auth_event_ids():
                     ae = persisted_events.get(auth_event_id)
                     if not ae:
+                        # the fact we can't find the auth event doesn't mean it doesn't
+                        # exist, which means it is premature to reject `event`. Instead we
+                        # just ignore it for now.
                         logger.warning(
-                            "Event %s relies on auth_event %s, which could not be found.",
+                            "Dropping event %s, which relies on auth_event %s, which could not be found",
                             event,
                             auth_event_id,
                         )
-                        # the fact we can't find the auth event doesn't mean it doesn't
-                        # exist, which means it is premature to reject `event`. Instead we
-                        # just ignore it for now.
                         return None
                     auth.append(ae)
 
@@ -1447,10 +1433,6 @@ class FederationEventHandler:
         except AuthError as e:
             logger.warning("Failed auth resolution for %r because %s", event, e)
             context.rejected = RejectedReason.AUTH_ERROR
-            return context
-
-        await self._check_for_soft_fail(event, state, backfilled, origin=origin)
-        await self._maybe_kick_guest_users(event)
 
         return context
 
@@ -1470,7 +1452,6 @@ class FederationEventHandler:
         self,
         event: EventBase,
         state: Optional[Iterable[EventBase]],
-        backfilled: bool,
         origin: str,
     ) -> None:
         """Checks if we should soft fail the event; if so, marks the event as
@@ -1479,15 +1460,8 @@ class FederationEventHandler:
         Args:
             event
             state: The state at the event if we don't have all the event's prev events
-            backfilled: Whether the event is from backfill
             origin: The host the event originates from.
         """
-        # For new (non-backfilled and non-outlier) events we check if the event
-        # passes auth based on the current state. If it doesn't then we
-        # "soft-fail" the event.
-        if backfilled or event.internal_metadata.is_outlier():
-            return
-
         extrem_ids_list = await self._store.get_latest_event_ids_in_room(event.room_id)
         extrem_ids = set(extrem_ids_list)
         prev_event_ids = set(event.prev_event_ids())
-- 
cgit 1.5.1


From 4387b791e01eb1a207fe44fecbc901eead8eb4db Mon Sep 17 00:00:00 2001
From: AndrewFerr <AndrewFerr@users.noreply.github.com>
Date: Mon, 25 Oct 2021 10:24:49 -0400
Subject: Don't set new room alias before potential 403 (#10930)

Fixes: #10929

Signed-off-by: Andrew Ferrazzutti <fair@miscworks.net>
---
 changelog.d/10930.bugfix         |   1 +
 synapse/handlers/directory.py    |   4 +-
 synapse/handlers/room.py         |  18 +++----
 tests/handlers/test_directory.py | 102 ++++++++++++++++++++++++++++++++++++++-
 4 files changed, 113 insertions(+), 12 deletions(-)
 create mode 100644 changelog.d/10930.bugfix

diff --git a/changelog.d/10930.bugfix b/changelog.d/10930.bugfix
new file mode 100644
index 0000000000..756bfe9107
--- /dev/null
+++ b/changelog.d/10930.bugfix
@@ -0,0 +1 @@
+Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr.
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index 14ed7d9879..8567cb0e00 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -145,7 +145,7 @@ class DirectoryHandler:
             if not self.config.roomdirectory.is_alias_creation_allowed(
                 user_id, room_id, room_alias_str
             ):
-                # Lets just return a generic message, as there may be all sorts of
+                # Let's just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?
                 raise SynapseError(403, "Not allowed to create alias")
@@ -461,7 +461,7 @@ class DirectoryHandler:
             if not self.config.roomdirectory.is_publishing_room_allowed(
                 user_id, room_id, room_aliases
             ):
-                # Lets just return a generic message, as there may be all sorts of
+                # Let's just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?
                 raise SynapseError(403, "Not allowed to publish room")
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 6f39e9446f..cf01d58ea1 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -773,6 +773,15 @@ class RoomCreationHandler:
         if not allowed_by_third_party_rules:
             raise SynapseError(403, "Room visibility value not allowed.")
 
+        if is_public:
+            if not self.config.roomdirectory.is_publishing_room_allowed(
+                user_id, room_id, room_alias
+            ):
+                # Let's just return a generic message, as there may be all sorts of
+                # reasons why we said no. TODO: Allow configurable error messages
+                # per alias creation rule?
+                raise SynapseError(403, "Not allowed to publish room")
+
         directory_handler = self.hs.get_directory_handler()
         if room_alias:
             await directory_handler.create_association(
@@ -783,15 +792,6 @@ class RoomCreationHandler:
                 check_membership=False,
             )
 
-        if is_public:
-            if not self.config.roomdirectory.is_publishing_room_allowed(
-                user_id, room_id, room_alias
-            ):
-                # Lets just return a generic message, as there may be all sorts of
-                # reasons why we said no. TODO: Allow configurable error messages
-                # per alias creation rule?
-                raise SynapseError(403, "Not allowed to publish room")
-
         preset_config = config.get(
             "preset",
             RoomCreationPreset.PRIVATE_CHAT
diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index 6a2e76ca4a..be008227df 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -15,8 +15,8 @@
 
 from unittest.mock import Mock
 
-import synapse
 import synapse.api.errors
+import synapse.rest.admin
 from synapse.api.constants import EventTypes
 from synapse.config.room_directory import RoomDirectoryConfig
 from synapse.rest.client import directory, login, room
@@ -432,6 +432,106 @@ class TestCreateAliasACL(unittest.HomeserverTestCase):
         self.assertEquals(200, channel.code, channel.result)
 
 
+class TestCreatePublishedRoomACL(unittest.HomeserverTestCase):
+    data = {"room_alias_name": "unofficial_test"}
+
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        login.register_servlets,
+        directory.register_servlets,
+        room.register_servlets,
+    ]
+    hijack_auth = False
+
+    def prepare(self, reactor, clock, hs):
+        self.allowed_user_id = self.register_user("allowed", "pass")
+        self.allowed_access_token = self.login("allowed", "pass")
+
+        self.denied_user_id = self.register_user("denied", "pass")
+        self.denied_access_token = self.login("denied", "pass")
+
+        # This time we add custom room list publication rules
+        config = {}
+        config["alias_creation_rules"] = []
+        config["room_list_publication_rules"] = [
+            {"user_id": "*", "alias": "*", "action": "deny"},
+            {"user_id": self.allowed_user_id, "alias": "*", "action": "allow"},
+        ]
+
+        rd_config = RoomDirectoryConfig()
+        rd_config.read_config(config)
+
+        self.hs.config.roomdirectory.is_publishing_room_allowed = (
+            rd_config.is_publishing_room_allowed
+        )
+
+        return hs
+
+    def test_denied_without_publication_permission(self):
+        """
+        Try to create a room, register an alias for it, and publish it,
+        as a user without permission to publish rooms.
+        (This is used as both a standalone test & as a helper function.)
+        """
+        self.helper.create_room_as(
+            self.denied_user_id,
+            tok=self.denied_access_token,
+            extra_content=self.data,
+            is_public=True,
+            expect_code=403,
+        )
+
+    def test_allowed_when_creating_private_room(self):
+        """
+        Try to create a room, register an alias for it, and NOT publish it,
+        as a user without permission to publish rooms.
+        (This is used as both a standalone test & as a helper function.)
+        """
+        self.helper.create_room_as(
+            self.denied_user_id,
+            tok=self.denied_access_token,
+            extra_content=self.data,
+            is_public=False,
+            expect_code=200,
+        )
+
+    def test_allowed_with_publication_permission(self):
+        """
+        Try to create a room, register an alias for it, and publish it,
+        as a user WITH permission to publish rooms.
+        (This is used as both a standalone test & as a helper function.)
+        """
+        self.helper.create_room_as(
+            self.allowed_user_id,
+            tok=self.allowed_access_token,
+            extra_content=self.data,
+            is_public=False,
+            expect_code=200,
+        )
+
+    def test_can_create_as_private_room_after_rejection(self):
+        """
+        After failing to publish a room with an alias as a user without publish permission,
+        retry as the same user, but without publishing the room.
+
+        This should pass, but used to fail because the alias was registered by the first
+        request, even though the room creation was denied.
+        """
+        self.test_denied_without_publication_permission()
+        self.test_allowed_when_creating_private_room()
+
+    def test_can_create_with_permission_after_rejection(self):
+        """
+        After failing to publish a room with an alias as a user without publish permission,
+        retry as someone with permission, using the same alias.
+
+        This also used to fail because of the alias having been registered by the first
+        request, leaving it unavailable for any other user's new rooms.
+        """
+        self.test_denied_without_publication_permission()
+        self.test_allowed_with_publication_permission()
+
+
 class TestRoomListSearchDisabled(unittest.HomeserverTestCase):
     user_id = "@test:test"
 
-- 
cgit 1.5.1


From c1510c97b56060b7ab470b11264ed10dad445e14 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Mon, 25 Oct 2021 18:45:19 +0200
Subject: Fix cyclic import in the module API (#11180)

Introduced in #10548

See https://github.com/matrix-org/synapse-email-account-validity/runs/3979337154?check_suite_focus=true for an example of a module's CI choking over this issue.
---
 changelog.d/11180.feature | 1 +
 synapse/handlers/auth.py  | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11180.feature

diff --git a/changelog.d/11180.feature b/changelog.d/11180.feature
new file mode 100644
index 0000000000..82c40bf1b2
--- /dev/null
+++ b/changelog.d/11180.feature
@@ -0,0 +1 @@
+Port the Password Auth Providers module interface to the new generic interface.
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index ebe75a9e9b..d508d7d32a 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -62,7 +62,6 @@ from synapse.http.server import finish_request, respond_with_html
 from synapse.http.site import SynapseRequest
 from synapse.logging.context import defer_to_thread
 from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.module_api import ModuleApi
 from synapse.storage.roommember import ProfileInfo
 from synapse.types import JsonDict, Requester, UserID
 from synapse.util import stringutils as stringutils
@@ -73,6 +72,7 @@ from synapse.util.stringutils import base62_encode
 from synapse.util.threepids import canonicalise_email
 
 if TYPE_CHECKING:
+    from synapse.module_api import ModuleApi
     from synapse.rest.client.login import LoginResponse
     from synapse.server import HomeServer
 
@@ -1818,7 +1818,9 @@ def load_legacy_password_auth_providers(hs: "HomeServer") -> None:
 
 
 def load_single_legacy_password_auth_provider(
-    module: Type, config: JsonDict, api: ModuleApi
+    module: Type,
+    config: JsonDict,
+    api: "ModuleApi",
 ) -> None:
     try:
         provider = module(config=config, account_handler=api)
-- 
cgit 1.5.1


From 63cbdd8af081839f245915a18ed57f1a44f1a5f4 Mon Sep 17 00:00:00 2001
From: Jason Robinson <jasonr@element.io>
Date: Tue, 26 Oct 2021 12:01:06 +0300
Subject: Enable changing user type via users admin API (#11174)

Users admin API can now also modify user
type in addition to allowing it to be
set on user creation.

Signed-off-by: Jason Robinson <jasonr@matrix.org>
Co-authored-by: Brendan Abolivier <babolivier@matrix.org>
---
 changelog.d/11174.feature                      |  1 +
 docs/admin_api/user_admin_api.md               |  9 ++++-
 synapse/rest/admin/users.py                    |  3 ++
 synapse/storage/databases/main/registration.py | 18 +++++++++
 tests/rest/admin/test_user.py                  | 51 ++++++++++++++++++++++++++
 5 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11174.feature

diff --git a/changelog.d/11174.feature b/changelog.d/11174.feature
new file mode 100644
index 0000000000..8eecd92681
--- /dev/null
+++ b/changelog.d/11174.feature
@@ -0,0 +1 @@
+Users admin API can now also modify user type in addition to allowing it to be set on user creation.
diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
index 534f8400ba..f03539c9f0 100644
--- a/docs/admin_api/user_admin_api.md
+++ b/docs/admin_api/user_admin_api.md
@@ -50,7 +50,8 @@ It returns a JSON body like the following:
             "auth_provider": "<provider2>",
             "external_id": "<user_id_provider_2>"
         }
-    ]
+    ],
+    "user_type": null
 }
 ```
 
@@ -97,7 +98,8 @@ with a body of:
     ],
     "avatar_url": "<avatar_url>",
     "admin": false,
-    "deactivated": false
+    "deactivated": false,
+    "user_type": null
 }
 ```
 
@@ -135,6 +137,9 @@ Body parameters:
   unchanged on existing accounts and set to `false` for new accounts.
   A user cannot be erased by deactivating with this API. For details on
   deactivating users see [Deactivate Account](#deactivate-account).
+- `user_type` - string or null, optional. If provided, the user type will be
+  adjusted. If `null` given, the user type will be cleared. Other 
+  allowed options are: `bot` and `support`.
 
 If the user already exists then optional parameters default to the current value.
 
diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py
index c0bebc3cf0..d14fafbbc9 100644
--- a/synapse/rest/admin/users.py
+++ b/synapse/rest/admin/users.py
@@ -326,6 +326,9 @@ class UserRestServletV2(RestServlet):
                         target_user.to_string()
                     )
 
+            if "user_type" in body:
+                await self.store.set_user_type(target_user, user_type)
+
             user = await self.admin_handler.get_user(target_user)
             assert user is not None
 
diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py
index 37d47aa823..6c7d6ba508 100644
--- a/synapse/storage/databases/main/registration.py
+++ b/synapse/storage/databases/main/registration.py
@@ -499,6 +499,24 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore):
 
         await self.db_pool.runInteraction("set_shadow_banned", set_shadow_banned_txn)
 
+    async def set_user_type(self, user: UserID, user_type: Optional[UserTypes]) -> None:
+        """Sets the user type.
+
+        Args:
+            user: user ID of the user.
+            user_type: type of the user or None for a user without a type.
+        """
+
+        def set_user_type_txn(txn):
+            self.db_pool.simple_update_one_txn(
+                txn, "users", {"name": user.to_string()}, {"user_type": user_type}
+            )
+            self._invalidate_cache_and_stream(
+                txn, self.get_user_by_id, (user.to_string(),)
+            )
+
+        await self.db_pool.runInteraction("set_user_type", set_user_type_txn)
+
     def _query_for_auth(self, txn, token: str) -> Optional[TokenLookupResult]:
         sql = """
             SELECT users.name as user_id,
diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py
index 839442ddba..25e8d6cf27 100644
--- a/tests/rest/admin/test_user.py
+++ b/tests/rest/admin/test_user.py
@@ -2270,6 +2270,57 @@ class UserRestTestCase(unittest.HomeserverTestCase):
         self.assertEqual("@user:test", channel.json_body["name"])
         self.assertTrue(channel.json_body["admin"])
 
+    def test_set_user_type(self):
+        """
+        Test changing user type.
+        """
+
+        # Set to support type
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"user_type": UserTypes.SUPPORT},
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(UserTypes.SUPPORT, channel.json_body["user_type"])
+
+        # Get user
+        channel = self.make_request(
+            "GET",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertEqual(UserTypes.SUPPORT, channel.json_body["user_type"])
+
+        # Change back to a regular user
+        channel = self.make_request(
+            "PUT",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+            content={"user_type": None},
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertIsNone(channel.json_body["user_type"])
+
+        # Get user
+        channel = self.make_request(
+            "GET",
+            self.url_other_user,
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(200, channel.code, msg=channel.json_body)
+        self.assertEqual("@user:test", channel.json_body["name"])
+        self.assertIsNone(channel.json_body["user_type"])
+
     def test_accidental_deactivation_prevention(self):
         """
         Ensure an account can't accidentally be deactivated by using a str value
-- 
cgit 1.5.1


From 8c8e36af0d6c3855de7bd786be14b85f5dae4ea7 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Tue, 26 Oct 2021 11:09:10 +0200
Subject: Document the version each module API method was added to Synapse
 (#11183)

---
 changelog.d/11183.doc          |  1 +
 synapse/module_api/__init__.py | 99 +++++++++++++++++++++++++++++++++++++-----
 2 files changed, 89 insertions(+), 11 deletions(-)
 create mode 100644 changelog.d/11183.doc

diff --git a/changelog.d/11183.doc b/changelog.d/11183.doc
new file mode 100644
index 0000000000..a171a107af
--- /dev/null
+++ b/changelog.d/11183.doc
@@ -0,0 +1 @@
+Document the version of Synapse that introduced each module API method.
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index d37252b6b3..d707a9325d 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -154,27 +154,42 @@ class ModuleApi:
 
     @property
     def register_spam_checker_callbacks(self):
-        """Registers callbacks for spam checking capabilities."""
+        """Registers callbacks for spam checking capabilities.
+
+        Added in Synapse v1.37.0.
+        """
         return self._spam_checker.register_callbacks
 
     @property
     def register_account_validity_callbacks(self):
-        """Registers callbacks for account validity capabilities."""
+        """Registers callbacks for account validity capabilities.
+
+        Added in Synapse v1.39.0.
+        """
         return self._account_validity_handler.register_account_validity_callbacks
 
     @property
     def register_third_party_rules_callbacks(self):
-        """Registers callbacks for third party event rules capabilities."""
+        """Registers callbacks for third party event rules capabilities.
+
+        Added in Synapse v1.39.0.
+        """
         return self._third_party_event_rules.register_third_party_rules_callbacks
 
     @property
     def register_presence_router_callbacks(self):
-        """Registers callbacks for presence router capabilities."""
+        """Registers callbacks for presence router capabilities.
+
+        Added in Synapse v1.42.0.
+        """
         return self._presence_router.register_presence_router_callbacks
 
     @property
     def register_password_auth_provider_callbacks(self):
-        """Registers callbacks for password auth provider capabilities."""
+        """Registers callbacks for password auth provider capabilities.
+
+        Added in Synapse v1.46.0.
+        """
         return self._password_auth_provider.register_password_auth_provider_callbacks
 
     def register_web_resource(self, path: str, resource: IResource):
@@ -185,6 +200,8 @@ class ModuleApi:
         If multiple modules register a resource for the same path, the module that
         appears the highest in the configuration file takes priority.
 
+        Added in Synapse v1.37.0.
+
         Args:
             path: The path to register the resource for.
             resource: The resource to attach to this path.
@@ -199,6 +216,8 @@ class ModuleApi:
         """Allows making outbound HTTP requests to remote resources.
 
         An instance of synapse.http.client.SimpleHttpClient
+
+        Added in Synapse v1.22.0.
         """
         return self._http_client
 
@@ -208,22 +227,32 @@ class ModuleApi:
         public room list.
 
         An instance of synapse.module_api.PublicRoomListManager
+
+        Added in Synapse v1.22.0.
         """
         return self._public_room_list_manager
 
     @property
     def public_baseurl(self) -> str:
-        """The configured public base URL for this homeserver."""
+        """The configured public base URL for this homeserver.
+
+        Added in Synapse v1.39.0.
+        """
         return self._hs.config.server.public_baseurl
 
     @property
     def email_app_name(self) -> str:
-        """The application name configured in the homeserver's configuration."""
+        """The application name configured in the homeserver's configuration.
+
+        Added in Synapse v1.39.0.
+        """
         return self._hs.config.email.email_app_name
 
     async def get_userinfo_by_id(self, user_id: str) -> Optional[UserInfo]:
         """Get user info by user_id
 
+        Added in Synapse v1.41.0.
+
         Args:
             user_id: Fully qualified user id.
         Returns:
@@ -239,6 +268,8 @@ class ModuleApi:
     ) -> Requester:
         """Check the access_token provided for a request
 
+        Added in Synapse v1.39.0.
+
         Args:
             req: Incoming HTTP request
             allow_guest: True if guest users should be allowed. If this
@@ -264,6 +295,8 @@ class ModuleApi:
     async def is_user_admin(self, user_id: str) -> bool:
         """Checks if a user is a server admin.
 
+        Added in Synapse v1.39.0.
+
         Args:
             user_id: The Matrix ID of the user to check.
 
@@ -278,6 +311,8 @@ class ModuleApi:
         Takes a user id provided by the user and adds the @ and :domain to
         qualify it, if necessary
 
+        Added in Synapse v0.25.0.
+
         Args:
             username (str): provided user id
 
@@ -291,6 +326,8 @@ class ModuleApi:
     async def get_profile_for_user(self, localpart: str) -> ProfileInfo:
         """Look up the profile info for the user with the given localpart.
 
+        Added in Synapse v1.39.0.
+
         Args:
             localpart: The localpart to look up profile information for.
 
@@ -303,6 +340,8 @@ class ModuleApi:
         """Look up the threepids (email addresses and phone numbers) associated with the
         given Matrix user ID.
 
+        Added in Synapse v1.39.0.
+
         Args:
             user_id: The Matrix user ID to look up threepids for.
 
@@ -317,6 +356,8 @@ class ModuleApi:
     def check_user_exists(self, user_id):
         """Check if user exists.
 
+        Added in Synapse v0.25.0.
+
         Args:
             user_id (str): Complete @user:id
 
@@ -336,6 +377,8 @@ class ModuleApi:
         return that device to the user. Prefer separate calls to register_user and
         register_device.
 
+        Added in Synapse v0.25.0.
+
         Args:
             localpart (str): The localpart of the new user.
             displayname (str|None): The displayname of the new user.
@@ -356,6 +399,8 @@ class ModuleApi:
     ):
         """Registers a new user with given localpart and optional displayname, emails.
 
+        Added in Synapse v1.2.0.
+
         Args:
             localpart (str): The localpart of the new user.
             displayname (str|None): The displayname of the new user.
@@ -379,6 +424,8 @@ class ModuleApi:
     def register_device(self, user_id, device_id=None, initial_display_name=None):
         """Register a device for a user and generate an access token.
 
+        Added in Synapse v1.2.0.
+
         Args:
             user_id (str): full canonical @user:id
             device_id (str|None): The device ID to check, or None to generate
@@ -402,6 +449,8 @@ class ModuleApi:
     ) -> defer.Deferred:
         """Record a mapping from an external user id to a mxid
 
+        Added in Synapse v1.9.0.
+
         Args:
             auth_provider: identifier for the remote auth provider
             external_id: id on that system
@@ -421,6 +470,8 @@ class ModuleApi:
     ) -> str:
         """Generate a login token suitable for m.login.token authentication
 
+        Added in Synapse v1.9.0.
+
         Args:
             user_id: gives the ID of the user that the token is for
 
@@ -440,6 +491,8 @@ class ModuleApi:
     def invalidate_access_token(self, access_token):
         """Invalidate an access token for a user
 
+        Added in Synapse v0.25.0.
+
         Args:
             access_token(str): access token
 
@@ -470,6 +523,8 @@ class ModuleApi:
     def run_db_interaction(self, desc, func, *args, **kwargs):
         """Run a function with a database connection
 
+        Added in Synapse v0.25.0.
+
         Args:
             desc (str): description for the transaction, for metrics etc
             func (func): function to be run. Passed a database cursor object
@@ -493,6 +548,8 @@ class ModuleApi:
 
         This is deprecated in favor of complete_sso_login_async.
 
+        Added in Synapse v1.11.1.
+
         Args:
             registered_user_id: The MXID that has been registered as a previous step of
                 of this SSO login.
@@ -519,6 +576,8 @@ class ModuleApi:
         want their access token sent to `client_redirect_url`, or redirect them to that
         URL with a token directly if the URL matches with one of the whitelisted clients.
 
+        Added in Synapse v1.13.0.
+
         Args:
             registered_user_id: The MXID that has been registered as a previous step of
                 of this SSO login.
@@ -547,6 +606,8 @@ class ModuleApi:
         (This is exposed for compatibility with the old SpamCheckerApi. We should
         probably deprecate it and replace it with an async method in a subclass.)
 
+        Added in Synapse v1.22.0.
+
         Args:
             room_id: The room ID to get state events in.
             types: The event type and state key (using None
@@ -567,6 +628,8 @@ class ModuleApi:
     async def create_and_send_event_into_room(self, event_dict: JsonDict) -> EventBase:
         """Create and send an event into a room. Membership events are currently not supported.
 
+        Added in Synapse v1.22.0.
+
         Args:
             event_dict: A dictionary representing the event to send.
                 Required keys are `type`, `room_id`, `sender` and `content`.
@@ -607,6 +670,8 @@ class ModuleApi:
 
         Note that this method can only be run on the process that is configured to write to the
         presence stream. By default this is the main process.
+
+        Added in Synapse v1.32.0.
         """
         if self._hs._instance_name not in self._hs.config.worker.writers.presence:
             raise Exception(
@@ -661,6 +726,8 @@ class ModuleApi:
 
         Waits `msec` initially before calling `f` for the first time.
 
+        Added in Synapse v1.39.0.
+
         Args:
             f: The function to call repeatedly. f can be either synchronous or
                 asynchronous, and must follow Synapse's logcontext rules.
@@ -700,6 +767,8 @@ class ModuleApi:
     ):
         """Send an email on behalf of the homeserver.
 
+        Added in Synapse v1.39.0.
+
         Args:
             recipient: The email address for the recipient.
             subject: The email's subject.
@@ -723,6 +792,8 @@ class ModuleApi:
         By default, Synapse will look for these templates in its configured template
         directory, but another directory to search in can be provided.
 
+        Added in Synapse v1.39.0.
+
         Args:
             filenames: The name of the template files to look for.
             custom_template_directory: An additional directory to look for the files in.
@@ -740,13 +811,13 @@ class ModuleApi:
         """
         Checks whether an ID (user id, room, ...) comes from this homeserver.
 
+        Added in Synapse v1.44.0.
+
         Args:
             id: any Matrix id (e.g. user id, room id, ...), either as a raw id,
                 e.g. string "@user:example.com" or as a parsed UserID, RoomID, ...
         Returns:
             True if id comes from this homeserver, False otherwise.
-
-        Added in Synapse v1.44.0.
         """
         if isinstance(id, DomainSpecificString):
             return self._hs.is_mine(id)
@@ -759,6 +830,8 @@ class ModuleApi:
         """
         Return the list of user IPs and agents for a user.
 
+        Added in Synapse v1.44.0.
+
         Args:
             user_id: the id of a user, local or remote
             since_ts: a timestamp in seconds since the epoch,
@@ -767,8 +840,6 @@ class ModuleApi:
             The list of all UserIpAndAgent that the user has
             used to connect to this homeserver since `since_ts`.
             If the user is remote, this list is empty.
-
-        Added in Synapse v1.44.0.
         """
         # Don't hit the db if this is not a local user.
         is_mine = False
@@ -807,6 +878,8 @@ class PublicRoomListManager:
     async def room_is_in_public_room_list(self, room_id: str) -> bool:
         """Checks whether a room is in the public room list.
 
+        Added in Synapse v1.22.0.
+
         Args:
             room_id: The ID of the room.
 
@@ -823,6 +896,8 @@ class PublicRoomListManager:
     async def add_room_to_public_room_list(self, room_id: str) -> None:
         """Publishes a room to the public room list.
 
+        Added in Synapse v1.22.0.
+
         Args:
             room_id: The ID of the room.
         """
@@ -831,6 +906,8 @@ class PublicRoomListManager:
     async def remove_room_from_public_room_list(self, room_id: str) -> None:
         """Removes a room from the public room list.
 
+        Added in Synapse v1.22.0.
+
         Args:
             room_id: The ID of the room.
         """
-- 
cgit 1.5.1


From d52c58dfa3f548b489dae0b1945cf733d4a6538c Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 26 Oct 2021 07:38:45 -0400
Subject: Add a background update for updating MSC3440 relation threads.
 (#11181)

---
 changelog.d/11181.feature                          |  1 +
 .../storage/databases/main/events_bg_updates.py    | 85 +++++++++++++++++++++-
 .../schema/main/delta/65/02_thread_relations.sql   | 18 +++++
 3 files changed, 102 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11181.feature
 create mode 100644 synapse/storage/schema/main/delta/65/02_thread_relations.sql

diff --git a/changelog.d/11181.feature b/changelog.d/11181.feature
new file mode 100644
index 0000000000..76b0d28084
--- /dev/null
+++ b/changelog.d/11181.feature
@@ -0,0 +1 @@
+Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
index fc49112063..f92d824876 100644
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -17,11 +17,15 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 
 import attr
 
-from synapse.api.constants import EventContentFields
+from synapse.api.constants import EventContentFields, RelationTypes
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.events import make_event_from_dict
 from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
-from synapse.storage.database import DatabasePool, make_tuple_comparison_clause
+from synapse.storage.database import (
+    DatabasePool,
+    LoggingTransaction,
+    make_tuple_comparison_clause,
+)
 from synapse.storage.databases.main.events import PersistEventsStore
 from synapse.storage.types import Cursor
 from synapse.types import JsonDict
@@ -167,6 +171,10 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
             self._purged_chain_cover_index,
         )
 
+        self.db_pool.updates.register_background_update_handler(
+            "event_thread_relation", self._event_thread_relation
+        )
+
         ################################################################################
 
         # bg updates for replacing stream_ordering with a BIGINT
@@ -1091,6 +1099,79 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
 
         return result
 
+    async def _event_thread_relation(self, progress: JsonDict, batch_size: int) -> int:
+        """Background update handler which will store thread relations for existing events."""
+        last_event_id = progress.get("last_event_id", "")
+
+        def _event_thread_relation_txn(txn: LoggingTransaction) -> int:
+            txn.execute(
+                """
+                SELECT event_id, json FROM event_json
+                LEFT JOIN event_relations USING (event_id)
+                WHERE event_id > ? AND relates_to_id IS NULL
+                ORDER BY event_id LIMIT ?
+                """,
+                (last_event_id, batch_size),
+            )
+
+            results = list(txn)
+            missing_thread_relations = []
+            for (event_id, event_json_raw) in results:
+                try:
+                    event_json = db_to_json(event_json_raw)
+                except Exception as e:
+                    logger.warning(
+                        "Unable to load event %s (no relations will be updated): %s",
+                        event_id,
+                        e,
+                    )
+                    continue
+
+                # If there's no relation (or it is not a thread), skip!
+                relates_to = event_json["content"].get("m.relates_to")
+                if not relates_to or not isinstance(relates_to, dict):
+                    continue
+                if relates_to.get("rel_type") != RelationTypes.THREAD:
+                    continue
+
+                # Get the parent ID.
+                parent_id = relates_to.get("event_id")
+                if not isinstance(parent_id, str):
+                    continue
+
+                missing_thread_relations.append((event_id, parent_id))
+
+            # Insert the missing data.
+            self.db_pool.simple_insert_many_txn(
+                txn=txn,
+                table="event_relations",
+                values=[
+                    {
+                        "event_id": event_id,
+                        "relates_to_Id": parent_id,
+                        "relation_type": RelationTypes.THREAD,
+                    }
+                    for event_id, parent_id in missing_thread_relations
+                ],
+            )
+
+            if results:
+                latest_event_id = results[-1][0]
+                self.db_pool.updates._background_update_progress_txn(
+                    txn, "event_thread_relation", {"last_event_id": latest_event_id}
+                )
+
+            return len(results)
+
+        num_rows = await self.db_pool.runInteraction(
+            desc="event_thread_relation", func=_event_thread_relation_txn
+        )
+
+        if not num_rows:
+            await self.db_pool.updates._end_background_update("event_thread_relation")
+
+        return num_rows
+
     async def _background_populate_stream_ordering2(
         self, progress: JsonDict, batch_size: int
     ) -> int:
diff --git a/synapse/storage/schema/main/delta/65/02_thread_relations.sql b/synapse/storage/schema/main/delta/65/02_thread_relations.sql
new file mode 100644
index 0000000000..d60517f7b4
--- /dev/null
+++ b/synapse/storage/schema/main/delta/65/02_thread_relations.sql
@@ -0,0 +1,18 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Check old events for thread relations.
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (6502, 'event_thread_relation', '{}');
-- 
cgit 1.5.1


From 7004f43da143f5d1d35c742add1238c51e62ca19 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 26 Oct 2021 13:45:38 +0100
Subject: Move DNS lookups into separate thread pool (#11177)

This is to stop large bursts of lookups starving out other users of the
thread pools.

Fixes #11049.
---
 changelog.d/11177.bugfix     |   1 +
 synapse/app/_base.py         |  13 ++++-
 synapse/util/gai_resolver.py | 136 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 149 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11177.bugfix
 create mode 100644 synapse/util/gai_resolver.py

diff --git a/changelog.d/11177.bugfix b/changelog.d/11177.bugfix
new file mode 100644
index 0000000000..ca5bc0df28
--- /dev/null
+++ b/changelog.d/11177.bugfix
@@ -0,0 +1 @@
+Fix a performance regression introduced in v1.44.0 which could cause client requests to time out when making large numbers of outbound requests.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 2ca2e051e4..03627cdcba 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -31,6 +31,7 @@ import twisted
 from twisted.internet import defer, error, reactor
 from twisted.logger import LoggingFile, LogLevel
 from twisted.protocols.tls import TLSMemoryBIOFactory
+from twisted.python.threadpool import ThreadPool
 
 import synapse
 from synapse.api.constants import MAX_PDU_SIZE
@@ -48,6 +49,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_proces
 from synapse.metrics.jemalloc import setup_jemalloc_stats
 from synapse.util.caches.lrucache import setup_expire_lru_cache_entries
 from synapse.util.daemonize import daemonize_process
+from synapse.util.gai_resolver import GAIResolver
 from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
 
@@ -338,9 +340,18 @@ async def start(hs: "HomeServer"):
     Args:
         hs: homeserver instance
     """
+    reactor = hs.get_reactor()
+
+    # We want to use a separate thread pool for the resolver so that large
+    # numbers of DNS requests don't starve out other users of the threadpool.
+    resolver_threadpool = ThreadPool(name="gai_resolver")
+    resolver_threadpool.start()
+    reactor.installNameResolver(
+        GAIResolver(reactor, getThreadPool=lambda: resolver_threadpool)
+    )
+
     # Set up the SIGHUP machinery.
     if hasattr(signal, "SIGHUP"):
-        reactor = hs.get_reactor()
 
         @wrap_as_background_process("sighup")
         def handle_sighup(*args, **kwargs):
diff --git a/synapse/util/gai_resolver.py b/synapse/util/gai_resolver.py
new file mode 100644
index 0000000000..a447ce4e55
--- /dev/null
+++ b/synapse/util/gai_resolver.py
@@ -0,0 +1,136 @@
+# This is a direct lift from
+# https://github.com/twisted/twisted/blob/release-21.2.0-10091/src/twisted/internet/_resolver.py.
+# We copy it here as we need to instantiate `GAIResolver` manually, but it is a
+# private class.
+
+
+from socket import (
+    AF_INET,
+    AF_INET6,
+    AF_UNSPEC,
+    SOCK_DGRAM,
+    SOCK_STREAM,
+    gaierror,
+    getaddrinfo,
+)
+
+from zope.interface import implementer
+
+from twisted.internet.address import IPv4Address, IPv6Address
+from twisted.internet.interfaces import IHostnameResolver, IHostResolution
+from twisted.internet.threads import deferToThreadPool
+
+
+@implementer(IHostResolution)
+class HostResolution:
+    """
+    The in-progress resolution of a given hostname.
+    """
+
+    def __init__(self, name):
+        """
+        Create a L{HostResolution} with the given name.
+        """
+        self.name = name
+
+    def cancel(self):
+        # IHostResolution.cancel
+        raise NotImplementedError()
+
+
+_any = frozenset([IPv4Address, IPv6Address])
+
+_typesToAF = {
+    frozenset([IPv4Address]): AF_INET,
+    frozenset([IPv6Address]): AF_INET6,
+    _any: AF_UNSPEC,
+}
+
+_afToType = {
+    AF_INET: IPv4Address,
+    AF_INET6: IPv6Address,
+}
+
+_transportToSocket = {
+    "TCP": SOCK_STREAM,
+    "UDP": SOCK_DGRAM,
+}
+
+_socktypeToType = {
+    SOCK_STREAM: "TCP",
+    SOCK_DGRAM: "UDP",
+}
+
+
+@implementer(IHostnameResolver)
+class GAIResolver:
+    """
+    L{IHostnameResolver} implementation that resolves hostnames by calling
+    L{getaddrinfo} in a thread.
+    """
+
+    def __init__(self, reactor, getThreadPool=None, getaddrinfo=getaddrinfo):
+        """
+        Create a L{GAIResolver}.
+        @param reactor: the reactor to schedule result-delivery on
+        @type reactor: L{IReactorThreads}
+        @param getThreadPool: a function to retrieve the thread pool to use for
+            scheduling name resolutions.  If not supplied, the use the given
+            C{reactor}'s thread pool.
+        @type getThreadPool: 0-argument callable returning a
+            L{twisted.python.threadpool.ThreadPool}
+        @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly
+            parameterized for testing.
+        @type getaddrinfo: callable with the same signature as L{getaddrinfo}
+        """
+        self._reactor = reactor
+        self._getThreadPool = (
+            reactor.getThreadPool if getThreadPool is None else getThreadPool
+        )
+        self._getaddrinfo = getaddrinfo
+
+    def resolveHostName(
+        self,
+        resolutionReceiver,
+        hostName,
+        portNumber=0,
+        addressTypes=None,
+        transportSemantics="TCP",
+    ):
+        """
+        See L{IHostnameResolver.resolveHostName}
+        @param resolutionReceiver: see interface
+        @param hostName: see interface
+        @param portNumber: see interface
+        @param addressTypes: see interface
+        @param transportSemantics: see interface
+        @return: see interface
+        """
+        pool = self._getThreadPool()
+        addressFamily = _typesToAF[
+            _any if addressTypes is None else frozenset(addressTypes)
+        ]
+        socketType = _transportToSocket[transportSemantics]
+
+        def get():
+            try:
+                return self._getaddrinfo(
+                    hostName, portNumber, addressFamily, socketType
+                )
+            except gaierror:
+                return []
+
+        d = deferToThreadPool(self._reactor, pool, get)
+        resolution = HostResolution(hostName)
+        resolutionReceiver.resolutionBegan(resolution)
+
+        @d.addCallback
+        def deliverResults(result):
+            for family, socktype, _proto, _cannoname, sockaddr in result:
+                addrType = _afToType[family]
+                resolutionReceiver.addressResolved(
+                    addrType(_socktypeToType.get(socktype, "TCP"), *sockaddr)
+                )
+            resolutionReceiver.resolutionComplete()
+
+        return resolution
-- 
cgit 1.5.1


From cc75a6b1b20f599c6ec6699fb77c8a72b87d1ec2 Mon Sep 17 00:00:00 2001
From: Sean Quah <seanq@element.io>
Date: Tue, 26 Oct 2021 14:04:51 +0100
Subject: 1.46.0rc1

---
 CHANGES.md                | 74 +++++++++++++++++++++++++++++++++++++++++++++++
 changelog.d/10548.feature |  1 -
 changelog.d/10930.bugfix  |  1 -
 changelog.d/10972.misc    |  1 -
 changelog.d/10975.feature |  1 -
 changelog.d/10984.misc    |  1 -
 changelog.d/11001.bugfix  |  1 -
 changelog.d/11008.misc    |  1 -
 changelog.d/11009.bugfix  |  1 -
 changelog.d/11012.bugfix  |  1 -
 changelog.d/11014.misc    |  1 -
 changelog.d/11024.misc    |  1 -
 changelog.d/11027.bugfix  |  1 -
 changelog.d/11035.misc    |  1 -
 changelog.d/11048.misc    |  1 -
 changelog.d/11051.bugfix  |  1 -
 changelog.d/11054.misc    |  1 -
 changelog.d/11055.misc    |  1 -
 changelog.d/11056.misc    |  1 -
 changelog.d/11057.misc    |  1 -
 changelog.d/11065.misc    |  1 -
 changelog.d/11066.misc    |  1 -
 changelog.d/11068.misc    |  1 -
 changelog.d/11069.doc     |  1 -
 changelog.d/11070.misc    |  1 -
 changelog.d/11071.misc    |  1 -
 changelog.d/11075.bugfix  |  1 -
 changelog.d/11076.misc    |  1 -
 changelog.d/11077.bugfix  |  1 -
 changelog.d/11078.bugfix  |  1 -
 changelog.d/11083.doc     |  1 -
 changelog.d/11088.feature |  1 -
 changelog.d/11089.bugfix  |  1 -
 changelog.d/11092.doc     |  1 -
 changelog.d/11093.doc     |  1 -
 changelog.d/11095.misc    |  1 -
 changelog.d/11096.doc     |  1 -
 changelog.d/11101.bugfix  |  1 -
 changelog.d/11103.bugfix  |  1 -
 changelog.d/11109.misc    |  1 -
 changelog.d/11112.bugfix  |  1 -
 changelog.d/11115.misc    |  1 -
 changelog.d/11116.misc    |  1 -
 changelog.d/11118.doc     |  1 -
 changelog.d/11120.bugfix  |  1 -
 changelog.d/11121.misc    |  1 -
 changelog.d/11122.misc    |  1 -
 changelog.d/11132.doc     |  1 -
 changelog.d/11138.misc    |  1 -
 changelog.d/11139.misc    |  1 -
 changelog.d/11143.misc    |  1 -
 changelog.d/11144.misc    |  1 -
 changelog.d/11145.bugfix  |  1 -
 changelog.d/11146.misc    |  1 -
 changelog.d/11174.feature |  1 -
 changelog.d/11177.bugfix  |  1 -
 changelog.d/11180.feature |  1 -
 changelog.d/11181.feature |  1 -
 changelog.d/11183.doc     |  1 -
 debian/changelog          |  6 ++++
 synapse/__init__.py       |  2 +-
 61 files changed, 81 insertions(+), 59 deletions(-)
 delete mode 100644 changelog.d/10548.feature
 delete mode 100644 changelog.d/10930.bugfix
 delete mode 100644 changelog.d/10972.misc
 delete mode 100644 changelog.d/10975.feature
 delete mode 100644 changelog.d/10984.misc
 delete mode 100644 changelog.d/11001.bugfix
 delete mode 100644 changelog.d/11008.misc
 delete mode 100644 changelog.d/11009.bugfix
 delete mode 100644 changelog.d/11012.bugfix
 delete mode 100644 changelog.d/11014.misc
 delete mode 100644 changelog.d/11024.misc
 delete mode 100644 changelog.d/11027.bugfix
 delete mode 100644 changelog.d/11035.misc
 delete mode 100644 changelog.d/11048.misc
 delete mode 100644 changelog.d/11051.bugfix
 delete mode 100644 changelog.d/11054.misc
 delete mode 100644 changelog.d/11055.misc
 delete mode 100644 changelog.d/11056.misc
 delete mode 100644 changelog.d/11057.misc
 delete mode 100644 changelog.d/11065.misc
 delete mode 100644 changelog.d/11066.misc
 delete mode 100644 changelog.d/11068.misc
 delete mode 100644 changelog.d/11069.doc
 delete mode 100644 changelog.d/11070.misc
 delete mode 100644 changelog.d/11071.misc
 delete mode 100644 changelog.d/11075.bugfix
 delete mode 100644 changelog.d/11076.misc
 delete mode 100644 changelog.d/11077.bugfix
 delete mode 100644 changelog.d/11078.bugfix
 delete mode 100644 changelog.d/11083.doc
 delete mode 100644 changelog.d/11088.feature
 delete mode 100644 changelog.d/11089.bugfix
 delete mode 100644 changelog.d/11092.doc
 delete mode 100644 changelog.d/11093.doc
 delete mode 100644 changelog.d/11095.misc
 delete mode 100644 changelog.d/11096.doc
 delete mode 100644 changelog.d/11101.bugfix
 delete mode 100644 changelog.d/11103.bugfix
 delete mode 100644 changelog.d/11109.misc
 delete mode 100644 changelog.d/11112.bugfix
 delete mode 100644 changelog.d/11115.misc
 delete mode 100644 changelog.d/11116.misc
 delete mode 100644 changelog.d/11118.doc
 delete mode 100644 changelog.d/11120.bugfix
 delete mode 100644 changelog.d/11121.misc
 delete mode 100644 changelog.d/11122.misc
 delete mode 100644 changelog.d/11132.doc
 delete mode 100644 changelog.d/11138.misc
 delete mode 100644 changelog.d/11139.misc
 delete mode 100644 changelog.d/11143.misc
 delete mode 100644 changelog.d/11144.misc
 delete mode 100644 changelog.d/11145.bugfix
 delete mode 100644 changelog.d/11146.misc
 delete mode 100644 changelog.d/11174.feature
 delete mode 100644 changelog.d/11177.bugfix
 delete mode 100644 changelog.d/11180.feature
 delete mode 100644 changelog.d/11181.feature
 delete mode 100644 changelog.d/11183.doc

diff --git a/CHANGES.md b/CHANGES.md
index 92e6c6873e..88f8b5e01d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,77 @@
+Synapse 1.46.0rc1 (2021-10-26)
+==============================
+
+Features
+--------
+
+- Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180))
+- Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975))
+- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181))
+- Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174))
+
+
+Bugfixes
+--------
+
+- Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr. ([\#10930](https://github.com/matrix-org/synapse/issues/10930))
+- Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#11001](https://github.com/matrix-org/synapse/issues/11001), [\#11009](https://github.com/matrix-org/synapse/issues/11009), [\#11012](https://github.com/matrix-org/synapse/issues/11012))
+- Fix 500 error on `/messages` when the server accumulates more than 5 backwards extremities at a given depth for a room. ([\#11027](https://github.com/matrix-org/synapse/issues/11027))
+- Fix a bug where setting a user's external_id via the admin API returns 500 and deletes users existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051))
+- Fix a long-standing bug where users excluded from the user directory were added into the directory if they belonged to a room which became public or private. ([\#11075](https://github.com/matrix-org/synapse/issues/11075))
+- Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding. ([\#11077](https://github.com/matrix-org/synapse/issues/11077), [\#11089](https://github.com/matrix-org/synapse/issues/11089))
+- Fix broken export-data admin command and add test script checking the command to CI. ([\#11078](https://github.com/matrix-org/synapse/issues/11078))
+- Show an error when timestamp in seconds is provided to the `/purge_media_cache` Admin API. ([\#11101](https://github.com/matrix-org/synapse/issues/11101))
+- Fix local users who left all their rooms being removed from the user directory, even if the "search_all_users" config option was enabled. ([\#11103](https://github.com/matrix-org/synapse/issues/11103))
+- Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112))
+- Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120))
+- Fix a bug introduced in Synapse v1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145))
+- Fix a performance regression introduced in v1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
+
+
+Improved Documentation
+----------------------
+
+- Fix broken links relating to module API deprecation in the upgrade notes. ([\#11069](https://github.com/matrix-org/synapse/issues/11069))
+- Add more information about what happens when a user is deactivated. ([\#11083](https://github.com/matrix-org/synapse/issues/11083))
+- Clarify the the sample log config can be copied from the documentation without issue. ([\#11092](https://github.com/matrix-org/synapse/issues/11092))
+- Update the admin API documentation with an updated list of the characters allowed in registration tokens. ([\#11093](https://github.com/matrix-org/synapse/issues/11093))
+- Document Synapse's behaviour when dealing with multiple modules registering the same callbacks and/or handlers for the same HTTP endpoints. ([\#11096](https://github.com/matrix-org/synapse/issues/11096))
+- Fix instances of `[example]{.title-ref}` in the upgrade documentation as a result of prior RST to Markdown conversion. ([\#11118](https://github.com/matrix-org/synapse/issues/11118))
+- Document the version of Synapse each module callback was introduced in. ([\#11132](https://github.com/matrix-org/synapse/issues/11132))
+- Document the version of Synapse that introduced each module API method. ([\#11183](https://github.com/matrix-org/synapse/issues/11183))
+
+
+Internal Changes
+----------------
+
+- Add type hints to `synapse.storage.databases.main.client_ips`. ([\#10972](https://github.com/matrix-org/synapse/issues/10972))
+- Fix spurious warnings about losing the logging context on the `ReplicationCommandHandler` when losing the replication connection. ([\#10984](https://github.com/matrix-org/synapse/issues/10984))
+- Include rejected status when we log events. ([\#11008](https://github.com/matrix-org/synapse/issues/11008))
+- Add some extra logging to the event persistence code. ([\#11014](https://github.com/matrix-org/synapse/issues/11014))
+- Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024))
+- Rearrange the internal workings of the incremental user directory updates. ([\#11035](https://github.com/matrix-org/synapse/issues/11035))
+- Simplify the user admin API tests. ([\#11048](https://github.com/matrix-org/synapse/issues/11048))
+- Mark the Synapse package as containing type annotations and fix export declarations so that Synapse pluggable modules may be type checked against Synapse. ([\#11054](https://github.com/matrix-org/synapse/issues/11054))
+- Improve type hints for `_wrap_in_base_path` decorator used by `MediaFilePaths`. ([\#11055](https://github.com/matrix-org/synapse/issues/11055))
+- Remove dead code from `MediaFilePaths`. ([\#11056](https://github.com/matrix-org/synapse/issues/11056))
+- Add tests for `MediaFilePaths` class. ([\#11057](https://github.com/matrix-org/synapse/issues/11057))
+- Be more lenient when parsing oEmbed response versions. ([\#11065](https://github.com/matrix-org/synapse/issues/11065))
+- Add type hints to `synapse.events`. ([\#11066](https://github.com/matrix-org/synapse/issues/11066))
+- Always dump logs from unit tests during CI runs. ([\#11068](https://github.com/matrix-org/synapse/issues/11068))
+- Create a separate module for the retention configuration. ([\#11070](https://github.com/matrix-org/synapse/issues/11070))
+- Add a test for the workaround introduced in [\#11042](https://github.com/matrix-org/synapse/pull/11042) concerning the behaviour of third-party rule modules and `SynapseError`s. ([\#11071](https://github.com/matrix-org/synapse/issues/11071))
+- Fix type hints in the relations tests. ([\#11076](https://github.com/matrix-org/synapse/issues/11076))
+- Add type hints to most `HomeServer` parameters. ([\#11095](https://github.com/matrix-org/synapse/issues/11095))
+- Add missing type hints to `synapse.api` module. ([\#11109](https://github.com/matrix-org/synapse/issues/11109))
+- Clean up some of the federation event authentication code for clarity. ([\#11115](https://github.com/matrix-org/synapse/issues/11115), [\#11116](https://github.com/matrix-org/synapse/issues/11116), [\#11122](https://github.com/matrix-org/synapse/issues/11122))
+- Add type hints for event fetching. ([\#11121](https://github.com/matrix-org/synapse/issues/11121))
+- Add docstrings and comments to the application service ephemeral event sending code. ([\#11138](https://github.com/matrix-org/synapse/issues/11138))
+- Update the `sign_json` script to support inline configuration of the signing key. ([\#11139](https://github.com/matrix-org/synapse/issues/11139))
+- Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room. ([\#11143](https://github.com/matrix-org/synapse/issues/11143))
+- Fix broken link in the docker image README. ([\#11144](https://github.com/matrix-org/synapse/issues/11144))
+- Add missing type hints to `synapse.crypto`. ([\#11146](https://github.com/matrix-org/synapse/issues/11146))
+
+
 Synapse 1.45.1 (2021-10-20)
 ===========================
 
diff --git a/changelog.d/10548.feature b/changelog.d/10548.feature
deleted file mode 100644
index 263a811faf..0000000000
--- a/changelog.d/10548.feature
+++ /dev/null
@@ -1 +0,0 @@
-Port the Password Auth Providers module interface to the new generic interface.
\ No newline at end of file
diff --git a/changelog.d/10930.bugfix b/changelog.d/10930.bugfix
deleted file mode 100644
index 756bfe9107..0000000000
--- a/changelog.d/10930.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr.
diff --git a/changelog.d/10972.misc b/changelog.d/10972.misc
deleted file mode 100644
index f66a7beaf0..0000000000
--- a/changelog.d/10972.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to `synapse.storage.databases.main.client_ips`.
diff --git a/changelog.d/10975.feature b/changelog.d/10975.feature
deleted file mode 100644
index 167426e1fc..0000000000
--- a/changelog.d/10975.feature
+++ /dev/null
@@ -1 +0,0 @@
-Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch.
diff --git a/changelog.d/10984.misc b/changelog.d/10984.misc
deleted file mode 100644
index 86c4081cc4..0000000000
--- a/changelog.d/10984.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix spurious warnings about losing the logging context on the `ReplicationCommandHandler` when losing the replication connection.
diff --git a/changelog.d/11001.bugfix b/changelog.d/11001.bugfix
deleted file mode 100644
index f51ffb3481..0000000000
--- a/changelog.d/11001.bugfix
+++ /dev/null
@@ -1 +0,0 @@
- Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state.
diff --git a/changelog.d/11008.misc b/changelog.d/11008.misc
deleted file mode 100644
index a67d95d66f..0000000000
--- a/changelog.d/11008.misc
+++ /dev/null
@@ -1 +0,0 @@
-Include rejected status when we log events.
diff --git a/changelog.d/11009.bugfix b/changelog.d/11009.bugfix
deleted file mode 100644
index 13b8e5983b..0000000000
--- a/changelog.d/11009.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state.
diff --git a/changelog.d/11012.bugfix b/changelog.d/11012.bugfix
deleted file mode 100644
index 13b8e5983b..0000000000
--- a/changelog.d/11012.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state.
diff --git a/changelog.d/11014.misc b/changelog.d/11014.misc
deleted file mode 100644
index 4b99ea354f..0000000000
--- a/changelog.d/11014.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add some extra logging to the event persistence code.
diff --git a/changelog.d/11024.misc b/changelog.d/11024.misc
deleted file mode 100644
index 51ad800d4d..0000000000
--- a/changelog.d/11024.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add support for Ubuntu 21.10 "Impish Indri".
\ No newline at end of file
diff --git a/changelog.d/11027.bugfix b/changelog.d/11027.bugfix
deleted file mode 100644
index ae6cc44470..0000000000
--- a/changelog.d/11027.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix 500 error on `/messages` when the server accumulates more than 5 backwards extremities at a given depth for a room.
diff --git a/changelog.d/11035.misc b/changelog.d/11035.misc
deleted file mode 100644
index 6b45b7e9bd..0000000000
--- a/changelog.d/11035.misc
+++ /dev/null
@@ -1 +0,0 @@
-Rearrange the internal workings of the incremental user directory updates.
\ No newline at end of file
diff --git a/changelog.d/11048.misc b/changelog.d/11048.misc
deleted file mode 100644
index 22d3c956f5..0000000000
--- a/changelog.d/11048.misc
+++ /dev/null
@@ -1 +0,0 @@
-Simplify the user admin API tests.
\ No newline at end of file
diff --git a/changelog.d/11051.bugfix b/changelog.d/11051.bugfix
deleted file mode 100644
index 63126843d2..0000000000
--- a/changelog.d/11051.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug where setting a user's external_id via the admin API returns 500 and deletes users existing external mappings if that external ID is already mapped.
\ No newline at end of file
diff --git a/changelog.d/11054.misc b/changelog.d/11054.misc
deleted file mode 100644
index 1103368fec..0000000000
--- a/changelog.d/11054.misc
+++ /dev/null
@@ -1 +0,0 @@
-Mark the Synapse package as containing type annotations and fix export declarations so that Synapse pluggable modules may be type checked against Synapse.
diff --git a/changelog.d/11055.misc b/changelog.d/11055.misc
deleted file mode 100644
index 27688c3214..0000000000
--- a/changelog.d/11055.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints for `_wrap_in_base_path` decorator used by `MediaFilePaths`.
diff --git a/changelog.d/11056.misc b/changelog.d/11056.misc
deleted file mode 100644
index dd701ed177..0000000000
--- a/changelog.d/11056.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove dead code from `MediaFilePaths`.
diff --git a/changelog.d/11057.misc b/changelog.d/11057.misc
deleted file mode 100644
index 4d412d3e9b..0000000000
--- a/changelog.d/11057.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add tests for `MediaFilePaths` class.
diff --git a/changelog.d/11065.misc b/changelog.d/11065.misc
deleted file mode 100644
index c6f37fc52b..0000000000
--- a/changelog.d/11065.misc
+++ /dev/null
@@ -1 +0,0 @@
-Be more lenient when parsing oEmbed response versions.
diff --git a/changelog.d/11066.misc b/changelog.d/11066.misc
deleted file mode 100644
index 1e337bee54..0000000000
--- a/changelog.d/11066.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to `synapse.events`.
diff --git a/changelog.d/11068.misc b/changelog.d/11068.misc
deleted file mode 100644
index 1fe69aecde..0000000000
--- a/changelog.d/11068.misc
+++ /dev/null
@@ -1 +0,0 @@
-Always dump logs from unit tests during CI runs.
diff --git a/changelog.d/11069.doc b/changelog.d/11069.doc
deleted file mode 100644
index dae4ae1777..0000000000
--- a/changelog.d/11069.doc
+++ /dev/null
@@ -1 +0,0 @@
-Fix broken links relating to module API deprecation in the upgrade notes.
diff --git a/changelog.d/11070.misc b/changelog.d/11070.misc
deleted file mode 100644
index 52b23f9671..0000000000
--- a/changelog.d/11070.misc
+++ /dev/null
@@ -1 +0,0 @@
-Create a separate module for the retention configuration.
diff --git a/changelog.d/11071.misc b/changelog.d/11071.misc
deleted file mode 100644
index 33a11abdd5..0000000000
--- a/changelog.d/11071.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a test for the workaround introduced in [\#11042](https://github.com/matrix-org/synapse/pull/11042) concerning the behaviour of third-party rule modules and `SynapseError`s.
diff --git a/changelog.d/11075.bugfix b/changelog.d/11075.bugfix
deleted file mode 100644
index 9b24971c5a..0000000000
--- a/changelog.d/11075.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where users excluded from the user directory were added into the directory if they belonged to a room which became public or private.
\ No newline at end of file
diff --git a/changelog.d/11076.misc b/changelog.d/11076.misc
deleted file mode 100644
index c581a86e47..0000000000
--- a/changelog.d/11076.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix type hints in the relations tests.
diff --git a/changelog.d/11077.bugfix b/changelog.d/11077.bugfix
deleted file mode 100644
index dc35c86440..0000000000
--- a/changelog.d/11077.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding.
diff --git a/changelog.d/11078.bugfix b/changelog.d/11078.bugfix
deleted file mode 100644
index cc813babe4..0000000000
--- a/changelog.d/11078.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix broken export-data admin command and add test script checking the command to CI.
\ No newline at end of file
diff --git a/changelog.d/11083.doc b/changelog.d/11083.doc
deleted file mode 100644
index 245dd3758d..0000000000
--- a/changelog.d/11083.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add more information about what happens when a user is deactivated.
\ No newline at end of file
diff --git a/changelog.d/11088.feature b/changelog.d/11088.feature
deleted file mode 100644
index 76b0d28084..0000000000
--- a/changelog.d/11088.feature
+++ /dev/null
@@ -1 +0,0 @@
-Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
diff --git a/changelog.d/11089.bugfix b/changelog.d/11089.bugfix
deleted file mode 100644
index dc35c86440..0000000000
--- a/changelog.d/11089.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding.
diff --git a/changelog.d/11092.doc b/changelog.d/11092.doc
deleted file mode 100644
index 916c2b3476..0000000000
--- a/changelog.d/11092.doc
+++ /dev/null
@@ -1 +0,0 @@
-Clarify the the sample log config can be copied from the documentation without issue.
diff --git a/changelog.d/11093.doc b/changelog.d/11093.doc
deleted file mode 100644
index 70fca0bdce..0000000000
--- a/changelog.d/11093.doc
+++ /dev/null
@@ -1 +0,0 @@
-Update the admin API documentation with an updated list of the characters allowed in registration tokens.
diff --git a/changelog.d/11095.misc b/changelog.d/11095.misc
deleted file mode 100644
index 786e90b595..0000000000
--- a/changelog.d/11095.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to most `HomeServer` parameters.
diff --git a/changelog.d/11096.doc b/changelog.d/11096.doc
deleted file mode 100644
index d8e7424289..0000000000
--- a/changelog.d/11096.doc
+++ /dev/null
@@ -1 +0,0 @@
-Document Synapse's behaviour when dealing with multiple modules registering the same callbacks and/or handlers for the same HTTP endpoints.
diff --git a/changelog.d/11101.bugfix b/changelog.d/11101.bugfix
deleted file mode 100644
index 0de507848f..0000000000
--- a/changelog.d/11101.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Show an error when timestamp in seconds is provided to the `/purge_media_cache` Admin API.
\ No newline at end of file
diff --git a/changelog.d/11103.bugfix b/changelog.d/11103.bugfix
deleted file mode 100644
index 3498f04a45..0000000000
--- a/changelog.d/11103.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix local users who left all their rooms being removed from the user directory, even if the "search_all_users" config option was enabled.
\ No newline at end of file
diff --git a/changelog.d/11109.misc b/changelog.d/11109.misc
deleted file mode 100644
index d83936ccc4..0000000000
--- a/changelog.d/11109.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add missing type hints to `synapse.api` module.
diff --git a/changelog.d/11112.bugfix b/changelog.d/11112.bugfix
deleted file mode 100644
index c8e22da8cf..0000000000
--- a/changelog.d/11112.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time.
diff --git a/changelog.d/11115.misc b/changelog.d/11115.misc
deleted file mode 100644
index 9a765435db..0000000000
--- a/changelog.d/11115.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
diff --git a/changelog.d/11116.misc b/changelog.d/11116.misc
deleted file mode 100644
index 9a765435db..0000000000
--- a/changelog.d/11116.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
diff --git a/changelog.d/11118.doc b/changelog.d/11118.doc
deleted file mode 100644
index 3c2187f3b1..0000000000
--- a/changelog.d/11118.doc
+++ /dev/null
@@ -1 +0,0 @@
-Fix instances of `[example]{.title-ref}` in the upgrade documentation as a result of prior RST to Markdown conversion.
diff --git a/changelog.d/11120.bugfix b/changelog.d/11120.bugfix
deleted file mode 100644
index 6b39e3e89d..0000000000
--- a/changelog.d/11120.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Identity server connection is no longer ignoring `ip_range_whitelist`.
diff --git a/changelog.d/11121.misc b/changelog.d/11121.misc
deleted file mode 100644
index 916beeaacb..0000000000
--- a/changelog.d/11121.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints for event fetching.
diff --git a/changelog.d/11122.misc b/changelog.d/11122.misc
deleted file mode 100644
index 9a765435db..0000000000
--- a/changelog.d/11122.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up some of the federation event authentication code for clarity.
diff --git a/changelog.d/11132.doc b/changelog.d/11132.doc
deleted file mode 100644
index 4f38be5b27..0000000000
--- a/changelog.d/11132.doc
+++ /dev/null
@@ -1 +0,0 @@
-Document the version of Synapse each module callback was introduced in.
diff --git a/changelog.d/11138.misc b/changelog.d/11138.misc
deleted file mode 100644
index 79b7776975..0000000000
--- a/changelog.d/11138.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add docstrings and comments to the application service ephemeral event sending code.
\ No newline at end of file
diff --git a/changelog.d/11139.misc b/changelog.d/11139.misc
deleted file mode 100644
index 86a9189200..0000000000
--- a/changelog.d/11139.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update the `sign_json` script to support inline configuration of the signing key.
diff --git a/changelog.d/11143.misc b/changelog.d/11143.misc
deleted file mode 100644
index 496e44a9c0..0000000000
--- a/changelog.d/11143.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room.
\ No newline at end of file
diff --git a/changelog.d/11144.misc b/changelog.d/11144.misc
deleted file mode 100644
index b5db109e2b..0000000000
--- a/changelog.d/11144.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix broken link in the docker image README.
diff --git a/changelog.d/11145.bugfix b/changelog.d/11145.bugfix
deleted file mode 100644
index f369feac42..0000000000
--- a/changelog.d/11145.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse v1.45.0 breaking the configuration file parsing script.
diff --git a/changelog.d/11146.misc b/changelog.d/11146.misc
deleted file mode 100644
index 6ce1c9f9f5..0000000000
--- a/changelog.d/11146.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add missing type hints to `synapse.crypto`.
diff --git a/changelog.d/11174.feature b/changelog.d/11174.feature
deleted file mode 100644
index 8eecd92681..0000000000
--- a/changelog.d/11174.feature
+++ /dev/null
@@ -1 +0,0 @@
-Users admin API can now also modify user type in addition to allowing it to be set on user creation.
diff --git a/changelog.d/11177.bugfix b/changelog.d/11177.bugfix
deleted file mode 100644
index ca5bc0df28..0000000000
--- a/changelog.d/11177.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a performance regression introduced in v1.44.0 which could cause client requests to time out when making large numbers of outbound requests.
diff --git a/changelog.d/11180.feature b/changelog.d/11180.feature
deleted file mode 100644
index 82c40bf1b2..0000000000
--- a/changelog.d/11180.feature
+++ /dev/null
@@ -1 +0,0 @@
-Port the Password Auth Providers module interface to the new generic interface.
diff --git a/changelog.d/11181.feature b/changelog.d/11181.feature
deleted file mode 100644
index 76b0d28084..0000000000
--- a/changelog.d/11181.feature
+++ /dev/null
@@ -1 +0,0 @@
-Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
diff --git a/changelog.d/11183.doc b/changelog.d/11183.doc
deleted file mode 100644
index a171a107af..0000000000
--- a/changelog.d/11183.doc
+++ /dev/null
@@ -1 +0,0 @@
-Document the version of Synapse that introduced each module API method.
diff --git a/debian/changelog b/debian/changelog
index 1ee81f2a34..ea96676f74 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.46.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.46.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Oct 2021 14:04:04 +0100
+
 matrix-synapse-py3 (1.45.1) stable; urgency=medium
 
   * New synapse release 1.45.1.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 2687d932ea..355b36fc63 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.45.1"
+__version__ = "1.46.0rc1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when
-- 
cgit 1.5.1


From 34b0222c2b240ad2c1e4fdafe2428e7d74c496b0 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 26 Oct 2021 14:08:52 +0100
Subject: Update CHANGES.md

---
 CHANGES.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 88f8b5e01d..8ca7d28be2 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,6 +1,8 @@
 Synapse 1.46.0rc1 (2021-10-26)
 ==============================
 
+The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed.
+
 Features
 --------
 
@@ -24,8 +26,8 @@ Bugfixes
 - Fix local users who left all their rooms being removed from the user directory, even if the "search_all_users" config option was enabled. ([\#11103](https://github.com/matrix-org/synapse/issues/11103))
 - Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112))
 - Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120))
-- Fix a bug introduced in Synapse v1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145))
-- Fix a performance regression introduced in v1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
+- Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145))
+- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
 
 
 Improved Documentation
-- 
cgit 1.5.1


From c7a5e49664ab0bd18a57336e282fa6c3b9a17ca0 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Tue, 26 Oct 2021 15:17:36 +0200
Subject: Implement an `on_new_event` callback (#11126)

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
---
 changelog.d/11126.feature                   |  1 +
 docs/modules/third_party_rules_callbacks.md | 21 +++++++
 synapse/events/third_party_rules.py         | 31 ++++++++++
 synapse/handlers/federation_event.py        |  2 +-
 synapse/handlers/message.py                 |  9 ++-
 synapse/notifier.py                         | 17 ++++--
 synapse/replication/tcp/client.py           |  3 +-
 tests/rest/client/test_third_party_rules.py | 93 ++++++++++++++++++++++++++++-
 8 files changed, 165 insertions(+), 12 deletions(-)
 create mode 100644 changelog.d/11126.feature

diff --git a/changelog.d/11126.feature b/changelog.d/11126.feature
new file mode 100644
index 0000000000..c6078fe081
--- /dev/null
+++ b/changelog.d/11126.feature
@@ -0,0 +1 @@
+Add an `on_new_event` third-party rules callback to allow Synapse modules to act after an event has been sent into a room.
diff --git a/docs/modules/third_party_rules_callbacks.md b/docs/modules/third_party_rules_callbacks.md
index 034923da0f..a16e272f79 100644
--- a/docs/modules/third_party_rules_callbacks.md
+++ b/docs/modules/third_party_rules_callbacks.md
@@ -119,6 +119,27 @@ callback returns `True`, Synapse falls through to the next one. The value of the
 callback that does not return `True` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.
 
+### `on_new_event`
+
+_First introduced in Synapse v1.47.0_
+
+```python
+async def on_new_event(
+    event: "synapse.events.EventBase",
+    state_events: "synapse.types.StateMap",
+) -> None:
+```
+
+Called after sending an event into a room. The module is passed the event, as well
+as the state of the room _after_ the event. This means that if the event is a state event,
+it will be included in this state.
+
+Note that this callback is called when the event has already been processed and stored
+into the room, which means this callback cannot be used to deny persisting the event. To
+deny an incoming event, see [`check_event_for_spam`](spam_checker_callbacks.md#check_event_for_spam) instead.
+
+If multiple modules implement this callback, Synapse runs them all in order.
+
 ## Example
 
 The example below is a module that implements the third-party rules callback
diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py
index 2a6dabdab6..8816ef4b76 100644
--- a/synapse/events/third_party_rules.py
+++ b/synapse/events/third_party_rules.py
@@ -36,6 +36,7 @@ CHECK_THREEPID_CAN_BE_INVITED_CALLBACK = Callable[
 CHECK_VISIBILITY_CAN_BE_MODIFIED_CALLBACK = Callable[
     [str, StateMap[EventBase], str], Awaitable[bool]
 ]
+ON_NEW_EVENT_CALLBACK = Callable[[EventBase, StateMap[EventBase]], Awaitable]
 
 
 def load_legacy_third_party_event_rules(hs: "HomeServer") -> None:
@@ -152,6 +153,7 @@ class ThirdPartyEventRules:
         self._check_visibility_can_be_modified_callbacks: List[
             CHECK_VISIBILITY_CAN_BE_MODIFIED_CALLBACK
         ] = []
+        self._on_new_event_callbacks: List[ON_NEW_EVENT_CALLBACK] = []
 
     def register_third_party_rules_callbacks(
         self,
@@ -163,6 +165,7 @@ class ThirdPartyEventRules:
         check_visibility_can_be_modified: Optional[
             CHECK_VISIBILITY_CAN_BE_MODIFIED_CALLBACK
         ] = None,
+        on_new_event: Optional[ON_NEW_EVENT_CALLBACK] = None,
     ) -> None:
         """Register callbacks from modules for each hook."""
         if check_event_allowed is not None:
@@ -181,6 +184,9 @@ class ThirdPartyEventRules:
                 check_visibility_can_be_modified,
             )
 
+        if on_new_event is not None:
+            self._on_new_event_callbacks.append(on_new_event)
+
     async def check_event_allowed(
         self, event: EventBase, context: EventContext
     ) -> Tuple[bool, Optional[dict]]:
@@ -321,6 +327,31 @@ class ThirdPartyEventRules:
 
         return True
 
+    async def on_new_event(self, event_id: str) -> None:
+        """Let modules act on events after they've been sent (e.g. auto-accepting
+        invites, etc.)
+
+        Args:
+            event_id: The ID of the event.
+
+        Raises:
+            ModuleFailureError if a callback raised any exception.
+        """
+        # Bail out early without hitting the store if we don't have any callbacks
+        if len(self._on_new_event_callbacks) == 0:
+            return
+
+        event = await self.store.get_event(event_id)
+        state_events = await self._get_state_map_for_room(event.room_id)
+
+        for callback in self._on_new_event_callbacks:
+            try:
+                await callback(event, state_events)
+            except Exception as e:
+                logger.exception(
+                    "Failed to run module API callback %s: %s", callback, e
+                )
+
     async def _get_state_map_for_room(self, room_id: str) -> StateMap[EventBase]:
         """Given a room ID, return the state events of that room.
 
diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index 9584d5bd46..bd1fa08cef 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -1916,7 +1916,7 @@ class FederationEventHandler:
         event_pos = PersistedEventPosition(
             self._instance_name, event.internal_metadata.stream_ordering
         )
-        self._notifier.on_new_room_event(
+        await self._notifier.on_new_room_event(
             event, event_pos, max_stream_token, extra_users=extra_users
         )
 
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 2e024b551f..4a0fccfcc6 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -1537,13 +1537,16 @@ class EventCreationHandler:
             # If there's an expiry timestamp on the event, schedule its expiry.
             self._message_handler.maybe_schedule_expiry(event)
 
-        def _notify() -> None:
+        async def _notify() -> None:
             try:
-                self.notifier.on_new_room_event(
+                await self.notifier.on_new_room_event(
                     event, event_pos, max_stream_token, extra_users=extra_users
                 )
             except Exception:
-                logger.exception("Error notifying about new room event")
+                logger.exception(
+                    "Error notifying about new room event %s",
+                    event.event_id,
+                )
 
         run_in_background(_notify)
 
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1acd899fab..1882fffd2a 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -220,6 +220,8 @@ class Notifier:
         # down.
         self.remote_server_up_callbacks: List[Callable[[str], None]] = []
 
+        self._third_party_rules = hs.get_third_party_event_rules()
+
         self.clock = hs.get_clock()
         self.appservice_handler = hs.get_application_service_handler()
         self._pusher_pool = hs.get_pusherpool()
@@ -267,7 +269,7 @@ class Notifier:
         """
         self.replication_callbacks.append(cb)
 
-    def on_new_room_event(
+    async def on_new_room_event(
         self,
         event: EventBase,
         event_pos: PersistedEventPosition,
@@ -275,9 +277,10 @@ class Notifier:
         extra_users: Optional[Collection[UserID]] = None,
     ):
         """Unwraps event and calls `on_new_room_event_args`."""
-        self.on_new_room_event_args(
+        await self.on_new_room_event_args(
             event_pos=event_pos,
             room_id=event.room_id,
+            event_id=event.event_id,
             event_type=event.type,
             state_key=event.get("state_key"),
             membership=event.content.get("membership"),
@@ -285,9 +288,10 @@ class Notifier:
             extra_users=extra_users or [],
         )
 
-    def on_new_room_event_args(
+    async def on_new_room_event_args(
         self,
         room_id: str,
+        event_id: str,
         event_type: str,
         state_key: Optional[str],
         membership: Optional[str],
@@ -302,7 +306,10 @@ class Notifier:
         listening to the room, and any listeners for the users in the
         `extra_users` param.
 
-        The events can be peristed out of order. The notifier will wait
+        This also notifies modules listening on new events via the
+        `on_new_event` callback.
+
+        The events can be persisted out of order. The notifier will wait
         until all previous events have been persisted before notifying
         the client streams.
         """
@@ -318,6 +325,8 @@ class Notifier:
         )
         self._notify_pending_new_room_events(max_room_stream_token)
 
+        await self._third_party_rules.on_new_event(event_id)
+
         self.notify_replication()
 
     def _notify_pending_new_room_events(self, max_room_stream_token: RoomStreamToken):
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index 961c17762e..e29ae1e375 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -207,11 +207,12 @@ class ReplicationDataHandler:
 
                 max_token = self.store.get_room_max_token()
                 event_pos = PersistedEventPosition(instance_name, token)
-                self.notifier.on_new_room_event_args(
+                await self.notifier.on_new_room_event_args(
                     event_pos=event_pos,
                     max_room_stream_token=max_token,
                     extra_users=extra_users,
                     room_id=row.data.room_id,
+                    event_id=row.data.event_id,
                     event_type=row.data.type,
                     state_key=row.data.state_key,
                     membership=row.data.membership,
diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py
index 531f09c48b..1c42c46630 100644
--- a/tests/rest/client/test_third_party_rules.py
+++ b/tests/rest/client/test_third_party_rules.py
@@ -15,7 +15,7 @@ import threading
 from typing import TYPE_CHECKING, Dict, Optional, Tuple
 from unittest.mock import Mock
 
-from synapse.api.constants import EventTypes
+from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import SynapseError
 from synapse.events import EventBase
 from synapse.events.third_party_rules import load_legacy_third_party_event_rules
@@ -25,6 +25,7 @@ from synapse.types import JsonDict, Requester, StateMap
 from synapse.util.frozenutils import unfreeze
 
 from tests import unittest
+from tests.test_utils import make_awaitable
 
 if TYPE_CHECKING:
     from synapse.module_api import ModuleApi
@@ -74,7 +75,7 @@ class LegacyChangeEvents(LegacyThirdPartyRulesTestModule):
         return d
 
 
-class ThirdPartyRulesTestCase(unittest.HomeserverTestCase):
+class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase):
     servlets = [
         admin.register_servlets,
         login.register_servlets,
@@ -86,11 +87,29 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase):
 
         load_legacy_third_party_event_rules(hs)
 
+        # We're not going to be properly signing events as our remote homeserver is fake,
+        # therefore disable event signature checks.
+        # Note that these checks are not relevant to this test case.
+
+        # Have this homeserver auto-approve all event signature checking.
+        async def approve_all_signature_checking(_, pdu):
+            return pdu
+
+        hs.get_federation_server()._check_sigs_and_hash = approve_all_signature_checking
+
+        # Have this homeserver skip event auth checks. This is necessary due to
+        # event auth checks ensuring that events were signed by the sender's homeserver.
+        async def _check_event_auth(origin, event, context, *args, **kwargs):
+            return context
+
+        hs.get_federation_event_handler()._check_event_auth = _check_event_auth
+
         return hs
 
     def prepare(self, reactor, clock, homeserver):
-        # Create a user and room to play with during the tests
+        # Create some users and a room to play with during the tests
         self.user_id = self.register_user("kermit", "monkey")
+        self.invitee = self.register_user("invitee", "hackme")
         self.tok = self.login("kermit", "monkey")
 
         # Some tests might prevent room creation on purpose.
@@ -424,6 +443,74 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase):
             self.assertEqual(channel.code, 200)
             self.assertEqual(channel.json_body["i"], i)
 
+    def test_on_new_event(self):
+        """Test that the on_new_event callback is called on new events"""
+        on_new_event = Mock(make_awaitable(None))
+        self.hs.get_third_party_event_rules()._on_new_event_callbacks.append(
+            on_new_event
+        )
+
+        # Send a message event to the room and check that the callback is called.
+        self.helper.send(room_id=self.room_id, tok=self.tok)
+        self.assertEqual(on_new_event.call_count, 1)
+
+        # Check that the callback is also called on membership updates.
+        self.helper.invite(
+            room=self.room_id,
+            src=self.user_id,
+            targ=self.invitee,
+            tok=self.tok,
+        )
+
+        self.assertEqual(on_new_event.call_count, 2)
+
+        args, _ = on_new_event.call_args
+
+        self.assertEqual(args[0].membership, Membership.INVITE)
+        self.assertEqual(args[0].state_key, self.invitee)
+
+        # Check that the invitee's membership is correct in the state that's passed down
+        # to the callback.
+        self.assertEqual(
+            args[1][(EventTypes.Member, self.invitee)].membership,
+            Membership.INVITE,
+        )
+
+        # Send an event over federation and check that the callback is also called.
+        self._send_event_over_federation()
+        self.assertEqual(on_new_event.call_count, 3)
+
+    def _send_event_over_federation(self) -> None:
+        """Send a dummy event over federation and check that the request succeeds."""
+        body = {
+            "origin": self.hs.config.server.server_name,
+            "origin_server_ts": self.clock.time_msec(),
+            "pdus": [
+                {
+                    "sender": self.user_id,
+                    "type": EventTypes.Message,
+                    "state_key": "",
+                    "content": {"body": "hello world", "msgtype": "m.text"},
+                    "room_id": self.room_id,
+                    "depth": 0,
+                    "origin_server_ts": self.clock.time_msec(),
+                    "prev_events": [],
+                    "auth_events": [],
+                    "signatures": {},
+                    "unsigned": {},
+                }
+            ],
+        }
+
+        channel = self.make_request(
+            method="PUT",
+            path="/_matrix/federation/v1/send/1",
+            content=body,
+            federation_auth_origin=self.hs.config.server.server_name.encode("utf8"),
+        )
+
+        self.assertEqual(channel.code, 200, channel.result)
+
     def _update_power_levels(self, event_default: int = 0):
         """Updates the room's power levels.
 
-- 
cgit 1.5.1


From 400f391f71e252d694fe97fffb35751910e55376 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 26 Oct 2021 14:21:09 +0100
Subject: Update CHANGES.md

---
 CHANGES.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 8ca7d28be2..7b67495154 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,7 @@
 Synapse 1.46.0rc1 (2021-10-26)
 ==============================
 
-The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed.
+The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
 
 Features
 --------
@@ -18,12 +18,12 @@ Bugfixes
 - Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr. ([\#10930](https://github.com/matrix-org/synapse/issues/10930))
 - Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#11001](https://github.com/matrix-org/synapse/issues/11001), [\#11009](https://github.com/matrix-org/synapse/issues/11009), [\#11012](https://github.com/matrix-org/synapse/issues/11012))
 - Fix 500 error on `/messages` when the server accumulates more than 5 backwards extremities at a given depth for a room. ([\#11027](https://github.com/matrix-org/synapse/issues/11027))
-- Fix a bug where setting a user's external_id via the admin API returns 500 and deletes users existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051))
+- Fix a bug where setting a user's `external_id` via the admin API returns 500 and deletes users existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051))
 - Fix a long-standing bug where users excluded from the user directory were added into the directory if they belonged to a room which became public or private. ([\#11075](https://github.com/matrix-org/synapse/issues/11075))
 - Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding. ([\#11077](https://github.com/matrix-org/synapse/issues/11077), [\#11089](https://github.com/matrix-org/synapse/issues/11089))
 - Fix broken export-data admin command and add test script checking the command to CI. ([\#11078](https://github.com/matrix-org/synapse/issues/11078))
 - Show an error when timestamp in seconds is provided to the `/purge_media_cache` Admin API. ([\#11101](https://github.com/matrix-org/synapse/issues/11101))
-- Fix local users who left all their rooms being removed from the user directory, even if the "search_all_users" config option was enabled. ([\#11103](https://github.com/matrix-org/synapse/issues/11103))
+- Fix local users who left all their rooms being removed from the user directory, even if the `search_all_users` config option was enabled. ([\#11103](https://github.com/matrix-org/synapse/issues/11103))
 - Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112))
 - Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120))
 - Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145))
-- 
cgit 1.5.1


From 60ad9460c4728f2009c9d21715d33e1b4e4b3925 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 26 Oct 2021 14:26:36 +0100
Subject: Move #10975 to bugfix section in changelog

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 7b67495154..d0c012d17b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -7,7 +7,6 @@ Features
 --------
 
 - Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180))
-- Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975))
 - Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181))
 - Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174))
 
@@ -28,6 +27,7 @@ Bugfixes
 - Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120))
 - Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145))
 - Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
+- Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975))
 
 
 Improved Documentation
-- 
cgit 1.5.1


From 2dbef6c10af521fe4f77105281726b40941c6810 Mon Sep 17 00:00:00 2001
From: Sean Quah <seanq@element.io>
Date: Tue, 26 Oct 2021 17:54:25 +0100
Subject: Update CHANGES.md

---
 CHANGES.md | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index d0c012d17b..56cd04756e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,6 +6,7 @@ The cause of the [performance regression affecting Synapse 1.44](https://github.
 Features
 --------
 
+- Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024))
 - Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180))
 - Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181))
 - Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174))
@@ -17,7 +18,7 @@ Bugfixes
 - Newly-created public rooms are now only assigned an alias if the room's creation has not been blocked by permission settings. Contributed by @AndrewFerr. ([\#10930](https://github.com/matrix-org/synapse/issues/10930))
 - Fix a long-standing bug which meant that events received over federation were sometimes incorrectly accepted into the room state. ([\#11001](https://github.com/matrix-org/synapse/issues/11001), [\#11009](https://github.com/matrix-org/synapse/issues/11009), [\#11012](https://github.com/matrix-org/synapse/issues/11012))
 - Fix 500 error on `/messages` when the server accumulates more than 5 backwards extremities at a given depth for a room. ([\#11027](https://github.com/matrix-org/synapse/issues/11027))
-- Fix a bug where setting a user's `external_id` via the admin API returns 500 and deletes users existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051))
+- Fix a bug where setting a user's `external_id` via the admin API returns 500 and deletes user's existing external mappings if that external ID is already mapped. ([\#11051](https://github.com/matrix-org/synapse/issues/11051))
 - Fix a long-standing bug where users excluded from the user directory were added into the directory if they belonged to a room which became public or private. ([\#11075](https://github.com/matrix-org/synapse/issues/11075))
 - Fix a long-standing bug when attempting to preview URLs which are in the `windows-1252` character encoding. ([\#11077](https://github.com/matrix-org/synapse/issues/11077), [\#11089](https://github.com/matrix-org/synapse/issues/11089))
 - Fix broken export-data admin command and add test script checking the command to CI. ([\#11078](https://github.com/matrix-org/synapse/issues/11078))
@@ -45,33 +46,24 @@ Improved Documentation
 
 Internal Changes
 ----------------
-
-- Add type hints to `synapse.storage.databases.main.client_ips`. ([\#10972](https://github.com/matrix-org/synapse/issues/10972))
 - Fix spurious warnings about losing the logging context on the `ReplicationCommandHandler` when losing the replication connection. ([\#10984](https://github.com/matrix-org/synapse/issues/10984))
 - Include rejected status when we log events. ([\#11008](https://github.com/matrix-org/synapse/issues/11008))
 - Add some extra logging to the event persistence code. ([\#11014](https://github.com/matrix-org/synapse/issues/11014))
-- Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024))
 - Rearrange the internal workings of the incremental user directory updates. ([\#11035](https://github.com/matrix-org/synapse/issues/11035))
-- Simplify the user admin API tests. ([\#11048](https://github.com/matrix-org/synapse/issues/11048))
+- Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room. ([\#11143](https://github.com/matrix-org/synapse/issues/11143))
+- Add and improve type hints. ([\#10972](https://github.com/matrix-org/synapse/issues/10972), [\#11055](https://github.com/matrix-org/synapse/issues/11055), [\#11066](https://github.com/matrix-org/synapse/issues/11066), [\#11076](https://github.com/matrix-org/synapse/issues/11076), [\#11095](https://github.com/matrix-org/synapse/issues/11095), [\#11109](https://github.com/matrix-org/synapse/issues/11109), [\#11121](https://github.com/matrix-org/synapse/issues/11121), [\#11146](https://github.com/matrix-org/synapse/issues/11146))
 - Mark the Synapse package as containing type annotations and fix export declarations so that Synapse pluggable modules may be type checked against Synapse. ([\#11054](https://github.com/matrix-org/synapse/issues/11054))
-- Improve type hints for `_wrap_in_base_path` decorator used by `MediaFilePaths`. ([\#11055](https://github.com/matrix-org/synapse/issues/11055))
 - Remove dead code from `MediaFilePaths`. ([\#11056](https://github.com/matrix-org/synapse/issues/11056))
-- Add tests for `MediaFilePaths` class. ([\#11057](https://github.com/matrix-org/synapse/issues/11057))
 - Be more lenient when parsing oEmbed response versions. ([\#11065](https://github.com/matrix-org/synapse/issues/11065))
-- Add type hints to `synapse.events`. ([\#11066](https://github.com/matrix-org/synapse/issues/11066))
-- Always dump logs from unit tests during CI runs. ([\#11068](https://github.com/matrix-org/synapse/issues/11068))
 - Create a separate module for the retention configuration. ([\#11070](https://github.com/matrix-org/synapse/issues/11070))
-- Add a test for the workaround introduced in [\#11042](https://github.com/matrix-org/synapse/pull/11042) concerning the behaviour of third-party rule modules and `SynapseError`s. ([\#11071](https://github.com/matrix-org/synapse/issues/11071))
-- Fix type hints in the relations tests. ([\#11076](https://github.com/matrix-org/synapse/issues/11076))
-- Add type hints to most `HomeServer` parameters. ([\#11095](https://github.com/matrix-org/synapse/issues/11095))
-- Add missing type hints to `synapse.api` module. ([\#11109](https://github.com/matrix-org/synapse/issues/11109))
 - Clean up some of the federation event authentication code for clarity. ([\#11115](https://github.com/matrix-org/synapse/issues/11115), [\#11116](https://github.com/matrix-org/synapse/issues/11116), [\#11122](https://github.com/matrix-org/synapse/issues/11122))
-- Add type hints for event fetching. ([\#11121](https://github.com/matrix-org/synapse/issues/11121))
 - Add docstrings and comments to the application service ephemeral event sending code. ([\#11138](https://github.com/matrix-org/synapse/issues/11138))
 - Update the `sign_json` script to support inline configuration of the signing key. ([\#11139](https://github.com/matrix-org/synapse/issues/11139))
-- Fix a long-standing bug where users excluded from the directory could still be added to the `users_who_share_private_rooms` table after a regular user joins a private room. ([\#11143](https://github.com/matrix-org/synapse/issues/11143))
 - Fix broken link in the docker image README. ([\#11144](https://github.com/matrix-org/synapse/issues/11144))
-- Add missing type hints to `synapse.crypto`. ([\#11146](https://github.com/matrix-org/synapse/issues/11146))
+- Always dump logs from unit tests during CI runs. ([\#11068](https://github.com/matrix-org/synapse/issues/11068))
+- Add tests for `MediaFilePaths` class. ([\#11057](https://github.com/matrix-org/synapse/issues/11057))
+- Simplify the user admin API tests. ([\#11048](https://github.com/matrix-org/synapse/issues/11048))
+- Add a test for the workaround introduced in [\#11042](https://github.com/matrix-org/synapse/pull/11042) concerning the behaviour of third-party rule modules and `SynapseError`s. ([\#11071](https://github.com/matrix-org/synapse/issues/11071))
 
 
 Synapse 1.45.1 (2021-10-20)
-- 
cgit 1.5.1


From 72626b78ef4aa9ab0bd11e332495f34bd43bbc26 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 27 Oct 2021 12:33:21 +0100
Subject: Fix thread BG update to not seq scan event_json (#11192)

For some reason the query optimiser decided to seq scan both tables,
rather than index scanning `event_json`.
---
 changelog.d/11192.feature                           | 1 +
 synapse/storage/databases/main/events_bg_updates.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11192.feature

diff --git a/changelog.d/11192.feature b/changelog.d/11192.feature
new file mode 100644
index 0000000000..76b0d28084
--- /dev/null
+++ b/changelog.d/11192.feature
@@ -0,0 +1 @@
+Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
index f92d824876..ae3a8a63e4 100644
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -1108,7 +1108,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
                 """
                 SELECT event_id, json FROM event_json
                 LEFT JOIN event_relations USING (event_id)
-                WHERE event_id > ? AND relates_to_id IS NULL
+                WHERE event_id > ? AND event_relations.event_id IS NULL
                 ORDER BY event_id LIMIT ?
                 """,
                 (last_event_id, batch_size),
-- 
cgit 1.5.1


From f3a4be870092e28531957702931c5d74b9d1f18f Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Wed, 27 Oct 2021 13:04:56 +0100
Subject: Shut down the DNS threadpool (#11190)

The DNS threadpool must be explicitly stopped, otherwise Synapse will
hang indefinitely when asked to shut down.
---
 changelog.d/11190.bugfix | 1 +
 synapse/app/_base.py     | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog.d/11190.bugfix

diff --git a/changelog.d/11190.bugfix b/changelog.d/11190.bugfix
new file mode 100644
index 0000000000..0d913805ac
--- /dev/null
+++ b/changelog.d/11190.bugfix
@@ -0,0 +1 @@
+Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 03627cdcba..f4c3f867a8 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -346,6 +346,7 @@ async def start(hs: "HomeServer"):
     # numbers of DNS requests don't starve out other users of the threadpool.
     resolver_threadpool = ThreadPool(name="gai_resolver")
     resolver_threadpool.start()
+    reactor.addSystemEventTrigger("during", "shutdown", resolver_threadpool.stop)
     reactor.installNameResolver(
         GAIResolver(reactor, getThreadPool=lambda: resolver_threadpool)
     )
-- 
cgit 1.5.1


From b615fc35d6a425f0241f1e9cf92886028ec36a72 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 27 Oct 2021 13:22:48 +0100
Subject: Update changelog with new changes

---
 CHANGES.md                | 4 ++--
 changelog.d/11190.bugfix  | 1 -
 changelog.d/11192.feature | 1 -
 3 files changed, 2 insertions(+), 4 deletions(-)
 delete mode 100644 changelog.d/11190.bugfix
 delete mode 100644 changelog.d/11192.feature

diff --git a/CHANGES.md b/CHANGES.md
index 56cd04756e..a58ab7c250 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,7 +8,7 @@ Features
 
 - Add support for Ubuntu 21.10 "Impish Indri". ([\#11024](https://github.com/matrix-org/synapse/issues/11024))
 - Port the Password Auth Providers module interface to the new generic interface. ([\#10548](https://github.com/matrix-org/synapse/issues/10548), [\#11180](https://github.com/matrix-org/synapse/issues/11180))
-- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181))
+- Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11088](https://github.com/matrix-org/synapse/issues/11088), [\#11181](https://github.com/matrix-org/synapse/issues/11181), [\#11192](https://github.com/matrix-org/synapse/issues/11192))
 - Users admin API can now also modify user type in addition to allowing it to be set on user creation. ([\#11174](https://github.com/matrix-org/synapse/issues/11174))
 
 
@@ -27,7 +27,7 @@ Bugfixes
 - Fix a bug which caused the module API's `get_user_ip_and_agents` function to always fail on workers. `get_user_ip_and_agents` was introduced in 1.44.0 and did not function correctly on worker processes at the time. ([\#11112](https://github.com/matrix-org/synapse/issues/11112))
 - Identity server connection is no longer ignoring `ip_range_whitelist`. ([\#11120](https://github.com/matrix-org/synapse/issues/11120))
 - Fix a bug introduced in Synapse 1.45.0 breaking the configuration file parsing script. ([\#11145](https://github.com/matrix-org/synapse/issues/11145))
-- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
+- Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests. ([\#11177](https://github.com/matrix-org/synapse/issues/11177), [\#11190](https://github.com/matrix-org/synapse/issues/11190))
 - Resolve and share `state_groups` for all [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical events in batch. ([\#10975](https://github.com/matrix-org/synapse/issues/10975))
 
 
diff --git a/changelog.d/11190.bugfix b/changelog.d/11190.bugfix
deleted file mode 100644
index 0d913805ac..0000000000
--- a/changelog.d/11190.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a performance regression introduced in 1.44.0 which could cause client requests to time out when making large numbers of outbound requests.
diff --git a/changelog.d/11192.feature b/changelog.d/11192.feature
deleted file mode 100644
index 76b0d28084..0000000000
--- a/changelog.d/11192.feature
+++ /dev/null
@@ -1 +0,0 @@
-Experimental support for the thread relation defined in [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
-- 
cgit 1.5.1


From e0ef8fe58d8592cedf635882590324183083b76e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 27 Oct 2021 14:44:45 +0100
Subject: Update release date

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index a58ab7c250..f61d5c706f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,4 +1,4 @@
-Synapse 1.46.0rc1 (2021-10-26)
+Synapse 1.46.0rc1 (2021-10-27)
 ==============================
 
 The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
-- 
cgit 1.5.1


From a930da3291b5b1d2375c3bd7c4a34f1588704292 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Wed, 27 Oct 2021 10:19:19 -0400
Subject: Include the stable identifier for MSC3288. (#11187)

Includes both the stable and unstable identifier to store-invite
calls to the identity server. In the future we should remove the
unstable identifier.
---
 changelog.d/11187.feature    | 1 +
 synapse/handlers/identity.py | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 changelog.d/11187.feature

diff --git a/changelog.d/11187.feature b/changelog.d/11187.feature
new file mode 100644
index 0000000000..dd28109030
--- /dev/null
+++ b/changelog.d/11187.feature
@@ -0,0 +1 @@
+Support the stable room type field for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288).
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 7ef8698a5e..6a315117ba 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -879,6 +879,8 @@ class IdentityHandler:
         }
 
         if room_type is not None:
+            invite_config["room_type"] = room_type
+            # TODO The unstable field is deprecated and should be removed in the future.
             invite_config["org.matrix.msc3288.room_type"] = room_type
 
         # If a custom web client location is available, include it in the request.
-- 
cgit 1.5.1


From b3e843be88d67633d11711ecc80d4e0390b1e723 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Wed, 27 Oct 2021 10:48:02 -0400
Subject: Fix URL preview errors when previewing XML documents. (#11196)

---
 changelog.d/11196.bugfix                      |  1 +
 synapse/rest/media/v1/preview_url_resource.py |  9 ++++++---
 tests/test_preview.py                         | 15 +++++++++++++++
 3 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/11196.bugfix

diff --git a/changelog.d/11196.bugfix b/changelog.d/11196.bugfix
new file mode 100644
index 0000000000..3861eeb908
--- /dev/null
+++ b/changelog.d/11196.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in v1.46.0rc1 where URL previews of some XML documents would fail.
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index 278fd901e2..8ca97b5b18 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -718,9 +718,12 @@ def decode_body(
     if not body:
         return None
 
+    # The idea here is that multiple encodings are tried until one works.
+    # Unfortunately the result is never used and then LXML will decode the string
+    # again with the found encoding.
     for encoding in get_html_media_encodings(body, content_type):
         try:
-            body_str = body.decode(encoding)
+            body.decode(encoding)
         except Exception:
             pass
         else:
@@ -732,11 +735,11 @@ def decode_body(
     from lxml import etree
 
     # Create an HTML parser.
-    parser = etree.HTMLParser(recover=True, encoding="utf-8")
+    parser = etree.HTMLParser(recover=True, encoding=encoding)
 
     # Attempt to parse the body. Returns None if the body was successfully
     # parsed, but no tree was found.
-    return etree.fromstring(body_str, parser)
+    return etree.fromstring(body, parser)
 
 
 def _calc_og(tree: "etree.Element", media_uri: str) -> Dict[str, Optional[str]]:
diff --git a/tests/test_preview.py b/tests/test_preview.py
index 9a576f9a4e..40b89fb2ef 100644
--- a/tests/test_preview.py
+++ b/tests/test_preview.py
@@ -277,6 +277,21 @@ class CalcOgTestCase(unittest.TestCase):
         tree = decode_body(html, "http://example.com/test.html")
         self.assertIsNone(tree)
 
+    def test_xml(self):
+        """Test decoding XML and ensure it works properly."""
+        # Note that the strip() call is important to ensure the xml tag starts
+        # at the initial byte.
+        html = b"""
+        <?xml version="1.0" encoding="UTF-8"?>
+
+        <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+        <head><title>Foo</title></head><body>Some text.</body></html>
+        """.strip()
+        tree = decode_body(html, "http://example.com/test.html")
+        og = _calc_og(tree, "http://example.com/test.html")
+        self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."})
+
     def test_invalid_encoding(self):
         """An invalid character encoding should be ignored and treated as UTF-8, if possible."""
         html = b"""
-- 
cgit 1.5.1


From 8d46fac98e07ac319c7ae21dfc24502993de3f1d Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Wed, 27 Oct 2021 17:01:18 +0200
Subject: Delete messages from `device_inbox` table when deleting device
 (#10969)

Fixes: #9346
---
 changelog.d/10969.bugfix                           |  1 +
 synapse/storage/databases/main/deviceinbox.py      | 92 +++++++++++++++++++++-
 synapse/storage/databases/main/devices.py          | 35 ++++----
 .../02remove_deleted_devices_from_device_inbox.sql | 22 ++++++
 tests/handlers/test_device.py                      | 31 ++++++++
 tests/storage/databases/main/test_deviceinbox.py   | 90 +++++++++++++++++++++
 6 files changed, 256 insertions(+), 15 deletions(-)
 create mode 100644 changelog.d/10969.bugfix
 create mode 100644 synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql
 create mode 100644 tests/storage/databases/main/test_deviceinbox.py

diff --git a/changelog.d/10969.bugfix b/changelog.d/10969.bugfix
new file mode 100644
index 0000000000..89c299b8e8
--- /dev/null
+++ b/changelog.d/10969.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where messages in the `device_inbox` table for deleted devices would persist indefinitely. Contributed by @dklimpel and @JohannesKleine.
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 8143168107..b0ccab0c9b 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -19,9 +19,10 @@ from synapse.logging import issue9533_logger
 from synapse.logging.opentracing import log_kv, set_tag, trace
 from synapse.replication.tcp.streams import ToDeviceStream
 from synapse.storage._base import SQLBaseStore, db_to_json
-from synapse.storage.database import DatabasePool
+from synapse.storage.database import DatabasePool, LoggingTransaction
 from synapse.storage.engines import PostgresEngine
 from synapse.storage.util.id_generators import MultiWriterIdGenerator, StreamIdGenerator
+from synapse.types import JsonDict
 from synapse.util import json_encoder
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.caches.stream_change_cache import StreamChangeCache
@@ -555,6 +556,7 @@ class DeviceInboxWorkerStore(SQLBaseStore):
 
 class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
     DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
+    REMOVE_DELETED_DEVICES = "remove_deleted_devices_from_device_inbox"
 
     def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
@@ -570,6 +572,11 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
             self.DEVICE_INBOX_STREAM_ID, self._background_drop_index_device_inbox
         )
 
+        self.db_pool.updates.register_background_update_handler(
+            self.REMOVE_DELETED_DEVICES,
+            self._remove_deleted_devices_from_device_inbox,
+        )
+
     async def _background_drop_index_device_inbox(self, progress, batch_size):
         def reindex_txn(conn):
             txn = conn.cursor()
@@ -582,6 +589,89 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
 
         return 1
 
+    async def _remove_deleted_devices_from_device_inbox(
+        self, progress: JsonDict, batch_size: int
+    ) -> int:
+        """A background update that deletes all device_inboxes for deleted devices.
+
+        This should only need to be run once (when users upgrade to v1.46.0)
+
+        Args:
+            progress: JsonDict used to store progress of this background update
+            batch_size: the maximum number of rows to retrieve in a single select query
+
+        Returns:
+            The number of deleted rows
+        """
+
+        def _remove_deleted_devices_from_device_inbox_txn(
+            txn: LoggingTransaction,
+        ) -> int:
+            """stream_id is not unique
+            we need to use an inclusive `stream_id >= ?` clause,
+            since we might not have deleted all dead device messages for the stream_id
+            returned from the previous query
+
+            Then delete only rows matching the `(user_id, device_id, stream_id)` tuple,
+            to avoid problems of deleting a large number of rows all at once
+            due to a single device having lots of device messages.
+            """
+
+            last_stream_id = progress.get("stream_id", 0)
+
+            sql = """
+                SELECT device_id, user_id, stream_id
+                FROM device_inbox
+                WHERE
+                    stream_id >= ?
+                    AND (device_id, user_id) NOT IN (
+                        SELECT device_id, user_id FROM devices
+                    )
+                ORDER BY stream_id
+                LIMIT ?
+            """
+
+            txn.execute(sql, (last_stream_id, batch_size))
+            rows = txn.fetchall()
+
+            num_deleted = 0
+            for row in rows:
+                num_deleted += self.db_pool.simple_delete_txn(
+                    txn,
+                    "device_inbox",
+                    {"device_id": row[0], "user_id": row[1], "stream_id": row[2]},
+                )
+
+            if rows:
+                # send more than stream_id to progress
+                # otherwise it can happen in large deployments that
+                # no change of status is visible in the log file
+                # it may be that the stream_id does not change in several runs
+                self.db_pool.updates._background_update_progress_txn(
+                    txn,
+                    self.REMOVE_DELETED_DEVICES,
+                    {
+                        "device_id": rows[-1][0],
+                        "user_id": rows[-1][1],
+                        "stream_id": rows[-1][2],
+                    },
+                )
+
+            return num_deleted
+
+        number_deleted = await self.db_pool.runInteraction(
+            "_remove_deleted_devices_from_device_inbox",
+            _remove_deleted_devices_from_device_inbox_txn,
+        )
+
+        # The task is finished when no more lines are deleted.
+        if not number_deleted:
+            await self.db_pool.updates._end_background_update(
+                self.REMOVE_DELETED_DEVICES
+            )
+
+        return number_deleted
+
 
 class DeviceInboxStore(DeviceInboxWorkerStore, DeviceInboxBackgroundUpdateStore):
     pass
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index a01bf2c5b7..b15cd030e0 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -1134,19 +1134,14 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
             raise StoreError(500, "Problem storing device.")
 
     async def delete_device(self, user_id: str, device_id: str) -> None:
-        """Delete a device.
+        """Delete a device and its device_inbox.
 
         Args:
             user_id: The ID of the user which owns the device
             device_id: The ID of the device to delete
         """
-        await self.db_pool.simple_delete_one(
-            table="devices",
-            keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
-            desc="delete_device",
-        )
 
-        self.device_id_exists_cache.invalidate((user_id, device_id))
+        await self.delete_devices(user_id, [device_id])
 
     async def delete_devices(self, user_id: str, device_ids: List[str]) -> None:
         """Deletes several devices.
@@ -1155,13 +1150,25 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
             user_id: The ID of the user which owns the devices
             device_ids: The IDs of the devices to delete
         """
-        await self.db_pool.simple_delete_many(
-            table="devices",
-            column="device_id",
-            iterable=device_ids,
-            keyvalues={"user_id": user_id, "hidden": False},
-            desc="delete_devices",
-        )
+
+        def _delete_devices_txn(txn: LoggingTransaction) -> None:
+            self.db_pool.simple_delete_many_txn(
+                txn,
+                table="devices",
+                column="device_id",
+                values=device_ids,
+                keyvalues={"user_id": user_id, "hidden": False},
+            )
+
+            self.db_pool.simple_delete_many_txn(
+                txn,
+                table="device_inbox",
+                column="device_id",
+                values=device_ids,
+                keyvalues={"user_id": user_id},
+            )
+
+        await self.db_pool.runInteraction("delete_devices", _delete_devices_txn)
         for device_id in device_ids:
             self.device_id_exists_cache.invalidate((user_id, device_id))
 
diff --git a/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql b/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql
new file mode 100644
index 0000000000..efe702f621
--- /dev/null
+++ b/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql
@@ -0,0 +1,22 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- Remove messages from the device_inbox table which were orphaned
+-- when a device was deleted using Synapse earlier than 1.46.0.
+-- This runs as background task, but may take a bit to finish.
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (6402, 'remove_deleted_devices_from_device_inbox', '{}');
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index 3ac48e5e95..43031e07ea 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -160,6 +160,37 @@ class DeviceTestCase(unittest.HomeserverTestCase):
         # we'd like to check the access token was invalidated, but that's a
         # bit of a PITA.
 
+    def test_delete_device_and_device_inbox(self):
+        self._record_users()
+
+        # add an device_inbox
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "device_inbox",
+                {
+                    "user_id": user1,
+                    "device_id": "abc",
+                    "stream_id": 1,
+                    "message_json": "{}",
+                },
+            )
+        )
+
+        # delete the device
+        self.get_success(self.handler.delete_device(user1, "abc"))
+
+        # check that the device_inbox was deleted
+        res = self.get_success(
+            self.store.db_pool.simple_select_one(
+                table="device_inbox",
+                keyvalues={"user_id": user1, "device_id": "abc"},
+                retcols=("user_id", "device_id"),
+                allow_none=True,
+                desc="get_device_id_from_device_inbox",
+            )
+        )
+        self.assertIsNone(res)
+
     def test_update_device(self):
         self._record_users()
 
diff --git a/tests/storage/databases/main/test_deviceinbox.py b/tests/storage/databases/main/test_deviceinbox.py
new file mode 100644
index 0000000000..4cfd2677f7
--- /dev/null
+++ b/tests/storage/databases/main/test_deviceinbox.py
@@ -0,0 +1,90 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest import admin
+from synapse.rest.client import devices
+
+from tests.unittest import HomeserverTestCase
+
+
+class DeviceInboxBackgroundUpdateStoreTestCase(HomeserverTestCase):
+
+    servlets = [
+        admin.register_servlets,
+        devices.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.store = hs.get_datastore()
+        self.user_id = self.register_user("foo", "pass")
+
+    def test_background_remove_deleted_devices_from_device_inbox(self):
+        """Test that the background task to delete old device_inboxes works properly."""
+
+        # create a valid device
+        self.get_success(
+            self.store.store_device(self.user_id, "cur_device", "display_name")
+        )
+
+        # Add device_inbox to devices
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "device_inbox",
+                {
+                    "user_id": self.user_id,
+                    "device_id": "cur_device",
+                    "stream_id": 1,
+                    "message_json": "{}",
+                },
+            )
+        )
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "device_inbox",
+                {
+                    "user_id": self.user_id,
+                    "device_id": "old_device",
+                    "stream_id": 2,
+                    "message_json": "{}",
+                },
+            )
+        )
+
+        # Insert and run the background update.
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "background_updates",
+                {
+                    "update_name": "remove_deleted_devices_from_device_inbox",
+                    "progress_json": "{}",
+                },
+            )
+        )
+
+        # ... and tell the DataStore that it hasn't finished all updates yet
+        self.store.db_pool.updates._all_done = False
+
+        self.wait_for_background_updates()
+
+        # Make sure the background task deleted old device_inbox
+        res = self.get_success(
+            self.store.db_pool.simple_select_onecol(
+                table="device_inbox",
+                keyvalues={},
+                retcol="device_id",
+                desc="get_device_id_from_device_inbox",
+            )
+        )
+        self.assertEqual(1, len(res))
+        self.assertEqual(res[0], "cur_device")
-- 
cgit 1.5.1


From 19d5dc69316a28035caf6a6519ad8a116023de81 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Wed, 27 Oct 2021 11:26:30 -0400
Subject: Refactor `Filter` to handle fields according to data being filtered.
 (#11194)

This avoids filtering against fields which cannot exist on an
event source. E.g. presence updates don't have a room.
---
 changelog.d/11194.misc         |   1 +
 synapse/api/filtering.py       | 139 +++++++++++++++++++++++------------------
 synapse/handlers/pagination.py |   2 +-
 synapse/handlers/room.py       |   2 +-
 synapse/handlers/search.py     |  12 ++--
 5 files changed, 87 insertions(+), 69 deletions(-)
 create mode 100644 changelog.d/11194.misc

diff --git a/changelog.d/11194.misc b/changelog.d/11194.misc
new file mode 100644
index 0000000000..fc1d06ba89
--- /dev/null
+++ b/changelog.d/11194.misc
@@ -0,0 +1 @@
+Refactor `Filter` to check different fields depending on the data type.
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index bc550ae646..4b0a9b2974 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -18,7 +18,8 @@ import json
 from typing import (
     TYPE_CHECKING,
     Awaitable,
-    Container,
+    Callable,
+    Dict,
     Iterable,
     List,
     Optional,
@@ -217,19 +218,19 @@ class FilterCollection:
         return self._filter_json
 
     def timeline_limit(self) -> int:
-        return self._room_timeline_filter.limit()
+        return self._room_timeline_filter.limit
 
     def presence_limit(self) -> int:
-        return self._presence_filter.limit()
+        return self._presence_filter.limit
 
     def ephemeral_limit(self) -> int:
-        return self._room_ephemeral_filter.limit()
+        return self._room_ephemeral_filter.limit
 
     def lazy_load_members(self) -> bool:
-        return self._room_state_filter.lazy_load_members()
+        return self._room_state_filter.lazy_load_members
 
     def include_redundant_members(self) -> bool:
-        return self._room_state_filter.include_redundant_members()
+        return self._room_state_filter.include_redundant_members
 
     def filter_presence(
         self, events: Iterable[UserPresenceState]
@@ -276,19 +277,25 @@ class Filter:
     def __init__(self, filter_json: JsonDict):
         self.filter_json = filter_json
 
-        self.types = self.filter_json.get("types", None)
-        self.not_types = self.filter_json.get("not_types", [])
+        self.limit = filter_json.get("limit", 10)
+        self.lazy_load_members = filter_json.get("lazy_load_members", False)
+        self.include_redundant_members = filter_json.get(
+            "include_redundant_members", False
+        )
+
+        self.types = filter_json.get("types", None)
+        self.not_types = filter_json.get("not_types", [])
 
-        self.rooms = self.filter_json.get("rooms", None)
-        self.not_rooms = self.filter_json.get("not_rooms", [])
+        self.rooms = filter_json.get("rooms", None)
+        self.not_rooms = filter_json.get("not_rooms", [])
 
-        self.senders = self.filter_json.get("senders", None)
-        self.not_senders = self.filter_json.get("not_senders", [])
+        self.senders = filter_json.get("senders", None)
+        self.not_senders = filter_json.get("not_senders", [])
 
-        self.contains_url = self.filter_json.get("contains_url", None)
+        self.contains_url = filter_json.get("contains_url", None)
 
-        self.labels = self.filter_json.get("org.matrix.labels", None)
-        self.not_labels = self.filter_json.get("org.matrix.not_labels", [])
+        self.labels = filter_json.get("org.matrix.labels", None)
+        self.not_labels = filter_json.get("org.matrix.not_labels", [])
 
     def filters_all_types(self) -> bool:
         return "*" in self.not_types
@@ -302,76 +309,95 @@ class Filter:
     def check(self, event: FilterEvent) -> bool:
         """Checks whether the filter matches the given event.
 
+        Args:
+            event: The event, account data, or presence to check against this
+                filter.
+
         Returns:
-            True if the event matches
+            True if the event matches the filter.
         """
         # We usually get the full "events" as dictionaries coming through,
         # except for presence which actually gets passed around as its own
         # namedtuple type.
         if isinstance(event, UserPresenceState):
-            sender: Optional[str] = event.user_id
-            room_id = None
-            ev_type = "m.presence"
-            contains_url = False
-            labels: List[str] = []
+            user_id = event.user_id
+            field_matchers = {
+                "senders": lambda v: user_id == v,
+                "types": lambda v: "m.presence" == v,
+            }
+            return self._check_fields(field_matchers)
         else:
+            content = event.get("content")
+            # Content is assumed to be a dict below, so ensure it is. This should
+            # always be true for events, but account_data has been allowed to
+            # have non-dict content.
+            if not isinstance(content, dict):
+                content = {}
+
             sender = event.get("sender", None)
             if not sender:
                 # Presence events had their 'sender' in content.user_id, but are
                 # now handled above. We don't know if anything else uses this
                 # form. TODO: Check this and probably remove it.
-                content = event.get("content")
-                # account_data has been allowed to have non-dict content, so
-                # check type first
-                if isinstance(content, dict):
-                    sender = content.get("user_id")
+                sender = content.get("user_id")
 
             room_id = event.get("room_id", None)
             ev_type = event.get("type", None)
 
-            content = event.get("content") or {}
             # check if there is a string url field in the content for filtering purposes
-            contains_url = isinstance(content.get("url"), str)
             labels = content.get(EventContentFields.LABELS, [])
 
-        return self.check_fields(room_id, sender, ev_type, labels, contains_url)
+            field_matchers = {
+                "rooms": lambda v: room_id == v,
+                "senders": lambda v: sender == v,
+                "types": lambda v: _matches_wildcard(ev_type, v),
+                "labels": lambda v: v in labels,
+            }
+
+            result = self._check_fields(field_matchers)
+            if not result:
+                return result
+
+            contains_url_filter = self.contains_url
+            if contains_url_filter is not None:
+                contains_url = isinstance(content.get("url"), str)
+                if contains_url_filter != contains_url:
+                    return False
+
+            return True
 
-    def check_fields(
-        self,
-        room_id: Optional[str],
-        sender: Optional[str],
-        event_type: Optional[str],
-        labels: Container[str],
-        contains_url: bool,
-    ) -> bool:
+    def _check_fields(self, field_matchers: Dict[str, Callable[[str], bool]]) -> bool:
         """Checks whether the filter matches the given event fields.
 
+        Args:
+            field_matchers: A map of attribute name to callable to use for checking
+                particular fields.
+
+                The attribute name and an inverse (not_<attribute name>) must
+                exist on the Filter.
+
+                The callable should return true if the event's value matches the
+                filter's value.
+
         Returns:
             True if the event fields match
         """
-        literal_keys = {
-            "rooms": lambda v: room_id == v,
-            "senders": lambda v: sender == v,
-            "types": lambda v: _matches_wildcard(event_type, v),
-            "labels": lambda v: v in labels,
-        }
-
-        for name, match_func in literal_keys.items():
+
+        for name, match_func in field_matchers.items():
+            # If the event matches one of the disallowed values, reject it.
             not_name = "not_%s" % (name,)
             disallowed_values = getattr(self, not_name)
             if any(map(match_func, disallowed_values)):
                 return False
 
+            # Other the event does not match at least one of the allowed values,
+            # reject it.
             allowed_values = getattr(self, name)
             if allowed_values is not None:
                 if not any(map(match_func, allowed_values)):
                     return False
 
-        contains_url_filter = self.filter_json.get("contains_url")
-        if contains_url_filter is not None:
-            if contains_url_filter != contains_url:
-                return False
-
+        # Otherwise, accept it.
         return True
 
     def filter_rooms(self, room_ids: Iterable[str]) -> Set[str]:
@@ -385,10 +411,10 @@ class Filter:
         """
         room_ids = set(room_ids)
 
-        disallowed_rooms = set(self.filter_json.get("not_rooms", []))
+        disallowed_rooms = set(self.not_rooms)
         room_ids -= disallowed_rooms
 
-        allowed_rooms = self.filter_json.get("rooms", None)
+        allowed_rooms = self.rooms
         if allowed_rooms is not None:
             room_ids &= set(allowed_rooms)
 
@@ -397,15 +423,6 @@ class Filter:
     def filter(self, events: Iterable[FilterEvent]) -> List[FilterEvent]:
         return list(filter(self.check, events))
 
-    def limit(self) -> int:
-        return self.filter_json.get("limit", 10)
-
-    def lazy_load_members(self) -> bool:
-        return self.filter_json.get("lazy_load_members", False)
-
-    def include_redundant_members(self) -> bool:
-        return self.filter_json.get("include_redundant_members", False)
-
     def with_room_ids(self, room_ids: Iterable[str]) -> "Filter":
         """Returns a new filter with the given room IDs appended.
 
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index 60ff896386..abfe7be0e3 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -438,7 +438,7 @@ class PaginationHandler:
             }
 
         state = None
-        if event_filter and event_filter.lazy_load_members() and len(events) > 0:
+        if event_filter and event_filter.lazy_load_members and len(events) > 0:
             # TODO: remove redundant members
 
             # FIXME: we also care about invite targets etc.
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index cf01d58ea1..99e9b37344 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -1173,7 +1173,7 @@ class RoomContextHandler:
         else:
             last_event_id = event_id
 
-        if event_filter and event_filter.lazy_load_members():
+        if event_filter and event_filter.lazy_load_members:
             state_filter = StateFilter.from_lazy_load_member_list(
                 ev.sender
                 for ev in itertools.chain(
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index a3ffa26be8..6e4dff8056 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -249,7 +249,7 @@ class SearchHandler:
             )
 
             events.sort(key=lambda e: -rank_map[e.event_id])
-            allowed_events = events[: search_filter.limit()]
+            allowed_events = events[: search_filter.limit]
 
             for e in allowed_events:
                 rm = room_groups.setdefault(
@@ -271,13 +271,13 @@ class SearchHandler:
             # We keep looping and we keep filtering until we reach the limit
             # or we run out of things.
             # But only go around 5 times since otherwise synapse will be sad.
-            while len(room_events) < search_filter.limit() and i < 5:
+            while len(room_events) < search_filter.limit and i < 5:
                 i += 1
                 search_result = await self.store.search_rooms(
                     room_ids,
                     search_term,
                     keys,
-                    search_filter.limit() * 2,
+                    search_filter.limit * 2,
                     pagination_token=pagination_token,
                 )
 
@@ -299,9 +299,9 @@ class SearchHandler:
                 )
 
                 room_events.extend(events)
-                room_events = room_events[: search_filter.limit()]
+                room_events = room_events[: search_filter.limit]
 
-                if len(results) < search_filter.limit() * 2:
+                if len(results) < search_filter.limit * 2:
                     pagination_token = None
                     break
                 else:
@@ -311,7 +311,7 @@ class SearchHandler:
                 group = room_groups.setdefault(event.room_id, {"results": []})
                 group["results"].append(event.event_id)
 
-            if room_events and len(room_events) >= search_filter.limit():
+            if room_events and len(room_events) >= search_filter.limit:
                 last_event_id = room_events[-1].event_id
                 pagination_token = results_map[last_event_id]["pagination_token"]
 
-- 
cgit 1.5.1


From 576921c66a35fa1023f7e9baf97b6304ff463549 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 27 Oct 2021 18:06:32 +0200
Subject: Force deb compression with `xz`. (#11197)

Fixes a problem where `impish` packages could not be processed by `reprepro`.
---
 debian/changelog | 6 ++++++
 debian/rules     | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index ea96676f74..c2ea5d2cfb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.46.0~rc1ubuntu1) UNRELEASED; urgency=medium
+
+  * Compress debs with xz, to fix incompatibility of impish debs with reprepro.
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 27 Oct 2021 15:32:51 +0100
+
 matrix-synapse-py3 (1.46.0~rc1) stable; urgency=medium
 
   * New synapse release 1.46.0~rc1.
diff --git a/debian/rules b/debian/rules
index b9d490adc9..5baf2475f0 100755
--- a/debian/rules
+++ b/debian/rules
@@ -51,5 +51,11 @@ override_dh_shlibdeps:
 override_dh_virtualenv:
 	./debian/build_virtualenv
 
+override_dh_builddeb:
+        # force the compression to xzip, to stop dpkg-deb on impish defaulting to zstd
+        # (which requires reprepro 5.3.0-1.3, which is currently only in 'experimental' in Debian:
+        # https://metadata.ftp-master.debian.org/changelogs/main/r/reprepro/reprepro_5.3.0-1.3_changelog)
+	dh_builddeb -- -Zxz
+
 %:
 	dh $@ --with python-virtualenv
-- 
cgit 1.5.1


From 4e393af52f6d15d195319fa240699522100e4844 Mon Sep 17 00:00:00 2001
From: Samuel Philipp <codes@samuel-philipp.de>
Date: Wed, 27 Oct 2021 18:25:18 +0200
Subject: Fixed config parse bug in review_recent_signups (#11191)

---
 changelog.d/11191.bugfix                  | 1 +
 synapse/_scripts/review_recent_signups.py | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11191.bugfix

diff --git a/changelog.d/11191.bugfix b/changelog.d/11191.bugfix
new file mode 100644
index 0000000000..9104db7f0e
--- /dev/null
+++ b/changelog.d/11191.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.45.0 which prevented the `synapse_review_recent_signups` script from running. Contributed by @samuel-p.
diff --git a/synapse/_scripts/review_recent_signups.py b/synapse/_scripts/review_recent_signups.py
index 9de913db88..8e66a38421 100644
--- a/synapse/_scripts/review_recent_signups.py
+++ b/synapse/_scripts/review_recent_signups.py
@@ -20,7 +20,12 @@ from typing import List
 
 import attr
 
-from synapse.config._base import RootConfig, find_config_files, read_config_files
+from synapse.config._base import (
+    Config,
+    RootConfig,
+    find_config_files,
+    read_config_files,
+)
 from synapse.config.database import DatabaseConfig
 from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn
 from synapse.storage.engines import create_engine
@@ -126,7 +131,7 @@ def main():
         config_dict,
     )
 
-    since_ms = time.time() * 1000 - config.parse_duration(config_args.since)
+    since_ms = time.time() * 1000 - Config.parse_duration(config_args.since)
     exclude_users_with_email = config_args.exclude_emails
     include_context = not config_args.only_users
 
-- 
cgit 1.5.1


From 75ca0a6168f92dab3255839cf85fb0df3a0076c3 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Wed, 27 Oct 2021 17:27:23 +0100
Subject: Annotate `log_function` decorator (#10943)

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
---
 changelog.d/10943.misc                           |  1 +
 synapse/federation/federation_client.py          | 17 +++++++++++++++--
 synapse/federation/federation_server.py          | 10 ++++++----
 synapse/federation/sender/transaction_manager.py |  1 -
 synapse/federation/transport/client.py           | 22 ++++++++++++++++++----
 synapse/handlers/directory.py                    |  2 +-
 synapse/handlers/federation_event.py             |  2 +-
 synapse/handlers/presence.py                     |  2 ++
 synapse/handlers/profile.py                      |  4 ++++
 synapse/logging/utils.py                         |  8 ++++++--
 synapse/state/__init__.py                        |  5 +++--
 synapse/storage/databases/main/profile.py        |  2 +-
 12 files changed, 58 insertions(+), 18 deletions(-)
 create mode 100644 changelog.d/10943.misc

diff --git a/changelog.d/10943.misc b/changelog.d/10943.misc
new file mode 100644
index 0000000000..3ce28d1a67
--- /dev/null
+++ b/changelog.d/10943.misc
@@ -0,0 +1 @@
+Add type annotations for the `log_function` decorator.
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 2ab4dec88f..670186f548 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -227,7 +227,7 @@ class FederationClient(FederationBase):
         )
 
     async def backfill(
-        self, dest: str, room_id: str, limit: int, extremities: Iterable[str]
+        self, dest: str, room_id: str, limit: int, extremities: Collection[str]
     ) -> Optional[List[EventBase]]:
         """Requests some more historic PDUs for the given room from the
         given destination server.
@@ -237,6 +237,8 @@ class FederationClient(FederationBase):
             room_id: The room_id to backfill.
             limit: The maximum number of events to return.
             extremities: our current backwards extremities, to backfill from
+                Must be a Collection that is falsy when empty.
+                (Iterable is not enough here!)
         """
         logger.debug("backfill extrem=%s", extremities)
 
@@ -250,11 +252,22 @@ class FederationClient(FederationBase):
 
         logger.debug("backfill transaction_data=%r", transaction_data)
 
+        if not isinstance(transaction_data, dict):
+            # TODO we probably want an exception type specific to federation
+            # client validation.
+            raise TypeError("Backfill transaction_data is not a dict.")
+
+        transaction_data_pdus = transaction_data.get("pdus")
+        if not isinstance(transaction_data_pdus, list):
+            # TODO we probably want an exception type specific to federation
+            # client validation.
+            raise TypeError("transaction_data.pdus is not a list.")
+
         room_version = await self.store.get_room_version(room_id)
 
         pdus = [
             event_from_pdu_json(p, room_version, outlier=False)
-            for p in transaction_data["pdus"]
+            for p in transaction_data_pdus
         ]
 
         # Check signatures and hash of pdus, removing any from the list that fail checks
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 0d66034f44..32a75993d9 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -295,14 +295,16 @@ class FederationServer(FederationBase):
         Returns:
             HTTP response code and body
         """
-        response = await self.transaction_actions.have_responded(origin, transaction)
+        existing_response = await self.transaction_actions.have_responded(
+            origin, transaction
+        )
 
-        if response:
+        if existing_response:
             logger.debug(
                 "[%s] We've already responded to this request",
                 transaction.transaction_id,
             )
-            return response
+            return existing_response
 
         logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
@@ -632,7 +634,7 @@ class FederationServer(FederationBase):
 
     async def on_make_knock_request(
         self, origin: str, room_id: str, user_id: str, supported_versions: List[str]
-    ) -> Dict[str, Union[EventBase, str]]:
+    ) -> JsonDict:
         """We've received a /make_knock/ request, so we create a partial knock
         event for the room and hand that back, along with the room version, to the knocking
         homeserver. We do *not* persist or process this event until the other server has
diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py
index dc555cca0b..ab935e5a7e 100644
--- a/synapse/federation/sender/transaction_manager.py
+++ b/synapse/federation/sender/transaction_manager.py
@@ -149,7 +149,6 @@ class TransactionManager:
                 )
             except HttpResponseException as e:
                 code = e.code
-                response = e.response
 
                 set_tag(tags.ERROR, True)
 
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 8b247fe206..d963178838 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -15,7 +15,19 @@
 
 import logging
 import urllib
-from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Collection,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 import attr
 import ijson
@@ -100,7 +112,7 @@ class TransportLayerClient:
 
     @log_function
     async def backfill(
-        self, destination: str, room_id: str, event_tuples: Iterable[str], limit: int
+        self, destination: str, room_id: str, event_tuples: Collection[str], limit: int
     ) -> Optional[JsonDict]:
         """Requests `limit` previous PDUs in a given context before list of
         PDUs.
@@ -108,7 +120,9 @@ class TransportLayerClient:
         Args:
             destination
             room_id
-            event_tuples
+            event_tuples:
+                Must be a Collection that is falsy when empty.
+                (Iterable is not enough here!)
             limit
 
         Returns:
@@ -786,7 +800,7 @@ class TransportLayerClient:
     @log_function
     def join_group(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
-    ) -> JsonDict:
+    ) -> Awaitable[JsonDict]:
         """Attempts to join a group"""
         path = _create_v1_path("/groups/%s/users/%s/join", group_id, user_id)
 
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index 8567cb0e00..8ca5f60b1c 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -245,7 +245,7 @@ class DirectoryHandler:
                 servers = result.servers
         else:
             try:
-                fed_result = await self.federation.make_query(
+                fed_result: Optional[JsonDict] = await self.federation.make_query(
                     destination=room_alias.domain,
                     query_type="directory",
                     args={"room_alias": room_alias.to_string()},
diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index bd1fa08cef..e617db4c0d 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -477,7 +477,7 @@ class FederationEventHandler:
 
     @log_function
     async def backfill(
-        self, dest: str, room_id: str, limit: int, extremities: Iterable[str]
+        self, dest: str, room_id: str, limit: int, extremities: Collection[str]
     ) -> None:
         """Trigger a backfill request to `dest` for the given `room_id`
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index fdab50da37..3df872c578 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -52,6 +52,7 @@ import synapse.metrics
 from synapse.api.constants import EventTypes, Membership, PresenceState
 from synapse.api.errors import SynapseError
 from synapse.api.presence import UserPresenceState
+from synapse.appservice import ApplicationService
 from synapse.events.presence_router import PresenceRouter
 from synapse.logging.context import run_in_background
 from synapse.logging.utils import log_function
@@ -1551,6 +1552,7 @@ class PresenceEventSource(EventSource[int, UserPresenceState]):
         is_guest: bool = False,
         explicit_room_id: Optional[str] = None,
         include_offline: bool = True,
+        service: Optional[ApplicationService] = None,
     ) -> Tuple[List[UserPresenceState], int]:
         # The process for getting presence events are:
         #  1. Get the rooms the user is in.
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index e6c3cf585b..6b5a6ded8b 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -456,7 +456,11 @@ class ProfileHandler:
                 continue
 
             new_name = profile.get("displayname")
+            if not isinstance(new_name, str):
+                new_name = None
             new_avatar = profile.get("avatar_url")
+            if not isinstance(new_avatar, str):
+                new_avatar = None
 
             # We always hit update to update the last_check timestamp
             await self.store.update_remote_profile_cache(user_id, new_name, new_avatar)
diff --git a/synapse/logging/utils.py b/synapse/logging/utils.py
index 08895e72ee..4a01b902c2 100644
--- a/synapse/logging/utils.py
+++ b/synapse/logging/utils.py
@@ -16,6 +16,7 @@
 import logging
 from functools import wraps
 from inspect import getcallargs
+from typing import Callable, TypeVar, cast
 
 _TIME_FUNC_ID = 0
 
@@ -41,7 +42,10 @@ def _log_debug_as_f(f, msg, msg_args):
         logger.handle(record)
 
 
-def log_function(f):
+F = TypeVar("F", bound=Callable)
+
+
+def log_function(f: F) -> F:
     """Function decorator that logs every call to that function."""
     func_name = f.__name__
 
@@ -69,4 +73,4 @@ def log_function(f):
         return f(*args, **kwargs)
 
     wrapped.__name__ = func_name
-    return wrapped
+    return cast(F, wrapped)
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index 5cf2e12575..98a0239759 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -26,6 +26,7 @@ from typing import (
     FrozenSet,
     Iterable,
     List,
+    Mapping,
     Optional,
     Sequence,
     Set,
@@ -519,7 +520,7 @@ class StateResolutionHandler:
         self,
         room_id: str,
         room_version: str,
-        state_groups_ids: Dict[int, StateMap[str]],
+        state_groups_ids: Mapping[int, StateMap[str]],
         event_map: Optional[Dict[str, EventBase]],
         state_res_store: "StateResolutionStore",
     ) -> _StateCacheEntry:
@@ -703,7 +704,7 @@ class StateResolutionHandler:
 
 
 def _make_state_cache_entry(
-    new_state: StateMap[str], state_groups_ids: Dict[int, StateMap[str]]
+    new_state: StateMap[str], state_groups_ids: Mapping[int, StateMap[str]]
 ) -> _StateCacheEntry:
     """Given a resolved state, and a set of input state groups, pick one to base
     a new state group on (if any), and return an appropriately-constructed
diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py
index ba7075caa5..dd8e27e226 100644
--- a/synapse/storage/databases/main/profile.py
+++ b/synapse/storage/databases/main/profile.py
@@ -91,7 +91,7 @@ class ProfileWorkerStore(SQLBaseStore):
         )
 
     async def update_remote_profile_cache(
-        self, user_id: str, displayname: str, avatar_url: str
+        self, user_id: str, displayname: Optional[str], avatar_url: Optional[str]
     ) -> int:
         return await self.db_pool.simple_update(
             table="remote_profile_cache",
-- 
cgit 1.5.1


From a19bf32a03cbc43169f1076c002287f193bffa4a Mon Sep 17 00:00:00 2001
From: Skyler Mäntysaari <samip5@users.noreply.github.com>
Date: Thu, 28 Oct 2021 13:31:22 +0300
Subject: docs/openid: Add Authentik documentation. (#11151)

---
 changelog.d/11151.doc |  1 +
 docs/openid.md        | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 changelog.d/11151.doc

diff --git a/changelog.d/11151.doc b/changelog.d/11151.doc
new file mode 100644
index 0000000000..68cd99471f
--- /dev/null
+++ b/changelog.d/11151.doc
@@ -0,0 +1 @@
+Add documentation for using Authentik as an OpenID Connect Identity Provider. Contributed by @samip5.
\ No newline at end of file
diff --git a/docs/openid.md b/docs/openid.md
index 49180eec52..4a340ef107 100644
--- a/docs/openid.md
+++ b/docs/openid.md
@@ -21,6 +21,7 @@ such as [Github][github-idp].
 
 [google-idp]: https://developers.google.com/identity/protocols/oauth2/openid-connect
 [auth0]: https://auth0.com/
+[authentik]: https://goauthentik.io/
 [okta]: https://www.okta.com/
 [dex-idp]: https://github.com/dexidp/dex
 [keycloak-idp]: https://www.keycloak.org/docs/latest/server_admin/#sso-protocols
@@ -209,6 +210,39 @@ oidc_providers:
         display_name_template: "{{ user.name }}"
 ```
 
+### Authentik
+
+[Authentik][authentik] is an open-source IdP solution.
+
+1. Create a provider in Authentik, with type OAuth2/OpenID.
+2. The parameters are:
+- Client Type: Confidential
+- JWT Algorithm: RS256
+- Scopes: OpenID, Email and Profile
+- RSA Key: Select any available key
+- Redirect URIs: `[synapse public baseurl]/_synapse/client/oidc/callback`
+3. Create an application for synapse in Authentik and link it to the provider.
+4. Note the slug of your application, Client ID and Client Secret.
+
+Synapse config:
+```yaml
+oidc_providers:
+  - idp_id: authentik
+    idp_name: authentik
+    discover: true
+    issuer: "https://your.authentik.example.org/application/o/your-app-slug/" # TO BE FILLED: domain and slug
+    client_id: "your client id" # TO BE FILLED
+    client_secret: "your client secret" # TO BE FILLED
+    scopes:
+      - "openid"
+      - "profile"
+      - "email"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}}"
+        display_name_template: "{{ user.preferred_username|capitalize }}" # TO BE FILLED: If your users have names in Authentik and you want those in Synapse, this should be replaced with user.name|capitalize.
+```
+
 ### GitHub
 
 [GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but
-- 
cgit 1.5.1


From 1bfd141205d8e2abceef3c277e47f20799bbd455 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 14:14:42 +0100
Subject: Type hints for the remaining two files in `synapse.http`. (#11164)

* Teach MyPy that the sentinel context is False

This means that if `ctx: LoggingContextOrSentinel`
then `bool(ctx)` narrows us to `ctx:LoggingContext`, which is a really
neat find!

* Annotate RequestMetrics

- Raise errors for sentry if we use the sentinel context
- Ensure we don't raise an error and carry on, but not recording stats
- Include stack trace in the error case to lower Sean's blood pressure

* Make mypy pass for synapse.http.request_metrics

* Make synapse.http.connectproxyclient pass mypy

Co-authored-by: reivilibre <oliverw@matrix.org>
---
 changelog.d/11164.misc             |  1 +
 mypy.ini                           | 12 ++-------
 synapse/http/connectproxyclient.py | 12 ++++++---
 synapse/http/request_metrics.py    | 50 +++++++++++++++++++++++++-------------
 synapse/logging/context.py         |  4 +--
 synapse/metrics/__init__.py        | 14 +++++++----
 6 files changed, 56 insertions(+), 37 deletions(-)
 create mode 100644 changelog.d/11164.misc

diff --git a/changelog.d/11164.misc b/changelog.d/11164.misc
new file mode 100644
index 0000000000..751da49183
--- /dev/null
+++ b/changelog.d/11164.misc
@@ -0,0 +1 @@
+Add type hints so that `synapse.http` passes `mypy` checks.
\ No newline at end of file
diff --git a/mypy.ini b/mypy.ini
index c5f44aea39..8f5386c179 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -16,6 +16,7 @@ no_implicit_optional = True
 
 files =
   scripts-dev/sign_json,
+  synapse/__init__.py,
   synapse/api,
   synapse/appservice,
   synapse/config,
@@ -31,16 +32,7 @@ files =
   synapse/federation,
   synapse/groups,
   synapse/handlers,
-  synapse/http/additional_resource.py,
-  synapse/http/client.py,
-  synapse/http/federation/matrix_federation_agent.py,
-  synapse/http/federation/srv_resolver.py,
-  synapse/http/federation/well_known_resolver.py,
-  synapse/http/matrixfederationclient.py,
-  synapse/http/proxyagent.py,
-  synapse/http/servlet.py,
-  synapse/http/server.py,
-  synapse/http/site.py,
+  synapse/http,
   synapse/logging,
   synapse/metrics,
   synapse/module_api,
diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py
index c577142268..fbafffd69b 100644
--- a/synapse/http/connectproxyclient.py
+++ b/synapse/http/connectproxyclient.py
@@ -84,7 +84,11 @@ class HTTPConnectProxyEndpoint:
     def __repr__(self):
         return "<HTTPConnectProxyEndpoint %s>" % (self._proxy_endpoint,)
 
-    def connect(self, protocolFactory: ClientFactory):
+    # Mypy encounters a false positive here: it complains that ClientFactory
+    # is incompatible with IProtocolFactory. But ClientFactory inherits from
+    # Factory, which implements IProtocolFactory. So I think this is a bug
+    # in mypy-zope.
+    def connect(self, protocolFactory: ClientFactory):  # type: ignore[override]
         f = HTTPProxiedClientFactory(
             self._host, self._port, protocolFactory, self._proxy_creds
         )
@@ -119,13 +123,15 @@ class HTTPProxiedClientFactory(protocol.ClientFactory):
         self.dst_port = dst_port
         self.wrapped_factory = wrapped_factory
         self.proxy_creds = proxy_creds
-        self.on_connection = defer.Deferred()
+        self.on_connection: "defer.Deferred[None]" = defer.Deferred()
 
     def startedConnecting(self, connector):
         return self.wrapped_factory.startedConnecting(connector)
 
     def buildProtocol(self, addr):
         wrapped_protocol = self.wrapped_factory.buildProtocol(addr)
+        if wrapped_protocol is None:
+            raise TypeError("buildProtocol produced None instead of a Protocol")
 
         return HTTPConnectProtocol(
             self.dst_host,
@@ -235,7 +241,7 @@ class HTTPConnectSetupClient(http.HTTPClient):
         self.host = host
         self.port = port
         self.proxy_creds = proxy_creds
-        self.on_connected = defer.Deferred()
+        self.on_connected: "defer.Deferred[None]" = defer.Deferred()
 
     def connectionMade(self):
         logger.debug("Connected to proxy, sending CONNECT")
diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py
index 602f93c497..4886626d50 100644
--- a/synapse/http/request_metrics.py
+++ b/synapse/http/request_metrics.py
@@ -15,6 +15,8 @@
 
 import logging
 import threading
+import traceback
+from typing import Dict, Mapping, Set, Tuple
 
 from prometheus_client.core import Counter, Histogram
 
@@ -105,19 +107,14 @@ in_flight_requests_db_sched_duration = Counter(
     ["method", "servlet"],
 )
 
-# The set of all in flight requests, set[RequestMetrics]
-_in_flight_requests = set()
+_in_flight_requests: Set["RequestMetrics"] = set()
 
 # Protects the _in_flight_requests set from concurrent access
 _in_flight_requests_lock = threading.Lock()
 
 
-def _get_in_flight_counts():
-    """Returns a count of all in flight requests by (method, server_name)
-
-    Returns:
-        dict[tuple[str, str], int]
-    """
+def _get_in_flight_counts() -> Mapping[Tuple[str, ...], int]:
+    """Returns a count of all in flight requests by (method, server_name)"""
     # Cast to a list to prevent it changing while the Prometheus
     # thread is collecting metrics
     with _in_flight_requests_lock:
@@ -127,8 +124,9 @@ def _get_in_flight_counts():
         rm.update_metrics()
 
     # Map from (method, name) -> int, the number of in flight requests of that
-    # type
-    counts = {}
+    # type. The key type is Tuple[str, str], but we leave the length unspecified
+    # for compatability with LaterGauge's annotations.
+    counts: Dict[Tuple[str, ...], int] = {}
     for rm in reqs:
         key = (rm.method, rm.name)
         counts[key] = counts.get(key, 0) + 1
@@ -145,15 +143,21 @@ LaterGauge(
 
 
 class RequestMetrics:
-    def start(self, time_sec, name, method):
-        self.start = time_sec
+    def start(self, time_sec: float, name: str, method: str) -> None:
+        self.start_ts = time_sec
         self.start_context = current_context()
         self.name = name
         self.method = method
 
-        # _request_stats records resource usage that we have already added
-        # to the "in flight" metrics.
-        self._request_stats = self.start_context.get_resource_usage()
+        if self.start_context:
+            # _request_stats records resource usage that we have already added
+            # to the "in flight" metrics.
+            self._request_stats = self.start_context.get_resource_usage()
+        else:
+            logger.error(
+                "Tried to start a RequestMetric from the sentinel context.\n%s",
+                "".join(traceback.format_stack()),
+            )
 
         with _in_flight_requests_lock:
             _in_flight_requests.add(self)
@@ -169,12 +173,18 @@ class RequestMetrics:
             tag = context.tag
 
             if context != self.start_context:
-                logger.warning(
+                logger.error(
                     "Context have unexpectedly changed %r, %r",
                     context,
                     self.start_context,
                 )
                 return
+        else:
+            logger.error(
+                "Trying to stop RequestMetrics in the sentinel context.\n%s",
+                "".join(traceback.format_stack()),
+            )
+            return
 
         response_code = str(response_code)
 
@@ -183,7 +193,7 @@ class RequestMetrics:
         response_count.labels(self.method, self.name, tag).inc()
 
         response_timer.labels(self.method, self.name, tag, response_code).observe(
-            time_sec - self.start
+            time_sec - self.start_ts
         )
 
         resource_usage = context.get_resource_usage()
@@ -213,6 +223,12 @@ class RequestMetrics:
 
     def update_metrics(self):
         """Updates the in flight metrics with values from this request."""
+        if not self.start_context:
+            logger.error(
+                "Tried to update a RequestMetric from the sentinel context.\n%s",
+                "".join(traceback.format_stack()),
+            )
+            return
         new_stats = self.start_context.get_resource_usage()
 
         diff = new_stats - self._request_stats
diff --git a/synapse/logging/context.py b/synapse/logging/context.py
index bdc0187743..d8ae3188b7 100644
--- a/synapse/logging/context.py
+++ b/synapse/logging/context.py
@@ -220,7 +220,7 @@ class _Sentinel:
         self.scope = None
         self.tag = None
 
-    def __str__(self):
+    def __str__(self) -> str:
         return "sentinel"
 
     def copy_to(self, record):
@@ -241,7 +241,7 @@ class _Sentinel:
     def record_event_fetch(self, event_count):
         pass
 
-    def __bool__(self):
+    def __bool__(self) -> Literal[False]:
         return False
 
 
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index f237b8a236..e902109af3 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -20,7 +20,7 @@ import os
 import platform
 import threading
 import time
-from typing import Callable, Dict, Iterable, Optional, Tuple, Union
+from typing import Callable, Dict, Iterable, Mapping, Optional, Tuple, Union
 
 import attr
 from prometheus_client import Counter, Gauge, Histogram
@@ -67,7 +67,11 @@ class LaterGauge:
     labels = attr.ib(hash=False, type=Optional[Iterable[str]])
     # callback: should either return a value (if there are no labels for this metric),
     # or dict mapping from a label tuple to a value
-    caller = attr.ib(type=Callable[[], Union[Dict[Tuple[str, ...], float], float]])
+    caller = attr.ib(
+        type=Callable[
+            [], Union[Mapping[Tuple[str, ...], Union[int, float]], Union[int, float]]
+        ]
+    )
 
     def collect(self):
 
@@ -80,11 +84,11 @@ class LaterGauge:
             yield g
             return
 
-        if isinstance(calls, dict):
+        if isinstance(calls, (int, float)):
+            g.add_metric([], calls)
+        else:
             for k, v in calls.items():
                 g.add_metric(k, v)
-        else:
-            g.add_metric([], calls)
 
         yield g
 
-- 
cgit 1.5.1


From adc0d35b17952b8b74fbfad663f9bff4e4dd975a Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Thu, 28 Oct 2021 18:45:53 +0200
Subject: Add a ModuleApi method to update a user's membership in a room
 (#11147)

Co-authored-by: reivilibre <oliverw@matrix.org>
---
 changelog.d/11147.feature      |   1 +
 synapse/module_api/__init__.py | 100 +++++++++++++++++++++++++++++++-
 tests/module_api/test_api.py   | 126 ++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11147.feature

diff --git a/changelog.d/11147.feature b/changelog.d/11147.feature
new file mode 100644
index 0000000000..af72d85c20
--- /dev/null
+++ b/changelog.d/11147.feature
@@ -0,0 +1 @@
+Add a module API method to update a user's membership in a room.
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index d707a9325d..36042ed2e0 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -33,6 +33,7 @@ import jinja2
 from twisted.internet import defer
 from twisted.web.resource import IResource
 
+from synapse.api.errors import SynapseError
 from synapse.events import EventBase
 from synapse.events.presence_router import PresenceRouter
 from synapse.http.client import SimpleHttpClient
@@ -625,8 +626,105 @@ class ModuleApi:
         state = yield defer.ensureDeferred(self._store.get_events(state_ids.values()))
         return state.values()
 
+    async def update_room_membership(
+        self,
+        sender: str,
+        target: str,
+        room_id: str,
+        new_membership: str,
+        content: Optional[JsonDict] = None,
+    ) -> EventBase:
+        """Updates the membership of a user to the given value.
+
+        Added in Synapse v1.46.0.
+
+        Args:
+            sender: The user performing the membership change. Must be a user local to
+                this homeserver.
+            target: The user whose membership is changing. This is often the same value
+                as `sender`, but it might differ in some cases (e.g. when kicking a user,
+                the `sender` is the user performing the kick and the `target` is the user
+                being kicked).
+            room_id: The room in which to change the membership.
+            new_membership: The new membership state of `target` after this operation. See
+                https://spec.matrix.org/unstable/client-server-api/#mroommember for the
+                list of allowed values.
+            content: Additional values to include in the resulting event's content.
+
+        Returns:
+            The newly created membership event.
+
+        Raises:
+            RuntimeError if the `sender` isn't a local user.
+            ShadowBanError if a shadow-banned requester attempts to send an invite.
+            SynapseError if the module attempts to send a membership event that isn't
+                allowed, either by the server's configuration (e.g. trying to set a
+                per-room display name that's too long) or by the validation rules around
+                membership updates (e.g. the `membership` value is invalid).
+        """
+        if not self.is_mine(sender):
+            raise RuntimeError(
+                "Tried to send an event as a user that isn't local to this homeserver",
+            )
+
+        requester = create_requester(sender)
+        target_user_id = UserID.from_string(target)
+
+        if content is None:
+            content = {}
+
+        # Set the profile if not already done by the module.
+        if "avatar_url" not in content or "displayname" not in content:
+            try:
+                # Try to fetch the user's profile.
+                profile = await self._hs.get_profile_handler().get_profile(
+                    target_user_id.to_string(),
+                )
+            except SynapseError as e:
+                # If the profile couldn't be found, use default values.
+                profile = {
+                    "displayname": target_user_id.localpart,
+                    "avatar_url": None,
+                }
+
+                if e.code != 404:
+                    # If the error isn't 404, it means we tried to fetch the profile over
+                    # federation but the remote server responded with a non-standard
+                    # status code.
+                    logger.error(
+                        "Got non-404 error status when fetching profile for %s",
+                        target_user_id.to_string(),
+                    )
+
+            # Set the profile where it needs to be set.
+            if "avatar_url" not in content:
+                content["avatar_url"] = profile["avatar_url"]
+
+            if "displayname" not in content:
+                content["displayname"] = profile["displayname"]
+
+        event_id, _ = await self._hs.get_room_member_handler().update_membership(
+            requester=requester,
+            target=target_user_id,
+            room_id=room_id,
+            action=new_membership,
+            content=content,
+        )
+
+        # Try to retrieve the resulting event.
+        event = await self._hs.get_datastore().get_event(event_id)
+
+        # update_membership is supposed to always return after the event has been
+        # successfully persisted.
+        assert event is not None
+
+        return event
+
     async def create_and_send_event_into_room(self, event_dict: JsonDict) -> EventBase:
-        """Create and send an event into a room. Membership events are currently not supported.
+        """Create and send an event into a room.
+
+        Membership events are not supported by this method. To update a user's membership
+        in a room, please use the `update_room_membership` method instead.
 
         Added in Synapse v1.22.0.
 
diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py
index e915dd5c7c..37852852a8 100644
--- a/tests/module_api/test_api.py
+++ b/tests/module_api/test_api.py
@@ -20,7 +20,7 @@ from synapse.events import EventBase
 from synapse.federation.units import Transaction
 from synapse.handlers.presence import UserPresenceState
 from synapse.rest import admin
-from synapse.rest.client import login, presence, room
+from synapse.rest.client import login, presence, profile, room
 from synapse.types import create_requester
 
 from tests.events.test_presence_router import send_presence_update, sync_presence
@@ -37,6 +37,7 @@ class ModuleApiTestCase(HomeserverTestCase):
         login.register_servlets,
         room.register_servlets,
         presence.register_servlets,
+        profile.register_servlets,
     ]
 
     def prepare(self, reactor, clock, homeserver):
@@ -385,6 +386,129 @@ class ModuleApiTestCase(HomeserverTestCase):
 
         self.assertTrue(found_update)
 
+    def test_update_membership(self):
+        """Tests that the module API can update the membership of a user in a room."""
+        peter = self.register_user("peter", "hackme")
+        lesley = self.register_user("lesley", "hackme")
+        tok = self.login("peter", "hackme")
+        lesley_tok = self.login("lesley", "hackme")
+
+        # Make peter create a public room.
+        room_id = self.helper.create_room_as(
+            room_creator=peter, is_public=True, tok=tok
+        )
+
+        # Set a profile for lesley.
+        channel = self.make_request(
+            method="PUT",
+            path="/_matrix/client/r0/profile/%s/displayname" % lesley,
+            content={"displayname": "Lesley May"},
+            access_token=lesley_tok,
+        )
+
+        self.assertEqual(channel.code, 200, channel.result)
+
+        channel = self.make_request(
+            method="PUT",
+            path="/_matrix/client/r0/profile/%s/avatar_url" % lesley,
+            content={"avatar_url": "some_url"},
+            access_token=lesley_tok,
+        )
+
+        self.assertEqual(channel.code, 200, channel.result)
+
+        # Make Peter invite Lesley to the room.
+        self.get_success(
+            defer.ensureDeferred(
+                self.module_api.update_room_membership(peter, lesley, room_id, "invite")
+            )
+        )
+
+        res = self.helper.get_state(
+            room_id=room_id,
+            event_type="m.room.member",
+            state_key=lesley,
+            tok=tok,
+        )
+
+        # Check the membership is correct.
+        self.assertEqual(res["membership"], "invite")
+
+        # Also check that the profile was correctly filled out, and that it's not
+        # Peter's.
+        self.assertEqual(res["displayname"], "Lesley May")
+        self.assertEqual(res["avatar_url"], "some_url")
+
+        # Make lesley join it.
+        self.get_success(
+            defer.ensureDeferred(
+                self.module_api.update_room_membership(lesley, lesley, room_id, "join")
+            )
+        )
+
+        # Check that the membership of lesley in the room is "join".
+        res = self.helper.get_state(
+            room_id=room_id,
+            event_type="m.room.member",
+            state_key=lesley,
+            tok=tok,
+        )
+
+        self.assertEqual(res["membership"], "join")
+
+        # Also check that the profile was correctly filled out.
+        self.assertEqual(res["displayname"], "Lesley May")
+        self.assertEqual(res["avatar_url"], "some_url")
+
+        # Make peter kick lesley from the room.
+        self.get_success(
+            defer.ensureDeferred(
+                self.module_api.update_room_membership(peter, lesley, room_id, "leave")
+            )
+        )
+
+        # Check that the membership of lesley in the room is "leave".
+        res = self.helper.get_state(
+            room_id=room_id,
+            event_type="m.room.member",
+            state_key=lesley,
+            tok=tok,
+        )
+
+        self.assertEqual(res["membership"], "leave")
+
+        # Try to send a membership update from a non-local user and check that it fails.
+        d = defer.ensureDeferred(
+            self.module_api.update_room_membership(
+                "@nicolas:otherserver.com",
+                lesley,
+                room_id,
+                "invite",
+            )
+        )
+
+        self.get_failure(d, RuntimeError)
+
+        # Check that inviting a user that doesn't have a profile falls back to using a
+        # default (localpart + no avatar) profile.
+        simone = "@simone:" + self.hs.config.server.server_name
+        self.get_success(
+            defer.ensureDeferred(
+                self.module_api.update_room_membership(peter, simone, room_id, "invite")
+            )
+        )
+
+        res = self.helper.get_state(
+            room_id=room_id,
+            event_type="m.room.member",
+            state_key=simone,
+            tok=tok,
+        )
+
+        self.assertEqual(res["membership"], "invite")
+        self.assertEqual(res["displayname"], "simone")
+        self.assertIsNone(res["avatar_url"])
+
 
 class ModuleApiWorkerTestCase(BaseMultiWorkerStreamTestCase):
     """For testing ModuleApi functionality in a multi-worker setup"""
-- 
cgit 1.5.1


From e002faee01615c1976437af28f66544c5f2eed84 Mon Sep 17 00:00:00 2001
From: Shay <shaysquared@gmail.com>
Date: Thu, 28 Oct 2021 10:27:17 -0700
Subject: Fetch verify key locally rather than trying to do so over federation
 if origin and host are the same. (#11129)

* add tests for fetching key locally

* add logic to check if origin server is same as host and fetch verify key locally rather than over federation

* add changelog

* slight refactor, add docstring, change changelog entry

* Make changelog entry one line

* remove verify_json_locally and push locality check to process_request, add function process_request_locally

* remove leftover code reference

* refactor to add common call to 'verify_json and associated handling code

* add type hint to process_json

* add some docstrings + very slight refactor
---
 changelog.d/11129.bugfix     |  1 +
 synapse/crypto/keyring.py    | 74 +++++++++++++++++++++++++++-----------------
 tests/crypto/test_keyring.py | 12 +++++++
 3 files changed, 58 insertions(+), 29 deletions(-)
 create mode 100644 changelog.d/11129.bugfix

diff --git a/changelog.d/11129.bugfix b/changelog.d/11129.bugfix
new file mode 100644
index 0000000000..5e9aa538ec
--- /dev/null
+++ b/changelog.d/11129.bugfix
@@ -0,0 +1 @@
+Fix long-standing bug where verification requests could fail in certain cases if whitelist was in place but did not include your own homeserver.
\ No newline at end of file
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 8628e951c4..f641ab7ef5 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -22,6 +22,7 @@ import attr
 from signedjson.key import (
     decode_verify_key_bytes,
     encode_verify_key_base64,
+    get_verify_key,
     is_signing_algorithm_supported,
 )
 from signedjson.sign import (
@@ -30,6 +31,7 @@ from signedjson.sign import (
     signature_ids,
     verify_signed_json,
 )
+from signedjson.types import VerifyKey
 from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
@@ -177,6 +179,8 @@ class Keyring:
             clock=hs.get_clock(),
             process_batch_callback=self._inner_fetch_key_requests,
         )
+        self.verify_key = get_verify_key(hs.signing_key)
+        self.hostname = hs.hostname
 
     async def verify_json_for_server(
         self,
@@ -196,6 +200,7 @@ class Keyring:
             validity_time: timestamp at which we require the signing key to
                 be valid. (0 implies we don't care)
         """
+
         request = VerifyJsonRequest.from_json_object(
             server_name,
             json_object,
@@ -262,6 +267,11 @@ class Keyring:
                 Codes.UNAUTHORIZED,
             )
 
+        # If we are the originating server don't fetch verify key for self over federation
+        if verify_request.server_name == self.hostname:
+            await self._process_json(self.verify_key, verify_request)
+            return
+
         # Add the keys we need to verify to the queue for retrieval. We queue
         # up requests for the same server so we don't end up with many in flight
         # requests for the same keys.
@@ -285,35 +295,8 @@ class Keyring:
             if key_result.valid_until_ts < verify_request.minimum_valid_until_ts:
                 continue
 
-            verify_key = key_result.verify_key
-            json_object = verify_request.get_json_object()
-            try:
-                verify_signed_json(
-                    json_object,
-                    verify_request.server_name,
-                    verify_key,
-                )
-                verified = True
-            except SignatureVerifyException as e:
-                logger.debug(
-                    "Error verifying signature for %s:%s:%s with key %s: %s",
-                    verify_request.server_name,
-                    verify_key.alg,
-                    verify_key.version,
-                    encode_verify_key_base64(verify_key),
-                    str(e),
-                )
-                raise SynapseError(
-                    401,
-                    "Invalid signature for server %s with key %s:%s: %s"
-                    % (
-                        verify_request.server_name,
-                        verify_key.alg,
-                        verify_key.version,
-                        str(e),
-                    ),
-                    Codes.UNAUTHORIZED,
-                )
+            await self._process_json(key_result.verify_key, verify_request)
+            verified = True
 
         if not verified:
             raise SynapseError(
@@ -322,6 +305,39 @@ class Keyring:
                 Codes.UNAUTHORIZED,
             )
 
+    async def _process_json(
+        self, verify_key: VerifyKey, verify_request: VerifyJsonRequest
+    ) -> None:
+        """Processes the `VerifyJsonRequest`. Raises if the signature can't be
+        verified.
+        """
+        try:
+            verify_signed_json(
+                verify_request.get_json_object(),
+                verify_request.server_name,
+                verify_key,
+            )
+        except SignatureVerifyException as e:
+            logger.debug(
+                "Error verifying signature for %s:%s:%s with key %s: %s",
+                verify_request.server_name,
+                verify_key.alg,
+                verify_key.version,
+                encode_verify_key_base64(verify_key),
+                str(e),
+            )
+            raise SynapseError(
+                401,
+                "Invalid signature for server %s with key %s:%s: %s"
+                % (
+                    verify_request.server_name,
+                    verify_key.alg,
+                    verify_key.version,
+                    str(e),
+                ),
+                Codes.UNAUTHORIZED,
+            )
+
     async def _inner_fetch_key_requests(
         self, requests: List[_FetchKeyRequest]
     ) -> Dict[str, Dict[str, FetchKeyResult]]:
diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 745c295d3b..cbecc1c20f 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -197,6 +197,18 @@ class KeyringTestCase(unittest.HomeserverTestCase):
         # self.assertFalse(d.called)
         self.get_success(d)
 
+    def test_verify_for_server_locally(self):
+        """Ensure that locally signed JSON can be verified without fetching keys
+        over federation
+        """
+        kr = keyring.Keyring(self.hs)
+        json1 = {}
+        signedjson.sign.sign_json(json1, self.hs.hostname, self.hs.signing_key)
+
+        # Test that verify_json_for_server succeeds on a object signed by ourselves
+        d = kr.verify_json_for_server(self.hs.hostname, json1, 0)
+        self.get_success(d)
+
     def test_verify_json_for_server_with_null_valid_until_ms(self):
         """Tests that we correctly handle key requests for keys we've stored
         with a null `ts_valid_until_ms`
-- 
cgit 1.5.1


From 0e16b418f6835c7a2a9aae4637b0a9f2ca47f518 Mon Sep 17 00:00:00 2001
From: Rafael Gonçalves <8217676+RafaelGoncalves8@users.noreply.github.com>
Date: Thu, 28 Oct 2021 14:54:38 -0300
Subject: Add knock information in admin exported data (#11171)

Signed-off-by: Rafael Goncalves <rafaelgoncalves@riseup.net>
---
 changelog.d/11171.misc       |  1 +
 synapse/app/admin_cmd.py     | 14 ++++++++++++++
 synapse/handlers/admin.py    | 22 ++++++++++++++++++++++
 tests/handlers/test_admin.py | 35 +++++++++++++++++++++++++++++++++--
 tests/rest/client/utils.py   | 29 +++++++++++++++++++++++++++++
 5 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11171.misc

diff --git a/changelog.d/11171.misc b/changelog.d/11171.misc
new file mode 100644
index 0000000000..b6a41a96da
--- /dev/null
+++ b/changelog.d/11171.misc
@@ -0,0 +1 @@
+Add knock information in admin export. Contributed by Rafael Gonçalves.
diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py
index 2fc848596d..ad20b1d6aa 100644
--- a/synapse/app/admin_cmd.py
+++ b/synapse/app/admin_cmd.py
@@ -145,6 +145,20 @@ class FileExfiltrationWriter(ExfiltrationWriter):
             for event in state.values():
                 print(json.dumps(event), file=f)
 
+    def write_knock(self, room_id, event, state):
+        self.write_events(room_id, [event])
+
+        # We write the knock state somewhere else as they aren't full events
+        # and are only a subset of the state at the event.
+        room_directory = os.path.join(self.base_directory, "rooms", room_id)
+        os.makedirs(room_directory, exist_ok=True)
+
+        knock_state = os.path.join(room_directory, "knock_state")
+
+        with open(knock_state, "a") as f:
+            for event in state.values():
+                print(json.dumps(event), file=f)
+
     def finished(self):
         return self.base_directory
 
diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py
index a53cd62d3c..be3203ac80 100644
--- a/synapse/handlers/admin.py
+++ b/synapse/handlers/admin.py
@@ -90,6 +90,7 @@ class AdminHandler:
                 Membership.LEAVE,
                 Membership.BAN,
                 Membership.INVITE,
+                Membership.KNOCK,
             ),
         )
 
@@ -122,6 +123,13 @@ class AdminHandler:
                         invited_state = invite.unsigned["invite_room_state"]
                         writer.write_invite(room_id, invite, invited_state)
 
+                if room.membership == Membership.KNOCK:
+                    event_id = room.event_id
+                    knock = await self.store.get_event(event_id, allow_none=True)
+                    if knock:
+                        knock_state = knock.unsigned["knock_room_state"]
+                        writer.write_knock(room_id, knock, knock_state)
+
                 continue
 
             # We only want to bother fetching events up to the last time they
@@ -238,6 +246,20 @@ class ExfiltrationWriter(metaclass=abc.ABCMeta):
         """
         raise NotImplementedError()
 
+    @abc.abstractmethod
+    def write_knock(
+        self, room_id: str, event: EventBase, state: StateMap[dict]
+    ) -> None:
+        """Write a knock for the room, with associated knock state.
+
+        Args:
+            room_id: The room ID the knock is for.
+            event: The knock event.
+            state: A subset of the state at the knock, with a subset of the
+                event keys (type, state_key content and sender).
+        """
+        raise NotImplementedError()
+
     @abc.abstractmethod
     def finished(self) -> Any:
         """Called when all data has successfully been exported and written.
diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py
index 59de1142b1..abf2a0fe0d 100644
--- a/tests/handlers/test_admin.py
+++ b/tests/handlers/test_admin.py
@@ -17,8 +17,9 @@ from unittest.mock import Mock
 
 import synapse.rest.admin
 import synapse.storage
-from synapse.api.constants import EventTypes
-from synapse.rest.client import login, room
+from synapse.api.constants import EventTypes, JoinRules
+from synapse.api.room_versions import RoomVersions
+from synapse.rest.client import knock, login, room
 
 from tests import unittest
 
@@ -28,6 +29,7 @@ class ExfiltrateData(unittest.HomeserverTestCase):
         synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
         room.register_servlets,
+        knock.register_servlets,
     ]
 
     def prepare(self, reactor, clock, hs):
@@ -201,3 +203,32 @@ class ExfiltrateData(unittest.HomeserverTestCase):
         self.assertEqual(args[0], room_id)
         self.assertEqual(args[1].content["membership"], "invite")
         self.assertTrue(args[2])  # Assert there is at least one bit of state
+
+    def test_knock(self):
+        """Tests that knock get handled correctly."""
+        # create a knockable v7 room
+        room_id = self.helper.create_room_as(
+            self.user1, room_version=RoomVersions.V7.identifier, tok=self.token1
+        )
+        self.helper.send_state(
+            room_id,
+            EventTypes.JoinRules,
+            {"join_rule": JoinRules.KNOCK},
+            tok=self.token1,
+        )
+
+        self.helper.send(room_id, body="Hello!", tok=self.token1)
+        self.helper.knock(room_id, self.user2, tok=self.token2)
+
+        writer = Mock()
+
+        self.get_success(self.admin_handler.export_user_data(self.user2, writer))
+
+        writer.write_events.assert_not_called()
+        writer.write_state.assert_not_called()
+        writer.write_knock.assert_called_once()
+
+        args = writer.write_knock.call_args[0]
+        self.assertEqual(args[0], room_id)
+        self.assertEqual(args[1].content["membership"], "knock")
+        self.assertTrue(args[2])  # Assert there is at least one bit of state
diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py
index 71fa87ce92..ec0979850b 100644
--- a/tests/rest/client/utils.py
+++ b/tests/rest/client/utils.py
@@ -120,6 +120,35 @@ class RestHelper:
             expect_code=expect_code,
         )
 
+    def knock(self, room=None, user=None, reason=None, expect_code=200, tok=None):
+        temp_id = self.auth_user_id
+        self.auth_user_id = user
+        path = "/knock/%s" % room
+        if tok:
+            path = path + "?access_token=%s" % tok
+
+        data = {}
+        if reason:
+            data["reason"] = reason
+
+        channel = make_request(
+            self.hs.get_reactor(),
+            self.site,
+            "POST",
+            path,
+            json.dumps(data).encode("utf8"),
+        )
+
+        assert (
+            int(channel.result["code"]) == expect_code
+        ), "Expected: %d, got: %d, resp: %r" % (
+            expect_code,
+            int(channel.result["code"]),
+            channel.result["body"],
+        )
+
+        self.auth_user_id = temp_id
+
     def leave(self, room=None, user=None, expect_code=200, tok=None):
         self.change_membership(
             room=room,
-- 
cgit 1.5.1


From 56e281bf6c4f58929d56e3901856f6d0fa4b1816 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Thu, 28 Oct 2021 14:35:12 -0400
Subject: Additional type hints for relations database class. (#11205)

---
 changelog.d/11205.misc                      |  1 +
 mypy.ini                                    |  1 +
 synapse/storage/databases/main/relations.py | 38 +++++++++++++++++------------
 3 files changed, 25 insertions(+), 15 deletions(-)
 create mode 100644 changelog.d/11205.misc

diff --git a/changelog.d/11205.misc b/changelog.d/11205.misc
new file mode 100644
index 0000000000..62395c9432
--- /dev/null
+++ b/changelog.d/11205.misc
@@ -0,0 +1 @@
+Improve type hints for the relations datastore.
diff --git a/mypy.ini b/mypy.ini
index 8f5386c179..119a7d8c91 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -53,6 +53,7 @@ files =
   synapse/storage/databases/main/keys.py,
   synapse/storage/databases/main/pusher.py,
   synapse/storage/databases/main/registration.py,
+  synapse/storage/databases/main/relations.py,
   synapse/storage/databases/main/session.py,
   synapse/storage/databases/main/stream.py,
   synapse/storage/databases/main/ui_auth.py,
diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py
index 40760fbd1b..53576ad52f 100644
--- a/synapse/storage/databases/main/relations.py
+++ b/synapse/storage/databases/main/relations.py
@@ -13,13 +13,14 @@
 # limitations under the License.
 
 import logging
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple, Union
 
 import attr
 
 from synapse.api.constants import RelationTypes
 from synapse.events import EventBase
 from synapse.storage._base import SQLBaseStore
+from synapse.storage.database import LoggingTransaction
 from synapse.storage.databases.main.stream import generate_pagination_where_clause
 from synapse.storage.relations import (
     AggregationPaginationToken,
@@ -63,7 +64,7 @@ class RelationsWorkerStore(SQLBaseStore):
         """
 
         where_clause = ["relates_to_id = ?"]
-        where_args = [event_id]
+        where_args: List[Union[str, int]] = [event_id]
 
         if relation_type is not None:
             where_clause.append("relation_type = ?")
@@ -80,8 +81,8 @@ class RelationsWorkerStore(SQLBaseStore):
         pagination_clause = generate_pagination_where_clause(
             direction=direction,
             column_names=("topological_ordering", "stream_ordering"),
-            from_token=attr.astuple(from_token) if from_token else None,
-            to_token=attr.astuple(to_token) if to_token else None,
+            from_token=attr.astuple(from_token) if from_token else None,  # type: ignore[arg-type]
+            to_token=attr.astuple(to_token) if to_token else None,  # type: ignore[arg-type]
             engine=self.database_engine,
         )
 
@@ -106,7 +107,9 @@ class RelationsWorkerStore(SQLBaseStore):
             order,
         )
 
-        def _get_recent_references_for_event_txn(txn):
+        def _get_recent_references_for_event_txn(
+            txn: LoggingTransaction,
+        ) -> PaginationChunk:
             txn.execute(sql, where_args + [limit + 1])
 
             last_topo_id = None
@@ -160,7 +163,7 @@ class RelationsWorkerStore(SQLBaseStore):
         """
 
         where_clause = ["relates_to_id = ?", "relation_type = ?"]
-        where_args = [event_id, RelationTypes.ANNOTATION]
+        where_args: List[Union[str, int]] = [event_id, RelationTypes.ANNOTATION]
 
         if event_type:
             where_clause.append("type = ?")
@@ -169,8 +172,8 @@ class RelationsWorkerStore(SQLBaseStore):
         having_clause = generate_pagination_where_clause(
             direction=direction,
             column_names=("COUNT(*)", "MAX(stream_ordering)"),
-            from_token=attr.astuple(from_token) if from_token else None,
-            to_token=attr.astuple(to_token) if to_token else None,
+            from_token=attr.astuple(from_token) if from_token else None,  # type: ignore[arg-type]
+            to_token=attr.astuple(to_token) if to_token else None,  # type: ignore[arg-type]
             engine=self.database_engine,
         )
 
@@ -199,7 +202,9 @@ class RelationsWorkerStore(SQLBaseStore):
             having_clause=having_clause,
         )
 
-        def _get_aggregation_groups_for_event_txn(txn):
+        def _get_aggregation_groups_for_event_txn(
+            txn: LoggingTransaction,
+        ) -> PaginationChunk:
             txn.execute(sql, where_args + [limit + 1])
 
             next_batch = None
@@ -254,11 +259,12 @@ class RelationsWorkerStore(SQLBaseStore):
             LIMIT 1
         """
 
-        def _get_applicable_edit_txn(txn):
+        def _get_applicable_edit_txn(txn: LoggingTransaction) -> Optional[str]:
             txn.execute(sql, (event_id, RelationTypes.REPLACE))
             row = txn.fetchone()
             if row:
                 return row[0]
+            return None
 
         edit_id = await self.db_pool.runInteraction(
             "get_applicable_edit", _get_applicable_edit_txn
@@ -267,7 +273,7 @@ class RelationsWorkerStore(SQLBaseStore):
         if not edit_id:
             return None
 
-        return await self.get_event(edit_id, allow_none=True)
+        return await self.get_event(edit_id, allow_none=True)  # type: ignore[attr-defined]
 
     @cached()
     async def get_thread_summary(
@@ -283,7 +289,9 @@ class RelationsWorkerStore(SQLBaseStore):
             The number of items in the thread and the most recent response, if any.
         """
 
-        def _get_thread_summary_txn(txn) -> Tuple[int, Optional[str]]:
+        def _get_thread_summary_txn(
+            txn: LoggingTransaction,
+        ) -> Tuple[int, Optional[str]]:
             # Fetch the count of threaded events and the latest event ID.
             # TODO Should this only allow m.room.message events.
             sql = """
@@ -312,7 +320,7 @@ class RelationsWorkerStore(SQLBaseStore):
                     AND relation_type = ?
             """
             txn.execute(sql, (event_id, RelationTypes.THREAD))
-            count = txn.fetchone()[0]
+            count = txn.fetchone()[0]  # type: ignore[index]
 
             return count, latest_event_id
 
@@ -322,7 +330,7 @@ class RelationsWorkerStore(SQLBaseStore):
 
         latest_event = None
         if latest_event_id:
-            latest_event = await self.get_event(latest_event_id, allow_none=True)
+            latest_event = await self.get_event(latest_event_id, allow_none=True)  # type: ignore[attr-defined]
 
         return count, latest_event
 
@@ -354,7 +362,7 @@ class RelationsWorkerStore(SQLBaseStore):
             LIMIT 1;
         """
 
-        def _get_if_user_has_annotated_event(txn):
+        def _get_if_user_has_annotated_event(txn: LoggingTransaction) -> bool:
             txn.execute(
                 sql,
                 (
-- 
cgit 1.5.1


From 3ed17ff651a39d64288c38cd56ecef3603a31d04 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Fri, 29 Oct 2021 14:03:58 +0100
Subject: Clarify lack of Windows support in documentation (#11198)

---
 changelog.d/11198.doc                  |  1 +
 docs/development/contributing_guide.md | 11 +++++------
 docs/setup/installation.md             | 14 ++++++++------
 3 files changed, 14 insertions(+), 12 deletions(-)
 create mode 100644 changelog.d/11198.doc

diff --git a/changelog.d/11198.doc b/changelog.d/11198.doc
new file mode 100644
index 0000000000..54ec94acbc
--- /dev/null
+++ b/changelog.d/11198.doc
@@ -0,0 +1 @@
+Clarify lack of support for Windows.
diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md
index 3bf08a72bb..abdb808438 100644
--- a/docs/development/contributing_guide.md
+++ b/docs/development/contributing_guide.md
@@ -15,6 +15,11 @@ license - in our case, this is almost always Apache Software License v2 (see
 
 # 2. What do I need?
 
+If you are running Windows, the Windows Subsystem for Linux (WSL) is strongly
+recommended for development. More information about WSL can be found at
+<https://docs.microsoft.com/en-us/windows/wsl/install>. Running Synapse natively
+on Windows is not officially supported.
+
 The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
 
 The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
@@ -41,8 +46,6 @@ can find many good git tutorials on the web.
 
 # 4. Install the dependencies
 
-## Under Unix (macOS, Linux, BSD, ...)
-
 Once you have installed Python 3 and added the source, please open a terminal and
 setup a *virtualenv*, as follows:
 
@@ -56,10 +59,6 @@ pip install tox
 
 This will install the developer dependencies for the project.
 
-## Under Windows
-
-TBD
-
 
 # 5. Get in touch.
 
diff --git a/docs/setup/installation.md b/docs/setup/installation.md
index 874925e927..3e08024441 100644
--- a/docs/setup/installation.md
+++ b/docs/setup/installation.md
@@ -356,12 +356,14 @@ make install
 
 ##### Windows
 
-If you wish to run or develop Synapse on Windows, the Windows Subsystem For
-Linux provides a Linux environment on Windows 10 which is capable of using the
-Debian, Fedora, or source installation methods. More information about WSL can
-be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
-Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
-for Windows Server.
+Running Synapse natively on Windows is not officially supported.
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem for
+Linux provides a Linux environment which is capable of using the Debian, Fedora,
+or source installation methods. More information about WSL can be found at
+<https://docs.microsoft.com/en-us/windows/wsl/install> for Windows 10/11 and
+<https://docs.microsoft.com/en-us/windows/wsl/install-on-server> for
+Windows Server.
 
 ## Setting up Synapse
 
-- 
cgit 1.5.1


From ad4eab9862348fff16d66954930c0f8c3feae6e1 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Fri, 29 Oct 2021 18:28:29 +0200
Subject: Add a module API method to retrieve state from a room (#11204)

---
 changelog.d/11204.feature      |  1 +
 synapse/module_api/__init__.py | 49 ++++++++++++++++++++++++++++++++++++++++++
 tests/module_api/test_api.py   | 25 ++++++++++++++++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11204.feature

diff --git a/changelog.d/11204.feature b/changelog.d/11204.feature
new file mode 100644
index 0000000000..f58ed4b3dc
--- /dev/null
+++ b/changelog.d/11204.feature
@@ -0,0 +1 @@
+Add a module API method to retrieve the current state of a room.
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index 36042ed2e0..6e7f5238fe 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -55,6 +55,7 @@ from synapse.types import (
     DomainSpecificString,
     JsonDict,
     Requester,
+    StateMap,
     UserID,
     UserInfo,
     create_requester,
@@ -89,6 +90,8 @@ __all__ = [
     "PRESENCE_ALL_USERS",
     "LoginResponse",
     "JsonDict",
+    "EventBase",
+    "StateMap",
 ]
 
 logger = logging.getLogger(__name__)
@@ -964,6 +967,52 @@ class ModuleApi:
         else:
             return []
 
+    async def get_room_state(
+        self,
+        room_id: str,
+        event_filter: Optional[Iterable[Tuple[str, Optional[str]]]] = None,
+    ) -> StateMap[EventBase]:
+        """Returns the current state of the given room.
+
+        The events are returned as a mapping, in which the key for each event is a tuple
+        which first element is the event's type and the second one is its state key.
+
+        Added in Synapse v1.47.0
+
+        Args:
+            room_id: The ID of the room to get state from.
+            event_filter: A filter to apply when retrieving events. None if no filter
+                should be applied. If provided, must be an iterable of tuples. A tuple's
+                first element is the event type and the second is the state key, or is
+                None if the state key should not be filtered on.
+                An example of a filter is:
+                    [
+                        ("m.room.member", "@alice:example.com"),  # Member event for @alice:example.com
+                        ("org.matrix.some_event", ""),  # State event of type "org.matrix.some_event"
+                                                        # with an empty string as its state key
+                        ("org.matrix.some_other_event", None),  # State events of type "org.matrix.some_other_event"
+                                                                # regardless of their state key
+                    ]
+        """
+        if event_filter:
+            # If a filter was provided, turn it into a StateFilter and retrieve a filtered
+            # view of the state.
+            state_filter = StateFilter.from_types(event_filter)
+            state_ids = await self._store.get_filtered_current_state_ids(
+                room_id,
+                state_filter,
+            )
+        else:
+            # If no filter was provided, get the whole state. We could also reuse the call
+            # to get_filtered_current_state_ids above, with `state_filter = StateFilter.all()`,
+            # but get_filtered_current_state_ids isn't cached and `get_current_state_ids`
+            # is, so using the latter when we can is better for perf.
+            state_ids = await self._store.get_current_state_ids(room_id)
+
+        state_events = await self._store.get_events(state_ids.values())
+
+        return {key: state_events[event_id] for key, event_id in state_ids.items()}
+
 
 class PublicRoomListManager:
     """Contains methods for adding to, removing from and querying whether a room
diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py
index 37852852a8..525b83141b 100644
--- a/tests/module_api/test_api.py
+++ b/tests/module_api/test_api.py
@@ -15,7 +15,7 @@ from unittest.mock import Mock
 
 from twisted.internet import defer
 
-from synapse.api.constants import EduTypes
+from synapse.api.constants import EduTypes, EventTypes
 from synapse.events import EventBase
 from synapse.federation.units import Transaction
 from synapse.handlers.presence import UserPresenceState
@@ -509,6 +509,29 @@ class ModuleApiTestCase(HomeserverTestCase):
         self.assertEqual(res["displayname"], "simone")
         self.assertIsNone(res["avatar_url"])
 
+    def test_get_room_state(self):
+        """Tests that a module can retrieve the state of a room through the module API."""
+        user_id = self.register_user("peter", "hackme")
+        tok = self.login("peter", "hackme")
+
+        # Create a room and send some custom state in it.
+        room_id = self.helper.create_room_as(tok=tok)
+        self.helper.send_state(room_id, "org.matrix.test", {}, tok=tok)
+
+        # Check that the module API can successfully fetch state for the room.
+        state = self.get_success(
+            defer.ensureDeferred(self.module_api.get_room_state(room_id))
+        )
+
+        # Check that a few standard events are in the returned state.
+        self.assertIn((EventTypes.Create, ""), state)
+        self.assertIn((EventTypes.Member, user_id), state)
+
+        # Check that our custom state event is in the returned state.
+        self.assertEqual(state[("org.matrix.test", "")].sender, user_id)
+        self.assertEqual(state[("org.matrix.test", "")].state_key, "")
+        self.assertEqual(state[("org.matrix.test", "")].content, {})
+
 
 class ModuleApiWorkerTestCase(BaseMultiWorkerStreamTestCase):
     """For testing ModuleApi functionality in a multi-worker setup"""
-- 
cgit 1.5.1


From bfd7a9b65c5e092c6a7ccdd46e59a278b1cbbd57 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Fri, 29 Oct 2021 19:43:51 +0200
Subject: Fix comments referencing v1.46.0 from PR #10969. (#11212)

#10969 was merged after 1.46.0rc1 was cut and will be included
in v1.47.0rc1 instead.
---
 changelog.d/11212.bugfix                                                | 1 +
 synapse/storage/databases/main/deviceinbox.py                           | 2 +-
 .../schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11212.bugfix

diff --git a/changelog.d/11212.bugfix b/changelog.d/11212.bugfix
new file mode 100644
index 0000000000..ba6efab25b
--- /dev/null
+++ b/changelog.d/11212.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where messages in the `device_inbox` table for deleted devices would persist indefinitely. Contributed by @dklimpel and @JohannesKleine.
\ No newline at end of file
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index b0ccab0c9b..d03b5e5a7d 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -594,7 +594,7 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
     ) -> int:
         """A background update that deletes all device_inboxes for deleted devices.
 
-        This should only need to be run once (when users upgrade to v1.46.0)
+        This should only need to be run once (when users upgrade to v1.47.0)
 
         Args:
             progress: JsonDict used to store progress of this background update
diff --git a/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql b/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql
index efe702f621..fca7290741 100644
--- a/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql
+++ b/synapse/storage/schema/main/delta/64/02remove_deleted_devices_from_device_inbox.sql
@@ -15,7 +15,7 @@
 
 
 -- Remove messages from the device_inbox table which were orphaned
--- when a device was deleted using Synapse earlier than 1.46.0.
+-- when a device was deleted using Synapse earlier than 1.47.0.
 -- This runs as background task, but may take a bit to finish.
 
 INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
-- 
cgit 1.5.1


From e320f5dba32f5b5818b6d5a0059ae388430b9a72 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Mon, 1 Nov 2021 10:46:08 +0100
Subject: Deprecate user_may_create_room_with_invites (#11206)

---
 changelog.d/11206.removal              |  1 +
 docs/modules/spam_checker_callbacks.md | 36 ----------------------------------
 docs/upgrade.md                        | 13 ++++++++++++
 3 files changed, 14 insertions(+), 36 deletions(-)
 create mode 100644 changelog.d/11206.removal

diff --git a/changelog.d/11206.removal b/changelog.d/11206.removal
new file mode 100644
index 0000000000..cf05b16672
--- /dev/null
+++ b/changelog.d/11206.removal
@@ -0,0 +1 @@
+The `user_may_create_room_with_invites` module callback is now deprecated. Please refer to the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1470) for more information.
diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md
index 534ea196e0..2eb9032f41 100644
--- a/docs/modules/spam_checker_callbacks.md
+++ b/docs/modules/spam_checker_callbacks.md
@@ -123,42 +123,6 @@ callback returns `True`, Synapse falls through to the next one. The value of the
 callback that does not return `True` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.
 
-### `user_may_create_room_with_invites`
-
-_First introduced in Synapse v1.44.0_
-
-```python
-async def user_may_create_room_with_invites(
-    user: str,
-    invites: List[str],
-    threepid_invites: List[Dict[str, str]],
-) -> bool
-```
-
-Called when processing a room creation request (right after `user_may_create_room`).
-The module is given the Matrix user ID of the user trying to create a room, as well as a
-list of Matrix users to invite and a list of third-party identifiers (3PID, e.g. email
-addresses) to invite.
-
-An invited Matrix user to invite is represented by their Matrix user IDs, and an invited
-3PIDs is represented by a dict that includes the 3PID medium (e.g. "email") through its
-`medium` key and its address (e.g. "alice@example.com") through its `address` key.
-
-See [the Matrix specification](https://matrix.org/docs/spec/appendices#pid-types) for more
-information regarding third-party identifiers.
-
-If no invite and/or 3PID invite were specified in the room creation request, the
-corresponding list(s) will be empty.
-
-**Note**: This callback is not called when a room is cloned (e.g. during a room upgrade)
-since no invites are sent when cloning a room. To cover this case, modules also need to
-implement `user_may_create_room`.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
-
 ### `user_may_create_room_alias`
 
 _First introduced in Synapse v1.37.0_
diff --git a/docs/upgrade.md b/docs/upgrade.md
index c47eef1a20..d32d1ab988 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -85,6 +85,19 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
+# Upgrading to v1.47.0
+
+## Deprecation of the `user_may_create_room_with_invites` module callback
+
+The `user_may_create_room_with_invites` is deprecated and will be removed in a future
+version of Synapse. Modules implementing this callback can instead implement
+[`user_may_invite`](https://matrix-org.github.io/synapse/latest/modules/spam_checker_callbacks.html#user_may_invite)
+and use the [`get_room_state`](https://github.com/matrix-org/synapse/blob/872f23b95fa980a61b0866c1475e84491991fa20/synapse/module_api/__init__.py#L869-L876)
+module API method to infer whether the invite is happening in the context of creating a
+room.
+
+We plan to remove this callback in January 2022.
+
 # Upgrading to v1.45.0
 
 ## Changes required to media storage provider modules when reading from the Synapse configuration object
-- 
cgit 1.5.1


From 29ffd680bf0d0bf50383ad23404b348bf9cf90aa Mon Sep 17 00:00:00 2001
From: JohannesKleine <linux@cj-k.de>
Date: Mon, 1 Nov 2021 11:40:41 +0100
Subject: Stop synapse from saving messages in device_inbox for hidden devices.
 (#10097)

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
---
 changelog.d/10097.bugfix                      | 1 +
 synapse/storage/databases/main/deviceinbox.py | 8 ++++++--
 2 files changed, 7 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/10097.bugfix

diff --git a/changelog.d/10097.bugfix b/changelog.d/10097.bugfix
new file mode 100644
index 0000000000..5d3d9587c2
--- /dev/null
+++ b/changelog.d/10097.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug which allowed hidden devices to receive to-device messages, resulting in unnecessary database bloat.
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index d03b5e5a7d..25e9c1efe1 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -489,10 +489,12 @@ class DeviceInboxWorkerStore(SQLBaseStore):
             devices = list(messages_by_device.keys())
             if len(devices) == 1 and devices[0] == "*":
                 # Handle wildcard device_ids.
+                # We exclude hidden devices (such as cross-signing keys) here as they are
+                # not expected to receive to-device messages.
                 devices = self.db_pool.simple_select_onecol_txn(
                     txn,
                     table="devices",
-                    keyvalues={"user_id": user_id},
+                    keyvalues={"user_id": user_id, "hidden": False},
                     retcol="device_id",
                 )
 
@@ -505,10 +507,12 @@ class DeviceInboxWorkerStore(SQLBaseStore):
                 if not devices:
                     continue
 
+                # We exclude hidden devices (such as cross-signing keys) here as they are
+                # not expected to receive to-device messages.
                 rows = self.db_pool.simple_select_many_txn(
                     txn,
                     table="devices",
-                    keyvalues={"user_id": user_id},
+                    keyvalues={"user_id": user_id, "hidden": False},
                     column="device_id",
                     iterable=devices,
                     retcols=("device_id",),
-- 
cgit 1.5.1


From 2451003f6fff3dbea8e0e5c8bf6a91ed26c97cb7 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Mon, 1 Nov 2021 11:20:54 +0000
Subject: Test that `ClientIpStore` combines database and in-memory data
 correctly (#11179)

---
 changelog.d/11179.misc           |   1 +
 tests/storage/test_client_ips.py | 206 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 207 insertions(+)
 create mode 100644 changelog.d/11179.misc

diff --git a/changelog.d/11179.misc b/changelog.d/11179.misc
new file mode 100644
index 0000000000..aded2e8367
--- /dev/null
+++ b/changelog.d/11179.misc
@@ -0,0 +1 @@
+Add tests to check that `ClientIpStore.get_last_client_ip_by_device` and `get_user_ip_and_agents` combine database and in-memory data correctly.
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 0e4013ebea..c8ac67e35b 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -20,6 +20,7 @@ from parameterized import parameterized
 import synapse.rest.admin
 from synapse.http.site import XForwardedForRequest
 from synapse.rest.client import login
+from synapse.storage.databases.main.client_ips import LAST_SEEN_GRANULARITY
 from synapse.types import UserID
 
 from tests import unittest
@@ -171,6 +172,27 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
         if after_persisting:
             # Trigger the storage loop
             self.reactor.advance(10)
+        else:
+            # Check that the new IP and user agent has not been stored yet
+            db_result = self.get_success(
+                self.store.db_pool.simple_select_list(
+                    table="devices",
+                    keyvalues={},
+                    retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
+                ),
+            )
+            self.assertEqual(
+                db_result,
+                [
+                    {
+                        "user_id": user_id,
+                        "device_id": device_id,
+                        "ip": None,
+                        "user_agent": None,
+                        "last_seen": None,
+                    },
+                ],
+            )
 
         result = self.get_success(
             self.store.get_last_client_ip_by_device(user_id, device_id)
@@ -189,6 +211,104 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
             },
         )
 
+    def test_get_last_client_ip_by_device_combined_data(self):
+        """Test that `get_last_client_ip_by_device` combines persisted and unpersisted
+        data together correctly
+        """
+        self.reactor.advance(12345678)
+
+        user_id = "@user:id"
+        device_id_1 = "MY_DEVICE_1"
+        device_id_2 = "MY_DEVICE_2"
+
+        # Insert user IPs
+        self.get_success(
+            self.store.store_device(
+                user_id,
+                device_id_1,
+                "display name",
+            )
+        )
+        self.get_success(
+            self.store.store_device(
+                user_id,
+                device_id_2,
+                "display name",
+            )
+        )
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token_1", "ip_1", "user_agent_1", device_id_1
+            )
+        )
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token_2", "ip_2", "user_agent_2", device_id_2
+            )
+        )
+
+        # Trigger the storage loop and wait for the rate limiting period to be over
+        self.reactor.advance(10 + LAST_SEEN_GRANULARITY / 1000)
+
+        # Update the user agent for the second device, without running the storage loop
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token_2", "ip_2", "user_agent_3", device_id_2
+            )
+        )
+
+        # Check that the new IP and user agent has not been stored yet
+        db_result = self.get_success(
+            self.store.db_pool.simple_select_list(
+                table="devices",
+                keyvalues={},
+                retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"),
+            ),
+        )
+        self.assertCountEqual(
+            db_result,
+            [
+                {
+                    "user_id": user_id,
+                    "device_id": device_id_1,
+                    "ip": "ip_1",
+                    "user_agent": "user_agent_1",
+                    "last_seen": 12345678000,
+                },
+                {
+                    "user_id": user_id,
+                    "device_id": device_id_2,
+                    "ip": "ip_2",
+                    "user_agent": "user_agent_2",
+                    "last_seen": 12345678000,
+                },
+            ],
+        )
+
+        # Check that data from the database and memory are combined together correctly
+        result = self.get_success(
+            self.store.get_last_client_ip_by_device(user_id, None)
+        )
+        self.assertEqual(
+            result,
+            {
+                (user_id, device_id_1): {
+                    "user_id": user_id,
+                    "device_id": device_id_1,
+                    "ip": "ip_1",
+                    "user_agent": "user_agent_1",
+                    "last_seen": 12345678000,
+                },
+                (user_id, device_id_2): {
+                    "user_id": user_id,
+                    "device_id": device_id_2,
+                    "ip": "ip_2",
+                    "user_agent": "user_agent_3",
+                    "last_seen": 12345688000 + LAST_SEEN_GRANULARITY,
+                },
+            },
+        )
+
     @parameterized.expand([(False,), (True,)])
     def test_get_user_ip_and_agents(self, after_persisting: bool):
         """Test `get_user_ip_and_agents` for persisted and unpersisted data"""
@@ -207,6 +327,16 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
         if after_persisting:
             # Trigger the storage loop
             self.reactor.advance(10)
+        else:
+            # Check that the new IP and user agent has not been stored yet
+            db_result = self.get_success(
+                self.store.db_pool.simple_select_list(
+                    table="user_ips",
+                    keyvalues={},
+                    retcols=("access_token", "ip", "user_agent", "last_seen"),
+                ),
+            )
+            self.assertEqual(db_result, [])
 
         self.assertEqual(
             self.get_success(self.store.get_user_ip_and_agents(user)),
@@ -220,6 +350,82 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
             ],
         )
 
+    def test_get_user_ip_and_agents_combined_data(self):
+        """Test that `get_user_ip_and_agents` combines persisted and unpersisted data
+        together correctly
+        """
+        self.reactor.advance(12345678)
+
+        user_id = "@user:id"
+        user = UserID.from_string(user_id)
+
+        # Insert user IPs
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip_1", "user_agent_1", "MY_DEVICE_1"
+            )
+        )
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip_2", "user_agent_2", "MY_DEVICE_2"
+            )
+        )
+
+        # Trigger the storage loop and wait for the rate limiting period to be over
+        self.reactor.advance(10 + LAST_SEEN_GRANULARITY / 1000)
+
+        # Update the user agent for the second device, without running the storage loop
+        self.get_success(
+            self.store.insert_client_ip(
+                user_id, "access_token", "ip_2", "user_agent_3", "MY_DEVICE_2"
+            )
+        )
+
+        # Check that the new IP and user agent has not been stored yet
+        db_result = self.get_success(
+            self.store.db_pool.simple_select_list(
+                table="user_ips",
+                keyvalues={},
+                retcols=("access_token", "ip", "user_agent", "last_seen"),
+            ),
+        )
+        self.assertEqual(
+            db_result,
+            [
+                {
+                    "access_token": "access_token",
+                    "ip": "ip_1",
+                    "user_agent": "user_agent_1",
+                    "last_seen": 12345678000,
+                },
+                {
+                    "access_token": "access_token",
+                    "ip": "ip_2",
+                    "user_agent": "user_agent_2",
+                    "last_seen": 12345678000,
+                },
+            ],
+        )
+
+        # Check that data from the database and memory are combined together correctly
+        self.assertCountEqual(
+            self.get_success(self.store.get_user_ip_and_agents(user)),
+            [
+                {
+                    "access_token": "access_token",
+                    "ip": "ip_1",
+                    "user_agent": "user_agent_1",
+                    "last_seen": 12345678000,
+                },
+                {
+                    "access_token": "access_token",
+                    "ip": "ip_2",
+                    "user_agent": "user_agent_3",
+                    "last_seen": 12345688000 + LAST_SEEN_GRANULARITY,
+                },
+            ],
+        )
+
     @override_config({"limit_usage_by_mau": False, "max_mau_value": 50})
     def test_disabled_monthly_active_user(self):
         user_id = "@user:server"
-- 
cgit 1.5.1


From 82d2168a15741ed4546c12c06d797627469fb684 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 1 Nov 2021 11:21:36 +0000
Subject:  Add metrics to the threadpools (#11178)

---
 changelog.d/11178.feature   |  1 +
 synapse/app/_base.py        |  5 +++++
 synapse/metrics/__init__.py | 37 +++++++++++++++++++++++++++++++++++++
 synapse/storage/database.py |  7 ++++++-
 4 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11178.feature

diff --git a/changelog.d/11178.feature b/changelog.d/11178.feature
new file mode 100644
index 0000000000..10b1cdffdc
--- /dev/null
+++ b/changelog.d/11178.feature
@@ -0,0 +1 @@
+Add metrics for thread pool usage.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index f4c3f867a8..f2c1028b5d 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -45,6 +45,7 @@ from synapse.events.spamcheck import load_legacy_spam_checkers
 from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.handlers.auth import load_legacy_password_auth_providers
 from synapse.logging.context import PreserveLoggingContext
+from synapse.metrics import register_threadpool
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.metrics.jemalloc import setup_jemalloc_stats
 from synapse.util.caches.lrucache import setup_expire_lru_cache_entries
@@ -351,6 +352,10 @@ async def start(hs: "HomeServer"):
         GAIResolver(reactor, getThreadPool=lambda: resolver_threadpool)
     )
 
+    # Register the threadpools with our metrics.
+    register_threadpool("default", reactor.getThreadPool())
+    register_threadpool("gai_resolver", resolver_threadpool)
+
     # Set up the SIGHUP machinery.
     if hasattr(signal, "SIGHUP"):
 
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e902109af3..91ee5c8193 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -32,6 +32,7 @@ from prometheus_client.core import (
 )
 
 from twisted.internet import reactor
+from twisted.python.threadpool import ThreadPool
 
 import synapse
 from synapse.metrics._exposition import (
@@ -526,6 +527,42 @@ threepid_send_requests = Histogram(
     labelnames=("type", "reason"),
 )
 
+threadpool_total_threads = Gauge(
+    "synapse_threadpool_total_threads",
+    "Total number of threads currently in the threadpool",
+    ["name"],
+)
+
+threadpool_total_working_threads = Gauge(
+    "synapse_threadpool_working_threads",
+    "Number of threads currently working in the threadpool",
+    ["name"],
+)
+
+threadpool_total_min_threads = Gauge(
+    "synapse_threadpool_min_threads",
+    "Minimum number of threads configured in the threadpool",
+    ["name"],
+)
+
+threadpool_total_max_threads = Gauge(
+    "synapse_threadpool_max_threads",
+    "Maximum number of threads configured in the threadpool",
+    ["name"],
+)
+
+
+def register_threadpool(name: str, threadpool: ThreadPool) -> None:
+    """Add metrics for the threadpool."""
+
+    threadpool_total_min_threads.labels(name).set(threadpool.min)
+    threadpool_total_max_threads.labels(name).set(threadpool.max)
+
+    threadpool_total_threads.labels(name).set_function(lambda: len(threadpool.threads))
+    threadpool_total_working_threads.labels(name).set_function(
+        lambda: len(threadpool.working)
+    )
+
 
 class ReactorLastSeenMetric:
     def collect(self):
diff --git a/synapse/storage/database.py b/synapse/storage/database.py
index fa4e89d35c..5c71e27518 100644
--- a/synapse/storage/database.py
+++ b/synapse/storage/database.py
@@ -48,6 +48,7 @@ from synapse.logging.context import (
     current_context,
     make_deferred_yieldable,
 )
+from synapse.metrics import register_threadpool
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.background_updates import BackgroundUpdater
 from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
@@ -104,13 +105,17 @@ def make_pool(
                 LoggingDatabaseConnection(conn, engine, "on_new_connection")
             )
 
-    return adbapi.ConnectionPool(
+    connection_pool = adbapi.ConnectionPool(
         db_config.config["name"],
         cp_reactor=reactor,
         cp_openfun=_on_new_connection,
         **db_args,
     )
 
+    register_threadpool(f"database-{db_config.name}", connection_pool.threadpool)
+
+    return connection_pool
+
 
 def make_conn(
     db_config: DatabaseConnectionConfig,
-- 
cgit 1.5.1


From ece84f2c450d986e54acc80971225fb02f4e1d05 Mon Sep 17 00:00:00 2001
From: Sumner Evans <me@sumnerevans.com>
Date: Mon, 1 Nov 2021 05:35:55 -0600
Subject: Improve code formatting and fix a few typos in docs (#11221)

* Labeled a lot more code blocks with the appropriate type
* Fixed a couple of minor typos (missing/extraneous commas)

Signed-off-by: Sumner Evans <me@sumnerevans.com>
---
 changelog.d/11221.doc                            |   1 +
 docs/CAPTCHA_SETUP.md                            |   4 +-
 docs/admin_api/event_reports.md                  |   4 +-
 docs/admin_api/purge_history_api.md              |   2 +-
 docs/admin_api/room_membership.md                |   2 +-
 docs/admin_api/rooms.md                          |   8 +-
 docs/code_style.md                               |  78 ++++++++++-------
 docs/consent_tracking.md                         |   2 +-
 docs/delegate.md                                 |   2 +-
 docs/development/cas.md                          |   8 +-
 docs/development/database_schema.md              |   4 +-
 docs/development/saml.md                         |   2 +-
 docs/message_retention_policies.md               |  26 +++---
 docs/modules/password_auth_provider_callbacks.md |   2 +-
 docs/postgres.md                                 |  50 +++++++----
 docs/reverse_proxy.md                            |   6 +-
 docs/synctl_workers.md                           |   8 +-
 docs/turn-howto.md                               |  70 +++++++++------
 docs/upgrade.md                                  | 104 +++++++++++++----------
 docs/workers.md                                  |  18 ++--
 20 files changed, 233 insertions(+), 168 deletions(-)
 create mode 100644 changelog.d/11221.doc

diff --git a/changelog.d/11221.doc b/changelog.d/11221.doc
new file mode 100644
index 0000000000..17010bac8b
--- /dev/null
+++ b/changelog.d/11221.doc
@@ -0,0 +1 @@
+Improve code formatting and fix a few typos in docs. Contributed by @sumnerevans at Beeper.
diff --git a/docs/CAPTCHA_SETUP.md b/docs/CAPTCHA_SETUP.md
index fabdd7b726..49419ce8df 100644
--- a/docs/CAPTCHA_SETUP.md
+++ b/docs/CAPTCHA_SETUP.md
@@ -15,12 +15,12 @@ in `homeserver.yaml`, to the list of authorized domains. If you have not set
 1. Agree to the terms of service and submit.
 1. Copy your site key and secret key and add them to your `homeserver.yaml`
 configuration file
-    ```
+    ```yaml
     recaptcha_public_key: YOUR_SITE_KEY
     recaptcha_private_key: YOUR_SECRET_KEY
     ```
 1. Enable the CAPTCHA for new registrations
-    ```
+    ```yaml
     enable_registration_captcha: true
     ```
 1. Go to the settings page for the CAPTCHA you just created
diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md
index 3abb06099c..f523774ba8 100644
--- a/docs/admin_api/event_reports.md
+++ b/docs/admin_api/event_reports.md
@@ -99,7 +99,7 @@ server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
-```jsonc
+```json
 {
     "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
     "event_json": {
@@ -132,7 +132,7 @@ It returns a JSON body like the following:
         },
         "type": "m.room.message",
         "unsigned": {
-            "age_ts": 1592291711430,
+            "age_ts": 1592291711430
         }
     },
     "id": <report_id>,
diff --git a/docs/admin_api/purge_history_api.md b/docs/admin_api/purge_history_api.md
index 13b991eacf..bd29e29ab8 100644
--- a/docs/admin_api/purge_history_api.md
+++ b/docs/admin_api/purge_history_api.md
@@ -27,7 +27,7 @@ Room state data (such as joins, leaves, topic) is always preserved.
 
 To delete local message events as well, set `delete_local_events` in the body:
 
-```
+```json
 {
    "delete_local_events": true
 }
diff --git a/docs/admin_api/room_membership.md b/docs/admin_api/room_membership.md
index 8a5ce191df..548b790a5c 100644
--- a/docs/admin_api/room_membership.md
+++ b/docs/admin_api/room_membership.md
@@ -28,7 +28,7 @@ server admin: see [Admin API](../usage/administration/admin_api).
 
 Response:
 
-```
+```json
 {
   "room_id": "!636q39766251:server.com"
 }
diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md
index 8e524e6509..acf1cab2a2 100644
--- a/docs/admin_api/rooms.md
+++ b/docs/admin_api/rooms.md
@@ -87,7 +87,7 @@ GET /_synapse/admin/v1/rooms
 
 A response body like the following is returned:
 
-```jsonc
+```json
 {
   "rooms": [
     {
@@ -170,7 +170,7 @@ GET /_synapse/admin/v1/rooms?order_by=size
 
 A response body like the following is returned:
 
-```jsonc
+```json
 {
   "rooms": [
     {
@@ -208,7 +208,7 @@ A response body like the following is returned:
     }
   ],
   "offset": 0,
-  "total_rooms": 150
+  "total_rooms": 150,
   "next_token": 100
 }
 ```
@@ -224,7 +224,7 @@ GET /_synapse/admin/v1/rooms?order_by=size&from=100
 
 A response body like the following is returned:
 
-```jsonc
+```json
 {
   "rooms": [
     {
diff --git a/docs/code_style.md b/docs/code_style.md
index 28fb7277c4..4d8e7c973d 100644
--- a/docs/code_style.md
+++ b/docs/code_style.md
@@ -10,7 +10,9 @@ The necessary tools are detailed below.
 
 First install them with:
 
-    pip install -e ".[lint,mypy]"
+```sh
+pip install -e ".[lint,mypy]"
+```
 
 -   **black**
 
@@ -21,7 +23,9 @@ First install them with:
     Have `black` auto-format your code (it shouldn't change any
     functionality) with:
 
-        black . --exclude="\.tox|build|env"
+    ```sh
+    black . --exclude="\.tox|build|env"
+    ```
 
 -   **flake8**
 
@@ -30,7 +34,9 @@ First install them with:
 
     Check all application and test code with:
 
-        flake8 synapse tests
+    ```sh
+    flake8 synapse tests
+    ```
 
 -   **isort**
 
@@ -39,7 +45,9 @@ First install them with:
 
     Auto-fix imports with:
 
-        isort -rc synapse tests
+    ```sh
+    isort -rc synapse tests
+    ```
 
     `-rc` means to recursively search the given directories.
 
@@ -66,15 +74,19 @@ save as it takes a while and is very resource intensive.
 
         Example:
 
-            from synapse.types import UserID
-            ...
-            user_id = UserID(local, server)
+        ```python
+        from synapse.types import UserID
+        ...
+        user_id = UserID(local, server)
+        ```
 
         is preferred over:
 
-            from synapse import types
-            ...
-            user_id = types.UserID(local, server)
+        ```python
+        from synapse import types
+        ...
+        user_id = types.UserID(local, server)
+        ```
 
         (or any other variant).
 
@@ -134,28 +146,30 @@ Some guidelines follow:
 
 Example:
 
-    ## Frobnication ##
-
-    # The frobnicator will ensure that all requests are fully frobnicated.
-    # To enable it, uncomment the following.
-    #
-    #frobnicator_enabled: true
-
-    # By default, the frobnicator will frobnicate with the default frobber.
-    # The following will make it use an alternative frobber.
-    #
-    #frobincator_frobber: special_frobber
-
-    # Settings for the frobber
-    #
-    frobber:
-      # frobbing speed. Defaults to 1.
-      #
-      #speed: 10
-
-      # frobbing distance. Defaults to 1000.
-      #
-      #distance: 100
+```yaml
+## Frobnication ##
+
+# The frobnicator will ensure that all requests are fully frobnicated.
+# To enable it, uncomment the following.
+#
+#frobnicator_enabled: true
+
+# By default, the frobnicator will frobnicate with the default frobber.
+# The following will make it use an alternative frobber.
+#
+#frobincator_frobber: special_frobber
+
+# Settings for the frobber
+#
+frobber:
+  # frobbing speed. Defaults to 1.
+  #
+  #speed: 10
+
+  # frobbing distance. Defaults to 1000.
+  #
+  #distance: 100
+```
 
 Note that the sample configuration is generated from the synapse code
 and is maintained by a script, `scripts-dev/generate_sample_config`.
diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md
index 911a1f95db..fb1fec80fe 100644
--- a/docs/consent_tracking.md
+++ b/docs/consent_tracking.md
@@ -99,7 +99,7 @@ construct URIs where users can give their consent.
    see if an unauthenticated user is viewing the page. This is typically
    wrapped around the form that would be used to actually agree to the document:
 
-   ```
+   ```html
    {% if not public_version %}
      <!-- The variables used here are only provided when the 'u' param is given to the homeserver -->
      <form method="post" action="consent">
diff --git a/docs/delegate.md b/docs/delegate.md
index 05cb635047..f3f89075d1 100644
--- a/docs/delegate.md
+++ b/docs/delegate.md
@@ -91,4 +91,4 @@ is running a modern version of Synapse.
 ### Do I need the same certificate for the client and federation port?
 
 No. There is nothing stopping you from using different certificates,
-particularly if you are using a reverse proxy.
\ No newline at end of file
+particularly if you are using a reverse proxy.
diff --git a/docs/development/cas.md b/docs/development/cas.md
index 592b2d8d4f..7c0668e034 100644
--- a/docs/development/cas.md
+++ b/docs/development/cas.md
@@ -8,23 +8,23 @@ easy to run CAS implementation built on top of Django.
 1. Create a new virtualenv: `python3 -m venv <your virtualenv>`
 2. Activate your virtualenv: `source /path/to/your/virtualenv/bin/activate`
 3. Install Django and django-mama-cas:
-   ```
+   ```sh
    python -m pip install "django<3" "django-mama-cas==2.4.0"
    ```
 4. Create a Django project in the current directory:
-   ```
+   ```sh
    django-admin startproject cas_test .
    ```
 5. Follow the [install directions](https://django-mama-cas.readthedocs.io/en/latest/installation.html#configuring) for django-mama-cas
 6. Setup the SQLite database: `python manage.py migrate`
 7. Create a user:
-   ```
+   ```sh
    python manage.py createsuperuser
    ```
    1. Use whatever you want as the username and password.
    2. Leave the other fields blank.
 8. Use the built-in Django test server to serve the CAS endpoints on port 8000:
-   ```
+   ```sh
    python manage.py runserver
    ```
 
diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md
index 20740cf5ac..256a629210 100644
--- a/docs/development/database_schema.md
+++ b/docs/development/database_schema.md
@@ -89,7 +89,9 @@ To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
 
 Ensure postgres is installed, then run:
 
-    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+```sh
+./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+```
 
 NB at the time of writing, this script predates the split into separate `state`/`main`
 databases so will require updates to handle that correctly.
diff --git a/docs/development/saml.md b/docs/development/saml.md
index 60a431d686..b08bcb7419 100644
--- a/docs/development/saml.md
+++ b/docs/development/saml.md
@@ -15,7 +15,7 @@ To make Synapse (and therefore Element) use it:
      sp_config:
        allow_unknown_attributes: true  # Works around a bug with AVA Hashes: https://github.com/IdentityPython/pysaml2/issues/388
        metadata:
-         local: ["samling.xml"]   
+         local: ["samling.xml"]
    ```
 5. Ensure that your `homeserver.yaml` has a setting for `public_baseurl`:
    ```yaml
diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md
index ea3d46cc10..9214d6d7e9 100644
--- a/docs/message_retention_policies.md
+++ b/docs/message_retention_policies.md
@@ -69,9 +69,9 @@ A default policy can be defined as such, in the `retention` section of
 the configuration file:
 
 ```yaml
-  default_policy:
-    min_lifetime: 1d
-    max_lifetime: 1y
+default_policy:
+  min_lifetime: 1d
+  max_lifetime: 1y
 ```
 
 Here, `min_lifetime` and `max_lifetime` have the same meaning and level
@@ -95,14 +95,14 @@ depending on an event's room's policy. This can be done by setting the
 file. An example of such configuration could be:
 
 ```yaml
-  purge_jobs:
-    - longest_max_lifetime: 3d
-      interval: 12h
-    - shortest_max_lifetime: 3d
-      longest_max_lifetime: 1w
-      interval: 1d
-    - shortest_max_lifetime: 1w
-      interval: 2d
+purge_jobs:
+  - longest_max_lifetime: 3d
+    interval: 12h
+  - shortest_max_lifetime: 3d
+    longest_max_lifetime: 1w
+    interval: 1d
+  - shortest_max_lifetime: 1w
+    interval: 2d
 ```
 
 In this example, we define three jobs:
@@ -141,8 +141,8 @@ purging old events in a room. These limits can be defined as such in the
 `retention` section of the configuration file:
 
 ```yaml
-  allowed_lifetime_min: 1d
-  allowed_lifetime_max: 1y
+allowed_lifetime_min: 1d
+allowed_lifetime_max: 1y
 ```
 
 The limits are considered when running purge jobs. If necessary, the
diff --git a/docs/modules/password_auth_provider_callbacks.md b/docs/modules/password_auth_provider_callbacks.md
index 9dddfdfaaa..0de60b128a 100644
--- a/docs/modules/password_auth_provider_callbacks.md
+++ b/docs/modules/password_auth_provider_callbacks.md
@@ -10,7 +10,7 @@ registered by using the Module API's `register_password_auth_provider_callbacks`
 
 _First introduced in Synapse v1.46.0_
 
-```
+```python
  auth_checkers: Dict[Tuple[str,Tuple], Callable]
 ```
 
diff --git a/docs/postgres.md b/docs/postgres.md
index 2c0a5b803a..083b0aaff0 100644
--- a/docs/postgres.md
+++ b/docs/postgres.md
@@ -29,16 +29,20 @@ connect to a postgres database.
 
 Assuming your PostgreSQL database user is called `postgres`, first authenticate as the database user with:
 
-    su - postgres
-    # Or, if your system uses sudo to get administrative rights
-    sudo -u postgres bash
+```sh
+su - postgres
+# Or, if your system uses sudo to get administrative rights
+sudo -u postgres bash
+```
 
 Then, create a postgres user and a database with:
 
-    # this will prompt for a password for the new user
-    createuser --pwprompt synapse_user
+```sh
+# this will prompt for a password for the new user
+createuser --pwprompt synapse_user
 
-    createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
+createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
+```
 
 The above will create a user called `synapse_user`, and a database called
 `synapse`.
@@ -145,20 +149,26 @@ Firstly, shut down the currently running synapse server and copy its
 database file (typically `homeserver.db`) to another location. Once the
 copy is complete, restart synapse. For instance:
 
-    ./synctl stop
-    cp homeserver.db homeserver.db.snapshot
-    ./synctl start
+```sh
+./synctl stop
+cp homeserver.db homeserver.db.snapshot
+./synctl start
+```
 
 Copy the old config file into a new config file:
 
-    cp homeserver.yaml homeserver-postgres.yaml
+```sh
+cp homeserver.yaml homeserver-postgres.yaml
+```
 
 Edit the database section as described in the section *Synapse config*
 above and with the SQLite snapshot located at `homeserver.db.snapshot`
 simply run:
 
-    synapse_port_db --sqlite-database homeserver.db.snapshot \
-        --postgres-config homeserver-postgres.yaml
+```sh
+synapse_port_db --sqlite-database homeserver.db.snapshot \
+    --postgres-config homeserver-postgres.yaml
+```
 
 The flag `--curses` displays a coloured curses progress UI.
 
@@ -170,16 +180,20 @@ To complete the conversion shut down the synapse server and run the port
 script one last time, e.g. if the SQLite database is at `homeserver.db`
 run:
 
-    synapse_port_db --sqlite-database homeserver.db \
-        --postgres-config homeserver-postgres.yaml
+```sh
+synapse_port_db --sqlite-database homeserver.db \
+    --postgres-config homeserver-postgres.yaml
+```
 
 Once that has completed, change the synapse config to point at the
 PostgreSQL database configuration file `homeserver-postgres.yaml`:
 
-    ./synctl stop
-    mv homeserver.yaml homeserver-old-sqlite.yaml
-    mv homeserver-postgres.yaml homeserver.yaml
-    ./synctl start
+```sh
+./synctl stop
+mv homeserver.yaml homeserver-old-sqlite.yaml
+mv homeserver-postgres.yaml homeserver.yaml
+./synctl start
+```
 
 Synapse should now be running against PostgreSQL.
 
diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md
index bc351d604e..9f18fa1818 100644
--- a/docs/reverse_proxy.md
+++ b/docs/reverse_proxy.md
@@ -52,7 +52,7 @@ to proxied traffic.)
 
 ### nginx
 
-```
+```nginx
 server {
     listen 443 ssl http2;
     listen [::]:443 ssl http2;
@@ -141,7 +141,7 @@ matrix.example.com {
 
 ### Apache
 
-```
+```apache
 <VirtualHost *:443>
     SSLEngine on
     ServerName matrix.example.com
@@ -170,7 +170,7 @@ matrix.example.com {
 
 **NOTE 2**: It appears that Synapse is currently incompatible with the ModSecurity module for Apache (`mod_security2`). If you need it enabled for other services on your web server, you can disable it for Synapse's two VirtualHosts by including the following lines before each of the two `</VirtualHost>` above:
 
-```
+```apache
 <IfModule security2_module>
     SecRuleEngine off
 </IfModule>
diff --git a/docs/synctl_workers.md b/docs/synctl_workers.md
index 8da4a31852..15e37f608d 100644
--- a/docs/synctl_workers.md
+++ b/docs/synctl_workers.md
@@ -20,7 +20,9 @@ Finally, to actually run your worker-based synapse, you must pass synctl the `-a
 commandline option to tell it to operate on all the worker configurations found
 in the given directory, e.g.:
 
-    synctl -a $CONFIG/workers start
+```sh
+synctl -a $CONFIG/workers start
+```
 
 Currently one should always restart all workers when restarting or upgrading
 synapse, unless you explicitly know it's safe not to.  For instance, restarting
@@ -29,4 +31,6 @@ notifications.
 
 To manipulate a specific worker, you pass the -w option to synctl:
 
-    synctl -w $CONFIG/workers/worker1.yaml restart
+```sh
+synctl -w $CONFIG/workers/worker1.yaml restart
+```
diff --git a/docs/turn-howto.md b/docs/turn-howto.md
index 6433446c2a..99f0bb2fc2 100644
--- a/docs/turn-howto.md
+++ b/docs/turn-howto.md
@@ -40,7 +40,9 @@ This will install and start a systemd service called `coturn`.
 
 1.  Configure it:
 
-        ./configure
+    ```sh
+    ./configure
+    ```
 
     You may need to install `libevent2`: if so, you should do so in
     the way recommended by your operating system. You can ignore
@@ -49,22 +51,28 @@ This will install and start a systemd service called `coturn`.
 
 1.  Build and install it:
 
-        make
-        make install
+    ```sh
+    make
+    make install
+    ```
 
 ### Configuration
 
 1.  Create or edit the config file in `/etc/turnserver.conf`. The relevant
     lines, with example values, are:
 
-        use-auth-secret
-        static-auth-secret=[your secret key here]
-        realm=turn.myserver.org
+    ```
+    use-auth-secret
+    static-auth-secret=[your secret key here]
+    realm=turn.myserver.org
+    ```
 
     See `turnserver.conf` for explanations of the options. One way to generate
     the `static-auth-secret` is with `pwgen`:
 
-        pwgen -s 64 1
+    ```sh
+    pwgen -s 64 1
+    ```
 
     A `realm` must be specified, but its value is somewhat arbitrary. (It is
     sent to clients as part of the authentication flow.) It is conventional to
@@ -73,7 +81,9 @@ This will install and start a systemd service called `coturn`.
 1.  You will most likely want to configure coturn to write logs somewhere. The
     easiest way is normally to send them to the syslog:
 
-        syslog
+    ```sh
+    syslog
+    ```
 
     (in which case, the logs will be available via `journalctl -u coturn` on a
     systemd system). Alternatively, coturn can be configured to write to a
@@ -83,31 +93,35 @@ This will install and start a systemd service called `coturn`.
     connect to arbitrary IP addresses and ports. The following configuration is
     suggested as a minimum starting point:
 
-        # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay.
-        no-tcp-relay
+    ```
+    # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay.
+    no-tcp-relay
 
-        # don't let the relay ever try to connect to private IP address ranges within your network (if any)
-        # given the turn server is likely behind your firewall, remember to include any privileged public IPs too.
-        denied-peer-ip=10.0.0.0-10.255.255.255
-        denied-peer-ip=192.168.0.0-192.168.255.255
-        denied-peer-ip=172.16.0.0-172.31.255.255
+    # don't let the relay ever try to connect to private IP address ranges within your network (if any)
+    # given the turn server is likely behind your firewall, remember to include any privileged public IPs too.
+    denied-peer-ip=10.0.0.0-10.255.255.255
+    denied-peer-ip=192.168.0.0-192.168.255.255
+    denied-peer-ip=172.16.0.0-172.31.255.255
 
-        # special case the turn server itself so that client->TURN->TURN->client flows work
-        allowed-peer-ip=10.0.0.1
+    # special case the turn server itself so that client->TURN->TURN->client flows work
+    allowed-peer-ip=10.0.0.1
 
-        # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS.
-        user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user.
-        total-quota=1200
+    # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS.
+    user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user.
+    total-quota=1200
+    ```
 
 1.  Also consider supporting TLS/DTLS. To do this, add the following settings
     to `turnserver.conf`:
 
-        # TLS certificates, including intermediate certs.
-        # For Let's Encrypt certificates, use `fullchain.pem` here.
-        cert=/path/to/fullchain.pem
+    ```
+    # TLS certificates, including intermediate certs.
+    # For Let's Encrypt certificates, use `fullchain.pem` here.
+    cert=/path/to/fullchain.pem
 
-        # TLS private key file
-        pkey=/path/to/privkey.pem
+    # TLS private key file
+    pkey=/path/to/privkey.pem
+    ```
 
     In this case, replace the `turn:` schemes in the `turn_uri` settings below
     with `turns:`.
@@ -126,7 +140,9 @@ This will install and start a systemd service called `coturn`.
     If you want to try it anyway, you will at least need to tell coturn its
     external IP address:
 
-        external-ip=192.88.99.1
+    ```
+    external-ip=192.88.99.1
+    ```
 
     ... and your NAT gateway must forward all of the relayed ports directly
     (eg, port 56789 on the external IP must be always be forwarded to port
@@ -186,7 +202,7 @@ After updating the homeserver configuration, you must restart synapse:
     ./synctl restart
     ```
   * If you use systemd:
-    ```
+    ```sh
     systemctl restart matrix-synapse.service
     ```
 ... and then reload any clients (or wait an hour for them to refresh their
diff --git a/docs/upgrade.md b/docs/upgrade.md
index d32d1ab988..06f479f86c 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -1176,16 +1176,20 @@ For more information on configuring TLS certificates see the
     For users who have installed Synapse into a virtualenv, we recommend
     doing this by creating a new virtualenv. For example:
 
-        virtualenv -p python3 ~/synapse/env3
-        source ~/synapse/env3/bin/activate
-        pip install matrix-synapse
+    ```sh
+    virtualenv -p python3 ~/synapse/env3
+    source ~/synapse/env3/bin/activate
+    pip install matrix-synapse
+    ```
 
     You can then start synapse as normal, having activated the new
     virtualenv:
 
-        cd ~/synapse
-        source env3/bin/activate
-        synctl start
+    ```sh
+    cd ~/synapse
+    source env3/bin/activate
+    synctl start
+    ```
 
     Users who have installed from distribution packages should see the
     relevant package documentation. See below for notes on Debian
@@ -1197,34 +1201,38 @@ For more information on configuring TLS certificates see the
         `<server>.log.config` file. For example, if your `log.config`
         file contains:
 
-            handlers:
-              file:
-                class: logging.handlers.RotatingFileHandler
-                formatter: precise
-                filename: homeserver.log
-                maxBytes: 104857600
-                backupCount: 10
-                filters: [context]
-              console:
-                class: logging.StreamHandler
-                formatter: precise
-                filters: [context]
+        ```yaml
+        handlers:
+          file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: homeserver.log
+            maxBytes: 104857600
+            backupCount: 10
+            filters: [context]
+          console:
+            class: logging.StreamHandler
+            formatter: precise
+            filters: [context]
+        ```
 
         Then you should update this to be:
 
-            handlers:
-              file:
-                class: logging.handlers.RotatingFileHandler
-                formatter: precise
-                filename: homeserver.log
-                maxBytes: 104857600
-                backupCount: 10
-                filters: [context]
-                encoding: utf8
-              console:
-                class: logging.StreamHandler
-                formatter: precise
-                filters: [context]
+        ```yaml
+        handlers:
+          file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: homeserver.log
+            maxBytes: 104857600
+            backupCount: 10
+            filters: [context]
+            encoding: utf8
+          console:
+            class: logging.StreamHandler
+            formatter: precise
+            filters: [context]
+        ```
 
         There is no need to revert this change if downgrading to
         Python 2.
@@ -1310,24 +1318,28 @@ with the HS remotely has been removed.
 It has been replaced by specifying a list of application service
 registrations in `homeserver.yaml`:
 
-    app_service_config_files: ["registration-01.yaml", "registration-02.yaml"]
+```yaml
+app_service_config_files: ["registration-01.yaml", "registration-02.yaml"]
+```
 
 Where `registration-01.yaml` looks like:
 
-    url: <String>  # e.g. "https://my.application.service.com"
-    as_token: <String>
-    hs_token: <String>
-    sender_localpart: <String>  # This is a new field which denotes the user_id localpart when using the AS token
-    namespaces:
-      users:
-        - exclusive: <Boolean>
-          regex: <String>  # e.g. "@prefix_.*"
-      aliases:
-        - exclusive: <Boolean>
-          regex: <String>
-      rooms:
-        - exclusive: <Boolean>
-          regex: <String>
+```yaml
+url: <String>  # e.g. "https://my.application.service.com"
+as_token: <String>
+hs_token: <String>
+sender_localpart: <String>  # This is a new field which denotes the user_id localpart when using the AS token
+namespaces:
+  users:
+    - exclusive: <Boolean>
+      regex: <String>  # e.g. "@prefix_.*"
+  aliases:
+    - exclusive: <Boolean>
+      regex: <String>
+  rooms:
+    - exclusive: <Boolean>
+      regex: <String>
+```
 
 # Upgrading to v0.8.0
 
diff --git a/docs/workers.md b/docs/workers.md
index f1673d67d0..f88e2c1de3 100644
--- a/docs/workers.md
+++ b/docs/workers.md
@@ -443,19 +443,19 @@ In the `media_repository` worker configuration file, configure the http listener
 expose the `media` resource. For example:
 
 ```yaml
-    worker_listeners:
-     - type: http
-       port: 8085
-       resources:
-         - names:
-           - media
+worker_listeners:
+ - type: http
+   port: 8085
+   resources:
+     - names:
+       - media
 ```
 
 Note that if running multiple media repositories they must be on the same server
 and you must configure a single instance to run the background tasks, e.g.:
 
 ```yaml
-    media_instance_running_background_jobs: "media-repository-1"
+media_instance_running_background_jobs: "media-repository-1"
 ```
 
 Note that if a reverse proxy is used , then `/_matrix/media/` must be routed for both inbound client and federation requests (if they are handled separately).
@@ -492,7 +492,9 @@ must therefore be configured with the location of the main instance, via
 the `worker_main_http_uri` setting in the `frontend_proxy` worker configuration
 file. For example:
 
-    worker_main_http_uri: http://127.0.0.1:8008
+```yaml
+worker_main_http_uri: http://127.0.0.1:8008
+```
 
 ### Historical apps
 
-- 
cgit 1.5.1


From 3ae1464efdc65b54bd860118163c9f38c82e7375 Mon Sep 17 00:00:00 2001
From: Aaron R <aaron@raim.ist>
Date: Mon, 1 Nov 2021 08:28:39 -0500
Subject: Support Client-Server API r0.6.1 (#11097)

Fixes #11064

Signed-off-by: Aaron Raimist <aaron@raim.ist>
---
 changelog.d/11097.feature       | 1 +
 synapse/rest/client/versions.py | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changelog.d/11097.feature

diff --git a/changelog.d/11097.feature b/changelog.d/11097.feature
new file mode 100644
index 0000000000..d7563a406c
--- /dev/null
+++ b/changelog.d/11097.feature
@@ -0,0 +1 @@
+Advertise support for Client-Server API r0.6.1.
\ No newline at end of file
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index b52a296d8f..8d888f4565 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -72,6 +72,7 @@ class VersionsRestServlet(RestServlet):
                     "r0.4.0",
                     "r0.5.0",
                     "r0.6.0",
+                    "r0.6.1",
                 ],
                 # as per MSC1497:
                 "unstable_features": {
-- 
cgit 1.5.1


From 0b99d4c8d219aca3f8a0e4f3542bb67dfc016420 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 1 Nov 2021 13:55:30 +0000
Subject: Docker: avoid changing userid unnecessarily (#11209)

* Docker image: avoid changing user during `generate`

The intention was always that the config files get written as the initial user
(normally root) - only the data directory needs to be writable by Synapse. This
got changed in https://github.com/matrix-org/synapse/pull/5970, but that seems
to have been a mistake.

* Avoid changing user if no explicit UID is given

* changelog
---
 changelog.d/11209.docker |  1 +
 docker/README.md         | 13 ++++++++-----
 docker/start.py          | 37 +++++++++++++++++++------------------
 3 files changed, 28 insertions(+), 23 deletions(-)
 create mode 100644 changelog.d/11209.docker

diff --git a/changelog.d/11209.docker b/changelog.d/11209.docker
new file mode 100644
index 0000000000..838b165ac9
--- /dev/null
+++ b/changelog.d/11209.docker
@@ -0,0 +1 @@
+Avoid changing userid when started as a non-root user, and no explicit `UID` is set.
diff --git a/docker/README.md b/docker/README.md
index 38d182bf45..4349e71f87 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -65,7 +65,8 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_DATA_DIR`: where the generated config will put persistent data
   such as the database and media store. Defaults to `/data`.
 * `UID`, `GID`: the user id and group id to use for creating the data
-  directories. Defaults to `991`, `991`.
+  directories. If unset, and no user is set via `docker run --user`, defaults
+  to `991`, `991`.
 
 ## Running synapse
 
@@ -97,7 +98,9 @@ The following environment variables are supported in `run` mode:
   `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
 * `SYNAPSE_WORKER`: module to execute, used when running synapse with workers.
    Defaults to `synapse.app.homeserver`, which is suitable for non-worker mode.
-* `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
+* `UID`, `GID`: the user and group id to run Synapse as. If unset, and no user
+  is set via `docker run --user`, defaults to `991`, `991`. Note that this user
+  must have permission to read the config files, and write to the data directories.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
 
 For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
@@ -186,7 +189,7 @@ point to another Dockerfile.
 ## Disabling the healthcheck
 
 If you are using a non-standard port or tls inside docker you can disable the healthcheck
-whilst running the above `docker run` commands. 
+whilst running the above `docker run` commands.
 
 ```
    --no-healthcheck
@@ -212,7 +215,7 @@ If you wish to point the healthcheck at a different port with docker command, ad
 ## Setting the healthcheck in docker-compose file
 
 You can add the following to set a custom healthcheck in a docker compose file.
-You will need docker-compose version >2.1 for this to work. 
+You will need docker-compose version >2.1 for this to work.
 
 ```
 healthcheck:
@@ -226,5 +229,5 @@ healthcheck:
 ## Using jemalloc
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse 
+You can read about jemalloc by reading the Synapse
 [README](https://github.com/matrix-org/synapse/blob/HEAD/README.rst#help-synapse-is-slow-and-eats-all-my-ram-cpu).
diff --git a/docker/start.py b/docker/start.py
index 16d6a8208a..ec9eeb49ae 100755
--- a/docker/start.py
+++ b/docker/start.py
@@ -120,6 +120,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
     ]
 
     if ownership is not None:
+        log(f"Setting ownership on /data to {ownership}")
         subprocess.check_output(["chown", "-R", ownership, "/data"])
         args = ["gosu", ownership] + args
 
@@ -144,12 +145,18 @@ def run_generate_config(environ, ownership):
     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
 
+    if ownership is not None:
+        # make sure that synapse has perms to write to the data dir.
+        log(f"Setting ownership on {data_dir} to {ownership}")
+        subprocess.check_output(["chown", ownership, data_dir])
+
     # create a suitable log config from our template
     log_config_file = "%s/%s.log.config" % (config_dir, server_name)
     if not os.path.exists(log_config_file):
         log("Creating log config %s" % (log_config_file,))
         convert("/conf/log.config", log_config_file, environ)
 
+    # generate the main config file, and a signing key.
     args = [
         "python",
         "-m",
@@ -168,29 +175,23 @@ def run_generate_config(environ, ownership):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-
-    if ownership is not None:
-        # make sure that synapse has perms to write to the data dir.
-        subprocess.check_output(["chown", ownership, data_dir])
-
-        args = ["gosu", ownership] + args
-        os.execv("/usr/sbin/gosu", args)
-    else:
-        os.execv("/usr/local/bin/python", args)
+    os.execv("/usr/local/bin/python", args)
 
 
 def main(args, environ):
     mode = args[1] if len(args) > 1 else "run"
-    desired_uid = int(environ.get("UID", "991"))
-    desired_gid = int(environ.get("GID", "991"))
-    synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
-    if (desired_uid == os.getuid()) and (desired_gid == os.getgid()):
-        ownership = None
-    else:
-        ownership = "{}:{}".format(desired_uid, desired_gid)
 
-    if ownership is None:
-        log("Will not perform chmod/gosu as UserID already matches request")
+    # if we were given an explicit user to switch to, do so
+    ownership = None
+    if "UID" in environ:
+        desired_uid = int(environ["UID"])
+        desired_gid = int(environ.get("GID", "991"))
+        ownership = f"{desired_uid}:{desired_gid}"
+    elif os.getuid() == 0:
+        # otherwise, if we are running as root, use user 991
+        ownership = "991:991"
+
+    synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
 
     # In generate mode, generate a configuration and missing keys, then exit
     if mode == "generate":
-- 
cgit 1.5.1


From 2014098d014844b89abd39ac86ab7ba39c5340f9 Mon Sep 17 00:00:00 2001
From: Brett Bethke <10068296+bb4242@users.noreply.github.com>
Date: Mon, 1 Nov 2021 09:16:02 -0500
Subject: Add domain specific matching for haproxy config (#11128)

---
 changelog.d/11128.doc | 1 +
 docs/reverse_proxy.md | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11128.doc

diff --git a/changelog.d/11128.doc b/changelog.d/11128.doc
new file mode 100644
index 0000000000..c024679218
--- /dev/null
+++ b/changelog.d/11128.doc
@@ -0,0 +1 @@
+Improve example HAProxy config in the docs to properly handle host headers with port information. This is required for federation over port 443 to work correctly.
diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md
index 9f18fa1818..f3b3aea732 100644
--- a/docs/reverse_proxy.md
+++ b/docs/reverse_proxy.md
@@ -188,7 +188,7 @@ frontend https
   http-request set-header X-Forwarded-For %[src]
 
   # Matrix client traffic
-  acl matrix-host hdr(host) -i matrix.example.com
+  acl matrix-host hdr(host) -i matrix.example.com matrix.example.com:443
   acl matrix-path path_beg /_matrix
   acl matrix-path path_beg /_synapse/client
 
-- 
cgit 1.5.1


From 71f9966f2790c6b24281bb9f109bff28ff05d962 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 1 Nov 2021 15:10:16 +0000
Subject: Support for serving server well-known files (#11211)

Fixes https://github.com/matrix-org/synapse/issues/8308
---
 changelog.d/11211.feature     |  1 +
 docs/delegate.md              | 82 ++++++++++++++++++++++++-------------------
 docs/sample_config.yaml       | 18 ++++++++++
 synapse/app/generic_worker.py |  3 ++
 synapse/app/homeserver.py     |  4 +--
 synapse/config/server.py      | 19 ++++++++++
 synapse/rest/well_known.py    | 47 +++++++++++++++++++++++--
 tests/rest/test_well_known.py | 32 +++++++++++++----
 8 files changed, 159 insertions(+), 47 deletions(-)
 create mode 100644 changelog.d/11211.feature

diff --git a/changelog.d/11211.feature b/changelog.d/11211.feature
new file mode 100644
index 0000000000..feeb0cf089
--- /dev/null
+++ b/changelog.d/11211.feature
@@ -0,0 +1 @@
+Add support for serving `/.well-known/matrix/server` files, to redirect federation traffic to port 443.
diff --git a/docs/delegate.md b/docs/delegate.md
index f3f89075d1..ee9cbb3b1c 100644
--- a/docs/delegate.md
+++ b/docs/delegate.md
@@ -1,4 +1,8 @@
-# Delegation
+# Delegation of incoming federation traffic
+
+In the following documentation, we use the term `server_name` to refer to that setting
+in your homeserver configuration file. It appears at the ends of user ids, and tells
+other homeservers where they can find your server.
 
 By default, other homeservers will expect to be able to reach yours via
 your `server_name`, on port 8448. For example, if you set your `server_name`
@@ -12,13 +16,21 @@ to a different server and/or port (e.g. `synapse.example.com:443`).
 
 ## .well-known delegation
 
-To use this method, you need to be able to alter the
-`server_name` 's https server to serve the `/.well-known/matrix/server`
-URL. Having an active server (with a valid TLS certificate) serving your
-`server_name` domain is out of the scope of this documentation.
+To use this method, you need to be able to configure the server at
+`https://<server_name>` to serve a file at
+`https://<server_name>/.well-known/matrix/server`.  There are two ways to do this, shown below.
+
+Note that the `.well-known` file is hosted on the default port for `https` (port 443).
+
+### External server
+
+For maximum flexibility, you need to configure an external server such as nginx, Apache
+or HAProxy to serve the `https://<server_name>/.well-known/matrix/server` file. Setting
+up such a server is out of the scope of this documentation, but note that it is often
+possible to configure your [reverse proxy](reverse_proxy.md) for this.
 
-The URL `https://<server_name>/.well-known/matrix/server` should
-return a JSON structure containing the key `m.server` like so:
+The URL `https://<server_name>/.well-known/matrix/server` should be configured
+return a JSON structure containing the key `m.server` like this:
 
 ```json
 {
@@ -26,8 +38,9 @@ return a JSON structure containing the key `m.server` like so:
 }
 ```
 
-In our example, this would mean that URL `https://example.com/.well-known/matrix/server`
-should return:
+In our example (where we want federation traffic to be routed to
+`https://synapse.example.com`, on port 443), this would mean that
+`https://example.com/.well-known/matrix/server` should return:
 
 ```json
 {
@@ -38,16 +51,29 @@ should return:
 Note, specifying a port is optional. If no port is specified, then it defaults
 to 8448.
 
-With .well-known delegation, federating servers will check for a valid TLS
-certificate for the delegated hostname (in our example: `synapse.example.com`).
+### Serving a `.well-known/matrix/server` file with Synapse
+
+If you are able to set up your domain so that `https://<server_name>` is routed to
+Synapse (i.e., the only change needed is to direct federation traffic to port 443
+instead of port 8448), then it is possible to configure Synapse to serve a suitable
+`.well-known/matrix/server` file. To do so, add the following to your `homeserver.yaml`
+file:
+
+```yaml
+serve_server_wellknown: true
+```
+
+**Note**: this *only* works if `https://<server_name>` is routed to Synapse, so is
+generally not suitable if Synapse is hosted at a subdomain such as
+`https://synapse.example.com`.
 
 ## SRV DNS record delegation
 
-It is also possible to do delegation using a SRV DNS record. However, that is
-considered an advanced topic since it's a bit complex to set up, and `.well-known`
-delegation is already enough in most cases.
+It is also possible to do delegation using a SRV DNS record. However, that is generally
+not recommended, as it can be difficult to configure the TLS certificates correctly in
+this case, and it offers little advantage over `.well-known` delegation.
 
-However, if you really need it, you can find some documentation on how such a
+However, if you really need it, you can find some documentation on what such a
 record should look like and how Synapse will use it in [the Matrix
 specification](https://matrix.org/docs/spec/server_server/latest#resolving-server-names).
 
@@ -68,27 +94,9 @@ wouldn't need any delegation set up.
 domain `server_name` points to, you will need to let other servers know how to
 find it using delegation.
 
-### Do you still recommend against using a reverse proxy on the federation port?
-
-We no longer actively recommend against using a reverse proxy. Many admins will
-find it easier to direct federation traffic to a reverse proxy and manage their
-own TLS certificates, and this is a supported configuration.
+### Should I use a reverse proxy for federation traffic?
 
-See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
+Generally, using a reverse proxy for both the federation and client traffic is a good
+idea, since it saves handling TLS traffic in Synapse. See
+[the reverse proxy documentation](reverse_proxy.md) for information on setting up a
 reverse proxy.
-
-### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
-
-This is no longer necessary. If you are using a reverse proxy for all of your
-TLS traffic, then you can set `no_tls: True` in the Synapse config.
-
-In that case, the only reason Synapse needs the certificate is to populate a legacy
-`tls_fingerprints` field in the federation API. This is ignored by Synapse 0.99.0
-and later, and the only time pre-0.99 Synapses will check it is when attempting to
-fetch the server keys - and generally this is delegated via `matrix.org`, which
-is running a modern version of Synapse.
-
-### Do I need the same certificate for the client and federation port?
-
-No. There is nothing stopping you from using different certificates,
-particularly if you are using a reverse proxy.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index b90ed62d61..c3a4148f74 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -93,6 +93,24 @@ pid_file: DATADIR/homeserver.pid
 #
 #public_baseurl: https://example.com/
 
+# Uncomment the following to tell other servers to send federation traffic on
+# port 443.
+#
+# By default, other servers will try to reach our server on port 8448, which can
+# be inconvenient in some environments.
+#
+# Provided 'https://<server_name>/' on port 443 is routed to Synapse, this
+# option configures Synapse to serve a file at
+# 'https://<server_name>/.well-known/matrix/server'. This will tell other
+# servers to send traffic to port 443 instead.
+#
+# See https://matrix-org.github.io/synapse/latest/delegate.html for more
+# information.
+#
+# Defaults to 'false'.
+#
+#serve_server_wellknown: true
+
 # Set the soft limit on the number of file descriptors synapse can use
 # Zero is used to indicate synapse should set the soft limit to the
 # hard limit.
diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py
index 51eadf122d..218826741e 100644
--- a/synapse/app/generic_worker.py
+++ b/synapse/app/generic_worker.py
@@ -100,6 +100,7 @@ from synapse.rest.client.register import (
 from synapse.rest.health import HealthResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
+from synapse.rest.well_known import well_known_resource
 from synapse.server import HomeServer
 from synapse.storage.databases.main.censor_events import CensorEventsStore
 from synapse.storage.databases.main.client_ips import ClientIpWorkerStore
@@ -318,6 +319,8 @@ class GenericWorkerServer(HomeServer):
                     resources.update({CLIENT_API_PREFIX: resource})
 
                     resources.update(build_synapse_client_resource_tree(self))
+                    resources.update({"/.well-known": well_known_resource(self)})
+
                 elif name == "federation":
                     resources.update({FEDERATION_PREFIX: TransportLayerServer(self)})
                 elif name == "media":
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 93e2299266..336c279a44 100644
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -66,7 +66,7 @@ from synapse.rest.admin import AdminRestResource
 from synapse.rest.health import HealthResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
-from synapse.rest.well_known import WellKnownResource
+from synapse.rest.well_known import well_known_resource
 from synapse.server import HomeServer
 from synapse.storage import DataStore
 from synapse.util.httpresourcetree import create_resource_tree
@@ -189,7 +189,7 @@ class SynapseHomeServer(HomeServer):
                     "/_matrix/client/unstable": client_resource,
                     "/_matrix/client/v2_alpha": client_resource,
                     "/_matrix/client/versions": client_resource,
-                    "/.well-known/matrix/client": WellKnownResource(self),
+                    "/.well-known": well_known_resource(self),
                     "/_synapse/admin": AdminRestResource(self),
                     **build_synapse_client_resource_tree(self),
                 }
diff --git a/synapse/config/server.py b/synapse/config/server.py
index ed094bdc44..a387fd9310 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -262,6 +262,7 @@ class ServerConfig(Config):
         self.print_pidfile = config.get("print_pidfile")
         self.user_agent_suffix = config.get("user_agent_suffix")
         self.use_frozen_dicts = config.get("use_frozen_dicts", False)
+        self.serve_server_wellknown = config.get("serve_server_wellknown", False)
 
         self.public_baseurl = config.get("public_baseurl")
         if self.public_baseurl is not None:
@@ -774,6 +775,24 @@ class ServerConfig(Config):
         #
         #public_baseurl: https://example.com/
 
+        # Uncomment the following to tell other servers to send federation traffic on
+        # port 443.
+        #
+        # By default, other servers will try to reach our server on port 8448, which can
+        # be inconvenient in some environments.
+        #
+        # Provided 'https://<server_name>/' on port 443 is routed to Synapse, this
+        # option configures Synapse to serve a file at
+        # 'https://<server_name>/.well-known/matrix/server'. This will tell other
+        # servers to send traffic to port 443 instead.
+        #
+        # See https://matrix-org.github.io/synapse/latest/delegate.html for more
+        # information.
+        #
+        # Defaults to 'false'.
+        #
+        #serve_server_wellknown: true
+
         # Set the soft limit on the number of file descriptors synapse can use
         # Zero is used to indicate synapse should set the soft limit to the
         # hard limit.
diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py
index 7ac01faab4..edbf5ce5d0 100644
--- a/synapse/rest/well_known.py
+++ b/synapse/rest/well_known.py
@@ -21,6 +21,7 @@ from twisted.web.server import Request
 from synapse.http.server import set_cors_headers
 from synapse.types import JsonDict
 from synapse.util import json_encoder
+from synapse.util.stringutils import parse_server_name
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -47,8 +48,8 @@ class WellKnownBuilder:
         return result
 
 
-class WellKnownResource(Resource):
-    """A Twisted web resource which renders the .well-known file"""
+class ClientWellKnownResource(Resource):
+    """A Twisted web resource which renders the .well-known/matrix/client file"""
 
     isLeaf = 1
 
@@ -67,3 +68,45 @@ class WellKnownResource(Resource):
         logger.debug("returning: %s", r)
         request.setHeader(b"Content-Type", b"application/json")
         return json_encoder.encode(r).encode("utf-8")
+
+
+class ServerWellKnownResource(Resource):
+    """Resource for .well-known/matrix/server, redirecting to port 443"""
+
+    isLeaf = 1
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        self._serve_server_wellknown = hs.config.server.serve_server_wellknown
+
+        host, port = parse_server_name(hs.config.server.server_name)
+
+        # If we've got this far, then https://<server_name>/ must route to us, so
+        # we just redirect the traffic to port 443 instead of 8448.
+        if port is None:
+            port = 443
+
+        self._response = json_encoder.encode({"m.server": f"{host}:{port}"}).encode(
+            "utf-8"
+        )
+
+    def render_GET(self, request: Request) -> bytes:
+        if not self._serve_server_wellknown:
+            request.setResponseCode(404)
+            request.setHeader(b"Content-Type", b"text/plain")
+            return b"404. Is anything ever truly *well* known?\n"
+
+        request.setHeader(b"Content-Type", b"application/json")
+        return self._response
+
+
+def well_known_resource(hs: "HomeServer") -> Resource:
+    """Returns a Twisted web resource which handles '.well-known' requests"""
+    res = Resource()
+    matrix_resource = Resource()
+    res.putChild(b"matrix", matrix_resource)
+
+    matrix_resource.putChild(b"server", ServerWellKnownResource(hs))
+    matrix_resource.putChild(b"client", ClientWellKnownResource(hs))
+
+    return res
diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py
index b2c0279ba0..118aa93a32 100644
--- a/tests/rest/test_well_known.py
+++ b/tests/rest/test_well_known.py
@@ -11,17 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from twisted.web.resource import Resource
 
-
-from synapse.rest.well_known import WellKnownResource
+from synapse.rest.well_known import well_known_resource
 
 from tests import unittest
 
 
 class WellKnownTests(unittest.HomeserverTestCase):
     def create_test_resource(self):
-        # replace the JsonResource with a WellKnownResource
-        return WellKnownResource(self.hs)
+        # replace the JsonResource with a Resource wrapping the WellKnownResource
+        res = Resource()
+        res.putChild(b".well-known", well_known_resource(self.hs))
+        return res
 
     @unittest.override_config(
         {
@@ -29,7 +31,7 @@ class WellKnownTests(unittest.HomeserverTestCase):
             "default_identity_server": "https://testis",
         }
     )
-    def test_well_known(self):
+    def test_client_well_known(self):
         channel = self.make_request(
             "GET", "/.well-known/matrix/client", shorthand=False
         )
@@ -48,9 +50,27 @@ class WellKnownTests(unittest.HomeserverTestCase):
             "public_baseurl": None,
         }
     )
-    def test_well_known_no_public_baseurl(self):
+    def test_client_well_known_no_public_baseurl(self):
         channel = self.make_request(
             "GET", "/.well-known/matrix/client", shorthand=False
         )
 
         self.assertEqual(channel.code, 404)
+
+    @unittest.override_config({"serve_server_wellknown": True})
+    def test_server_well_known(self):
+        channel = self.make_request(
+            "GET", "/.well-known/matrix/server", shorthand=False
+        )
+
+        self.assertEqual(channel.code, 200)
+        self.assertEqual(
+            channel.json_body,
+            {"m.server": "test:443"},
+        )
+
+    def test_server_well_known_disabled(self):
+        channel = self.make_request(
+            "GET", "/.well-known/matrix/server", shorthand=False
+        )
+        self.assertEqual(channel.code, 404)
-- 
cgit 1.5.1


From 66bdca3e317d1fa764cf52547aee7409acc59676 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Mon, 1 Nov 2021 16:11:24 +0100
Subject: Remove deprecated delete room admin API (#11213)

Remove deprecated delete room admin API,
`POST /_synapse/admin/v1/rooms/<room_id>/delete`
---
 changelog.d/11213.removal      |   1 +
 docs/admin_api/rooms.md        |  10 ---
 docs/upgrade.md                |  10 +++
 synapse/rest/admin/__init__.py |   2 -
 synapse/rest/admin/rooms.py    | 141 ++++++++++++++++-------------------------
 tests/rest/admin/test_room.py  |  39 +++++-------
 6 files changed, 79 insertions(+), 124 deletions(-)
 create mode 100644 changelog.d/11213.removal

diff --git a/changelog.d/11213.removal b/changelog.d/11213.removal
new file mode 100644
index 0000000000..9e5ec936e3
--- /dev/null
+++ b/changelog.d/11213.removal
@@ -0,0 +1 @@
+Remove deprecated admin API to delete rooms (`POST /_synapse/admin/v1/rooms/<room_id>/delete`).
\ No newline at end of file
diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md
index acf1cab2a2..62eeff9e1a 100644
--- a/docs/admin_api/rooms.md
+++ b/docs/admin_api/rooms.md
@@ -520,16 +520,6 @@ With all that being said, if you still want to try and recover the room:
 4. If `new_room_user_id` was given, a 'Content Violation' will have been
    created. Consider whether you want to delete that roomm.
 
-## Deprecated endpoint
-
-The previous deprecated API will be removed in a future release, it was:
-
-```
-POST /_synapse/admin/v1/rooms/<room_id>/delete
-```
-
-It behaves the same way than the current endpoint except the path and the method.
-
 # Make Room Admin API
 
 Grants another user the highest power available to a local user who is in the room.
diff --git a/docs/upgrade.md b/docs/upgrade.md
index 06f479f86c..136c806c41 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -87,6 +87,16 @@ process, for example:
 
 # Upgrading to v1.47.0
 
+## Removal of old Room Admin API
+
+The following admin APIs were deprecated in [Synapse 1.34](https://github.com/matrix-org/synapse/blob/v1.34.0/CHANGES.md#deprecations-and-removals)
+(released on 2021-05-17) and have now been removed:
+
+- `POST /_synapse/admin/v1/<room_id>/delete`
+
+Any scripts still using the above APIs should be converted to use the
+[Delete Room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api).
+
 ## Deprecation of the `user_may_create_room_with_invites` module callback
 
 The `user_may_create_room_with_invites` is deprecated and will be removed in a future
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index e1506deb2b..70514e814f 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -42,7 +42,6 @@ from synapse.rest.admin.registration_tokens import (
     RegistrationTokenRestServlet,
 )
 from synapse.rest.admin.rooms import (
-    DeleteRoomRestServlet,
     ForwardExtremitiesRestServlet,
     JoinRoomAliasServlet,
     ListRoomRestServlet,
@@ -221,7 +220,6 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
     RoomStateRestServlet(hs).register(http_server)
     RoomRestServlet(hs).register(http_server)
     RoomMembersRestServlet(hs).register(http_server)
-    DeleteRoomRestServlet(hs).register(http_server)
     JoinRoomAliasServlet(hs).register(http_server)
     VersionServlet(hs).register(http_server)
     UserAdminServlet(hs).register(http_server)
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index a4823ca6e7..05c5b4bf0c 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -46,41 +46,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-class DeleteRoomRestServlet(RestServlet):
-    """Delete a room from server.
-
-    It is a combination and improvement of shutdown and purge room.
-
-    Shuts down a room by removing all local users from the room.
-    Blocking all future invites and joins to the room is optional.
-
-    If desired any local aliases will be repointed to a new room
-    created by `new_room_user_id` and kicked users will be auto-
-    joined to the new room.
-
-    If 'purge' is true, it will remove all traces of a room from the database.
-    """
-
-    PATTERNS = admin_patterns("/rooms/(?P<room_id>[^/]+)/delete$")
-
-    def __init__(self, hs: "HomeServer"):
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self.room_shutdown_handler = hs.get_room_shutdown_handler()
-        self.pagination_handler = hs.get_pagination_handler()
-
-    async def on_POST(
-        self, request: SynapseRequest, room_id: str
-    ) -> Tuple[int, JsonDict]:
-        return await _delete_room(
-            request,
-            room_id,
-            self.auth,
-            self.room_shutdown_handler,
-            self.pagination_handler,
-        )
-
-
 class ListRoomRestServlet(RestServlet):
     """
     List all rooms that are known to the homeserver. Results are returned
@@ -218,7 +183,7 @@ class RoomRestServlet(RestServlet):
     async def on_DELETE(
         self, request: SynapseRequest, room_id: str
     ) -> Tuple[int, JsonDict]:
-        return await _delete_room(
+        return await self._delete_room(
             request,
             room_id,
             self.auth,
@@ -226,6 +191,58 @@ class RoomRestServlet(RestServlet):
             self.pagination_handler,
         )
 
+    async def _delete_room(
+        self,
+        request: SynapseRequest,
+        room_id: str,
+        auth: "Auth",
+        room_shutdown_handler: "RoomShutdownHandler",
+        pagination_handler: "PaginationHandler",
+    ) -> Tuple[int, JsonDict]:
+        requester = await auth.get_user_by_req(request)
+        await assert_user_is_admin(auth, requester.user)
+
+        content = parse_json_object_from_request(request)
+
+        block = content.get("block", False)
+        if not isinstance(block, bool):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Param 'block' must be a boolean, if given",
+                Codes.BAD_JSON,
+            )
+
+        purge = content.get("purge", True)
+        if not isinstance(purge, bool):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Param 'purge' must be a boolean, if given",
+                Codes.BAD_JSON,
+            )
+
+        force_purge = content.get("force_purge", False)
+        if not isinstance(force_purge, bool):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Param 'force_purge' must be a boolean, if given",
+                Codes.BAD_JSON,
+            )
+
+        ret = await room_shutdown_handler.shutdown_room(
+            room_id=room_id,
+            new_room_user_id=content.get("new_room_user_id"),
+            new_room_name=content.get("room_name"),
+            message=content.get("message"),
+            requester_user_id=requester.user.to_string(),
+            block=block,
+        )
+
+        # Purge room
+        if purge:
+            await pagination_handler.purge_room(room_id, force=force_purge)
+
+        return 200, ret
+
 
 class RoomMembersRestServlet(RestServlet):
     """
@@ -617,55 +634,3 @@ class RoomEventContextServlet(RestServlet):
         )
 
         return 200, results
-
-
-async def _delete_room(
-    request: SynapseRequest,
-    room_id: str,
-    auth: "Auth",
-    room_shutdown_handler: "RoomShutdownHandler",
-    pagination_handler: "PaginationHandler",
-) -> Tuple[int, JsonDict]:
-    requester = await auth.get_user_by_req(request)
-    await assert_user_is_admin(auth, requester.user)
-
-    content = parse_json_object_from_request(request)
-
-    block = content.get("block", False)
-    if not isinstance(block, bool):
-        raise SynapseError(
-            HTTPStatus.BAD_REQUEST,
-            "Param 'block' must be a boolean, if given",
-            Codes.BAD_JSON,
-        )
-
-    purge = content.get("purge", True)
-    if not isinstance(purge, bool):
-        raise SynapseError(
-            HTTPStatus.BAD_REQUEST,
-            "Param 'purge' must be a boolean, if given",
-            Codes.BAD_JSON,
-        )
-
-    force_purge = content.get("force_purge", False)
-    if not isinstance(force_purge, bool):
-        raise SynapseError(
-            HTTPStatus.BAD_REQUEST,
-            "Param 'force_purge' must be a boolean, if given",
-            Codes.BAD_JSON,
-        )
-
-    ret = await room_shutdown_handler.shutdown_room(
-        room_id=room_id,
-        new_room_user_id=content.get("new_room_user_id"),
-        new_room_name=content.get("room_name"),
-        message=content.get("message"),
-        requester_user_id=requester.user.to_string(),
-        block=block,
-    )
-
-    # Purge room
-    if purge:
-        await pagination_handler.purge_room(room_id, force=force_purge)
-
-    return 200, ret
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index 0fa55e03b4..ba6db51c4c 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -17,8 +17,6 @@ import urllib.parse
 from typing import List, Optional
 from unittest.mock import Mock
 
-from parameterized import parameterized_class
-
 import synapse.rest.admin
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import Codes
@@ -29,13 +27,6 @@ from tests import unittest
 """Tests admin REST events for /rooms paths."""
 
 
-@parameterized_class(
-    ("method", "url_template"),
-    [
-        ("POST", "/_synapse/admin/v1/rooms/%s/delete"),
-        ("DELETE", "/_synapse/admin/v1/rooms/%s"),
-    ],
-)
 class DeleteRoomTestCase(unittest.HomeserverTestCase):
     servlets = [
         synapse.rest.admin.register_servlets,
@@ -67,7 +58,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         self.room_id = self.helper.create_room_as(
             self.other_user, tok=self.other_user_tok
         )
-        self.url = self.url_template % self.room_id
+        self.url = "/_synapse/admin/v1/rooms/%s" % self.room_id
 
     def test_requester_is_no_admin(self):
         """
@@ -75,7 +66,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         """
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             json.dumps({}),
             access_token=self.other_user_tok,
@@ -88,10 +79,10 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         """
         Check that unknown rooms/server return error 404.
         """
-        url = self.url_template % "!unknown:test"
+        url = "/_synapse/admin/v1/rooms/%s" % "!unknown:test"
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             url,
             json.dumps({}),
             access_token=self.admin_user_tok,
@@ -104,10 +95,10 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         """
         Check that invalid room names, return an error 400.
         """
-        url = self.url_template % "invalidroom"
+        url = "/_synapse/admin/v1/rooms/%s" % "invalidroom"
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             url,
             json.dumps({}),
             access_token=self.admin_user_tok,
@@ -126,7 +117,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"new_room_user_id": "@unknown:test"})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -145,7 +136,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"new_room_user_id": "@not:exist.bla"})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -164,7 +155,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"block": "NotBool"})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -180,7 +171,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"purge": "NotBool"})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -206,7 +197,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"block": True, "purge": True})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url.encode("ascii"),
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -239,7 +230,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"block": False, "purge": True})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url.encode("ascii"),
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -273,7 +264,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         body = json.dumps({"block": False, "purge": False})
 
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url.encode("ascii"),
             content=body.encode(encoding="utf_8"),
             access_token=self.admin_user_tok,
@@ -319,7 +310,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
 
         # Test that the admin can still send shutdown
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             json.dumps({"new_room_user_id": self.admin_user}),
             access_token=self.admin_user_tok,
@@ -365,7 +356,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
 
         # Test that the admin can still send shutdown
         channel = self.make_request(
-            self.method,
+            "DELETE",
             self.url,
             json.dumps({"new_room_user_id": self.admin_user}),
             access_token=self.admin_user_tok,
-- 
cgit 1.5.1


From 69ab3dddbc1595ee64c428df7a7f3c861a84b5b0 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Mon, 1 Nov 2021 15:45:56 +0000
Subject: Make `check_event_allowed` module API callback not fail open (accept
 events) when an exception is raised (#11033)

---
 changelog.d/11033.bugfix                    |  1 +
 docs/modules/third_party_rules_callbacks.md |  8 ++++++++
 synapse/api/errors.py                       |  7 +++++++
 synapse/events/third_party_rules.py         |  9 +++++----
 tests/rest/client/test_third_party_rules.py | 16 +++-------------
 5 files changed, 24 insertions(+), 17 deletions(-)
 create mode 100644 changelog.d/11033.bugfix

diff --git a/changelog.d/11033.bugfix b/changelog.d/11033.bugfix
new file mode 100644
index 0000000000..fa99f187b8
--- /dev/null
+++ b/changelog.d/11033.bugfix
@@ -0,0 +1 @@
+Do not accept events if a third-party rule module API callback raises an exception.
diff --git a/docs/modules/third_party_rules_callbacks.md b/docs/modules/third_party_rules_callbacks.md
index a16e272f79..a3a17096a8 100644
--- a/docs/modules/third_party_rules_callbacks.md
+++ b/docs/modules/third_party_rules_callbacks.md
@@ -43,6 +43,14 @@ event with new data by returning the new event's data as a dictionary. In order
 that, it is recommended the module calls `event.get_dict()` to get the current event as a
 dictionary, and modify the returned dictionary accordingly.
 
+If `check_event_allowed` raises an exception, the module is assumed to have failed.
+The event will not be accepted but is not treated as explicitly rejected, either.
+An HTTP request causing the module check will likely result in a 500 Internal
+Server Error.
+
+When the boolean returned by the module is `False`, the event is rejected.
+(Module developers should not use exceptions for rejection.)
+
 Note that replacing the event only works for events sent by local users, not for events
 received over federation.
 
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 685d1c25cf..85302163da 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -596,3 +596,10 @@ class ShadowBanError(Exception):
 
     This should be caught and a proper "fake" success response sent to the user.
     """
+
+
+class ModuleFailedException(Exception):
+    """
+    Raised when a module API callback fails, for example because it raised an
+    exception.
+    """
diff --git a/synapse/events/third_party_rules.py b/synapse/events/third_party_rules.py
index 8816ef4b76..1bb8ca7145 100644
--- a/synapse/events/third_party_rules.py
+++ b/synapse/events/third_party_rules.py
@@ -14,7 +14,7 @@
 import logging
 from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Optional, Tuple
 
-from synapse.api.errors import SynapseError
+from synapse.api.errors import ModuleFailedException, SynapseError
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.types import Requester, StateMap
@@ -233,9 +233,10 @@ class ThirdPartyEventRules:
                 # This module callback needs a rework so that hacks such as
                 # this one are not necessary.
                 raise e
-            except Exception as e:
-                logger.warning("Failed to run module API callback %s: %s", callback, e)
-                continue
+            except Exception:
+                raise ModuleFailedException(
+                    "Failed to run `check_event_allowed` module API callback"
+                )
 
             # Return if the event shouldn't be allowed or if the module came up with a
             # replacement dict for the event.
diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py
index 1c42c46630..4e71b6ec12 100644
--- a/tests/rest/client/test_third_party_rules.py
+++ b/tests/rest/client/test_third_party_rules.py
@@ -216,19 +216,9 @@ class ThirdPartyRulesTestCase(unittest.FederatingHomeserverTestCase):
             {"x": "x"},
             access_token=self.tok,
         )
-        # check_event_allowed has some error handling, so it shouldn't 500 just because a
-        # module did something bad.
-        self.assertEqual(channel.code, 200, channel.result)
-        event_id = channel.json_body["event_id"]
-
-        channel = self.make_request(
-            "GET",
-            "/_matrix/client/r0/rooms/%s/event/%s" % (self.room_id, event_id),
-            access_token=self.tok,
-        )
-        self.assertEqual(channel.code, 200, channel.result)
-        ev = channel.json_body
-        self.assertEqual(ev["content"]["x"], "x")
+        # Because check_event_allowed raises an exception, it leads to a
+        # 500 Internal Server Error
+        self.assertEqual(channel.code, 500, channel.result)
 
     def test_modify_event(self):
         """The module can return a modified version of the event"""
-- 
cgit 1.5.1


From caa706d82545cda8d0f7c7243623a6de898b55bc Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Mon, 1 Nov 2021 17:10:09 +0100
Subject: Fix a bug in unit test `test_block_room_and_not_purge` (#11226)

---
 changelog.d/11226.misc        | 1 +
 tests/rest/admin/test_room.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/11226.misc

diff --git a/changelog.d/11226.misc b/changelog.d/11226.misc
new file mode 100644
index 0000000000..9ed4760ae0
--- /dev/null
+++ b/changelog.d/11226.misc
@@ -0,0 +1 @@
+Fix a bug in unit test `test_block_room_and_not_purge`.
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index ba6db51c4c..b62a7248e8 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -261,7 +261,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
         # Assert one user in room
         self._is_member(room_id=self.room_id, user_id=self.other_user)
 
-        body = json.dumps({"block": False, "purge": False})
+        body = json.dumps({"block": True, "purge": False})
 
         channel = self.make_request(
             "DELETE",
@@ -278,7 +278,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
 
         with self.assertRaises(AssertionError):
             self._is_purged(self.room_id)
-        self._is_blocked(self.room_id, expect=False)
+        self._is_blocked(self.room_id, expect=True)
         self._has_no_members(self.room_id)
 
     def test_shutdown_room_consent(self):
-- 
cgit 1.5.1


From e81fa9264873369653171157514ff68226491fff Mon Sep 17 00:00:00 2001
From: Shay <shaysquared@gmail.com>
Date: Mon, 1 Nov 2021 09:28:04 -0700
Subject: Add `use_float=true` to ijson calls in Synapse (#11217)

* add use_float=true to ijson calls

* lints

* add changelog

* Update changelog.d/11217.bugfix

Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com>

Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com>
---
 changelog.d/11217.bugfix               | 1 +
 synapse/federation/transport/client.py | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 changelog.d/11217.bugfix

diff --git a/changelog.d/11217.bugfix b/changelog.d/11217.bugfix
new file mode 100644
index 0000000000..67ebb0d0e3
--- /dev/null
+++ b/changelog.d/11217.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in 1.35.0 which made it impossible to join rooms that return a `send_join` response containing floats.
\ No newline at end of file
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index d963178838..10b5aa5af8 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1310,14 +1310,17 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
         self._coro_state = ijson.items_coro(
             _event_list_parser(room_version, self._response.state),
             prefix + "state.item",
+            use_float=True,
         )
         self._coro_auth = ijson.items_coro(
             _event_list_parser(room_version, self._response.auth_events),
             prefix + "auth_chain.item",
+            use_float=True,
         )
         self._coro_event = ijson.kvitems_coro(
             _event_parser(self._response.event_dict),
             prefix + "org.matrix.msc3083.v2.event",
+            use_float=True,
         )
 
     def write(self, data: bytes) -> int:
-- 
cgit 1.5.1


From f5c6a80886ac00482aaffa8e8ce3d98b31eab661 Mon Sep 17 00:00:00 2001
From: Shay <shaysquared@gmail.com>
Date: Mon, 1 Nov 2021 10:26:02 -0700
Subject: Handle missing Content-Type header when accessing remote media
 (#11200)

* add code to handle missing content-type header and a test to verify that it works

* add handling for missing content-type in the /upload endpoint as well

* slightly refactor test code to put private method in approriate place

* handle possible null value for content-type when pulling from the local db

* add changelog

* refactor test and add code to handle missing content-type in cached remote media

* requested changes

* Update changelog.d/11200.bugfix

Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com>

Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com>
---
 changelog.d/11200.bugfix                  |  1 +
 synapse/rest/media/v1/media_repository.py | 12 +++++++++++-
 synapse/rest/media/v1/upload_resource.py  |  2 +-
 tests/rest/media/v1/test_media_storage.py | 18 ++++++++++++++++--
 4 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/11200.bugfix

diff --git a/changelog.d/11200.bugfix b/changelog.d/11200.bugfix
new file mode 100644
index 0000000000..c855081986
--- /dev/null
+++ b/changelog.d/11200.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug wherein a missing `Content-Type` header when downloading remote media would cause Synapse to throw an error.
\ No newline at end of file
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index abd88a2d4f..244ba261bb 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -215,6 +215,8 @@ class MediaRepository:
         self.mark_recently_accessed(None, media_id)
 
         media_type = media_info["media_type"]
+        if not media_type:
+            media_type = "application/octet-stream"
         media_length = media_info["media_length"]
         upload_name = name if name else media_info["upload_name"]
         url_cache = media_info["url_cache"]
@@ -333,6 +335,9 @@ class MediaRepository:
                 logger.info("Media is quarantined")
                 raise NotFoundError()
 
+            if not media_info["media_type"]:
+                media_info["media_type"] = "application/octet-stream"
+
             responder = await self.media_storage.fetch_media(file_info)
             if responder:
                 return responder, media_info
@@ -354,6 +359,8 @@ class MediaRepository:
                 raise e
 
         file_id = media_info["filesystem_id"]
+        if not media_info["media_type"]:
+            media_info["media_type"] = "application/octet-stream"
         file_info = FileInfo(server_name, file_id)
 
         # We generate thumbnails even if another process downloaded the media
@@ -445,7 +452,10 @@ class MediaRepository:
 
             await finish()
 
-            media_type = headers[b"Content-Type"][0].decode("ascii")
+            if b"Content-Type" in headers:
+                media_type = headers[b"Content-Type"][0].decode("ascii")
+            else:
+                media_type = "application/octet-stream"
             upload_name = get_filename_from_headers(headers)
             time_now_ms = self.clock.time_msec()
 
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index 7dcb1428e4..8162094cf6 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -80,7 +80,7 @@ class UploadResource(DirectServeJsonResource):
             assert content_type_headers  # for mypy
             media_type = content_type_headers[0].decode("ascii")
         else:
-            raise SynapseError(msg="Upload request missing 'Content-Type'", code=400)
+            media_type = "application/octet-stream"
 
         # if headers.hasHeader(b"Content-Disposition"):
         #     disposition = headers.getRawHeaders(b"Content-Disposition")[0]
diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py
index 4ae00755c9..4cf1ed5ddf 100644
--- a/tests/rest/media/v1/test_media_storage.py
+++ b/tests/rest/media/v1/test_media_storage.py
@@ -248,7 +248,7 @@ class MediaRepoTests(unittest.HomeserverTestCase):
 
         self.media_id = "example.com/12345"
 
-    def _req(self, content_disposition):
+    def _req(self, content_disposition, include_content_type=True):
 
         channel = make_request(
             self.reactor,
@@ -271,8 +271,11 @@ class MediaRepoTests(unittest.HomeserverTestCase):
 
         headers = {
             b"Content-Length": [b"%d" % (len(self.test_image.data))],
-            b"Content-Type": [self.test_image.content_type],
         }
+
+        if include_content_type:
+            headers[b"Content-Type"] = [self.test_image.content_type]
+
         if content_disposition:
             headers[b"Content-Disposition"] = [content_disposition]
 
@@ -285,6 +288,17 @@ class MediaRepoTests(unittest.HomeserverTestCase):
 
         return channel
 
+    def test_handle_missing_content_type(self):
+        channel = self._req(
+            b"inline; filename=out" + self.test_image.extension,
+            include_content_type=False,
+        )
+        headers = channel.headers
+        self.assertEqual(channel.code, 200)
+        self.assertEqual(
+            headers.getRawHeaders(b"Content-Type"), [b"application/octet-stream"]
+        )
+
     def test_disposition_filename_ascii(self):
         """
         If the filename is filename=<ascii> then Synapse will decode it as an
-- 
cgit 1.5.1


From 93aa670642cafcb6ed732094887bb2aac4b34b0f Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Mon, 1 Nov 2021 18:29:51 +0100
Subject: Update outdated links in `PULL_REQUEST_TEMPLATE.md` (#11225)

---
 .github/PULL_REQUEST_TEMPLATE.md | 9 +++++----
 changelog.d/11225.misc           | 1 +
 2 files changed, 6 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/11225.misc

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index fc22d89426..6c3a998499 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,12 +1,13 @@
 ### Pull Request Checklist
 
-<!-- Please read CONTRIBUTING.md before submitting your pull request -->
+<!-- Please read https://matrix-org.github.io/synapse/latest/development/contributing_guide.html before submitting your pull request -->
 
 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#changelog). The entry should:
+* [ ] Pull request includes a [changelog file](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should:
   - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
   - Use markdown where necessary, mostly for `code blocks`.
   - End with either a period (.) or an exclamation mark (!).
   - Start with a capital letter.
-* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#sign-off)
-* [ ] Code style is correct (run the [linters](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#code-style))
+* [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
+* [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
+  (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))
diff --git a/changelog.d/11225.misc b/changelog.d/11225.misc
new file mode 100644
index 0000000000..f14f65f9d4
--- /dev/null
+++ b/changelog.d/11225.misc
@@ -0,0 +1 @@
+Replace outdated links in the pull request checklist with links to the rendered documentation.
-- 
cgit 1.5.1


From 46d0937447479761a22a8c843f6ba51bbcdc914b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 2 Nov 2021 00:17:35 +0000
Subject: ObservableDeferred: run observers in order (#11229)

---
 changelog.d/11229.misc                   |   1 +
 synapse/util/async_helpers.py            |  34 +++---
 tests/util/caches/test_deferred_cache.py |   4 +-
 tests/util/test_async_helpers.py         | 173 +++++++++++++++++++++++++++++++
 tests/util/test_async_utils.py           | 106 -------------------
 5 files changed, 193 insertions(+), 125 deletions(-)
 create mode 100644 changelog.d/11229.misc
 create mode 100644 tests/util/test_async_helpers.py
 delete mode 100644 tests/util/test_async_utils.py

diff --git a/changelog.d/11229.misc b/changelog.d/11229.misc
new file mode 100644
index 0000000000..7bb01cf079
--- /dev/null
+++ b/changelog.d/11229.misc
@@ -0,0 +1 @@
+`ObservableDeferred`: run registered observers in order.
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index 5df80ea8e7..96efc5f3e3 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -22,11 +22,11 @@ from typing import (
     Any,
     Awaitable,
     Callable,
+    Collection,
     Dict,
     Generic,
     Hashable,
     Iterable,
-    List,
     Optional,
     Set,
     TypeVar,
@@ -76,12 +76,17 @@ class ObservableDeferred(Generic[_T]):
     def __init__(self, deferred: "defer.Deferred[_T]", consumeErrors: bool = False):
         object.__setattr__(self, "_deferred", deferred)
         object.__setattr__(self, "_result", None)
-        object.__setattr__(self, "_observers", set())
+        object.__setattr__(self, "_observers", [])
 
         def callback(r):
             object.__setattr__(self, "_result", (True, r))
-            while self._observers:
-                observer = self._observers.pop()
+
+            # once we have set _result, no more entries will be added to _observers,
+            # so it's safe to replace it with the empty tuple.
+            observers = self._observers
+            object.__setattr__(self, "_observers", ())
+
+            for observer in observers:
                 try:
                     observer.callback(r)
                 except Exception as e:
@@ -95,12 +100,16 @@ class ObservableDeferred(Generic[_T]):
 
         def errback(f):
             object.__setattr__(self, "_result", (False, f))
-            while self._observers:
+
+            # once we have set _result, no more entries will be added to _observers,
+            # so it's safe to replace it with the empty tuple.
+            observers = self._observers
+            object.__setattr__(self, "_observers", ())
+
+            for observer in observers:
                 # This is a little bit of magic to correctly propagate stack
                 # traces when we `await` on one of the observer deferreds.
                 f.value.__failure__ = f
-
-                observer = self._observers.pop()
                 try:
                     observer.errback(f)
                 except Exception as e:
@@ -127,20 +136,13 @@ class ObservableDeferred(Generic[_T]):
         """
         if not self._result:
             d: "defer.Deferred[_T]" = defer.Deferred()
-
-            def remove(r):
-                self._observers.discard(d)
-                return r
-
-            d.addBoth(remove)
-
-            self._observers.add(d)
+            self._observers.append(d)
             return d
         else:
             success, res = self._result
             return defer.succeed(res) if success else defer.fail(res)
 
-    def observers(self) -> "List[defer.Deferred[_T]]":
+    def observers(self) -> "Collection[defer.Deferred[_T]]":
         return self._observers
 
     def has_called(self) -> bool:
diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py
index 54a88a8325..c613ce3f10 100644
--- a/tests/util/caches/test_deferred_cache.py
+++ b/tests/util/caches/test_deferred_cache.py
@@ -47,9 +47,7 @@ class DeferredCacheTestCase(TestCase):
             self.assertTrue(set_d.called)
             return r
 
-        # TODO: Actually ObservableDeferred *doesn't* run its tests in order on py3.8.
-        #   maybe we should fix that?
-        # get_d.addCallback(check1)
+        get_d.addCallback(check1)
 
         # now fire off all the deferreds
         origin_d.callback(99)
diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py
new file mode 100644
index 0000000000..ab89cab812
--- /dev/null
+++ b/tests/util/test_async_helpers.py
@@ -0,0 +1,173 @@
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+from twisted.internet.defer import CancelledError, Deferred
+from twisted.internet.task import Clock
+
+from synapse.logging.context import (
+    SENTINEL_CONTEXT,
+    LoggingContext,
+    PreserveLoggingContext,
+    current_context,
+)
+from synapse.util.async_helpers import ObservableDeferred, timeout_deferred
+
+from tests.unittest import TestCase
+
+
+class ObservableDeferredTest(TestCase):
+    def test_succeed(self):
+        origin_d = Deferred()
+        observable = ObservableDeferred(origin_d)
+
+        observer1 = observable.observe()
+        observer2 = observable.observe()
+
+        self.assertFalse(observer1.called)
+        self.assertFalse(observer2.called)
+
+        # check the first observer is called first
+        def check_called_first(res):
+            self.assertFalse(observer2.called)
+            return res
+
+        observer1.addBoth(check_called_first)
+
+        # store the results
+        results = [None, None]
+
+        def check_val(res, idx):
+            results[idx] = res
+            return res
+
+        observer1.addCallback(check_val, 0)
+        observer2.addCallback(check_val, 1)
+
+        origin_d.callback(123)
+        self.assertEqual(results[0], 123, "observer 1 callback result")
+        self.assertEqual(results[1], 123, "observer 2 callback result")
+
+    def test_failure(self):
+        origin_d = Deferred()
+        observable = ObservableDeferred(origin_d, consumeErrors=True)
+
+        observer1 = observable.observe()
+        observer2 = observable.observe()
+
+        self.assertFalse(observer1.called)
+        self.assertFalse(observer2.called)
+
+        # check the first observer is called first
+        def check_called_first(res):
+            self.assertFalse(observer2.called)
+            return res
+
+        observer1.addBoth(check_called_first)
+
+        # store the results
+        results = [None, None]
+
+        def check_val(res, idx):
+            results[idx] = res
+            return None
+
+        observer1.addErrback(check_val, 0)
+        observer2.addErrback(check_val, 1)
+
+        try:
+            raise Exception("gah!")
+        except Exception as e:
+            origin_d.errback(e)
+        self.assertEqual(str(results[0].value), "gah!", "observer 1 errback result")
+        self.assertEqual(str(results[1].value), "gah!", "observer 2 errback result")
+
+
+class TimeoutDeferredTest(TestCase):
+    def setUp(self):
+        self.clock = Clock()
+
+    def test_times_out(self):
+        """Basic test case that checks that the original deferred is cancelled and that
+        the timing-out deferred is errbacked
+        """
+        cancelled = [False]
+
+        def canceller(_d):
+            cancelled[0] = True
+
+        non_completing_d = Deferred(canceller)
+        timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock)
+
+        self.assertNoResult(timing_out_d)
+        self.assertFalse(cancelled[0], "deferred was cancelled prematurely")
+
+        self.clock.pump((1.0,))
+
+        self.assertTrue(cancelled[0], "deferred was not cancelled by timeout")
+        self.failureResultOf(timing_out_d, defer.TimeoutError)
+
+    def test_times_out_when_canceller_throws(self):
+        """Test that we have successfully worked around
+        https://twistedmatrix.com/trac/ticket/9534"""
+
+        def canceller(_d):
+            raise Exception("can't cancel this deferred")
+
+        non_completing_d = Deferred(canceller)
+        timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock)
+
+        self.assertNoResult(timing_out_d)
+
+        self.clock.pump((1.0,))
+
+        self.failureResultOf(timing_out_d, defer.TimeoutError)
+
+    def test_logcontext_is_preserved_on_cancellation(self):
+        blocking_was_cancelled = [False]
+
+        @defer.inlineCallbacks
+        def blocking():
+            non_completing_d = Deferred()
+            with PreserveLoggingContext():
+                try:
+                    yield non_completing_d
+                except CancelledError:
+                    blocking_was_cancelled[0] = True
+                    raise
+
+        with LoggingContext("one") as context_one:
+            # the errbacks should be run in the test logcontext
+            def errback(res, deferred_name):
+                self.assertIs(
+                    current_context(),
+                    context_one,
+                    "errback %s run in unexpected logcontext %s"
+                    % (deferred_name, current_context()),
+                )
+                return res
+
+            original_deferred = blocking()
+            original_deferred.addErrback(errback, "orig")
+            timing_out_d = timeout_deferred(original_deferred, 1.0, self.clock)
+            self.assertNoResult(timing_out_d)
+            self.assertIs(current_context(), SENTINEL_CONTEXT)
+            timing_out_d.addErrback(errback, "timingout")
+
+            self.clock.pump((1.0,))
+
+            self.assertTrue(
+                blocking_was_cancelled[0], "non-completing deferred was not cancelled"
+            )
+            self.failureResultOf(timing_out_d, defer.TimeoutError)
+            self.assertIs(current_context(), context_one)
diff --git a/tests/util/test_async_utils.py b/tests/util/test_async_utils.py
deleted file mode 100644
index 069f875962..0000000000
--- a/tests/util/test_async_utils.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Copyright 2019 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from twisted.internet import defer
-from twisted.internet.defer import CancelledError, Deferred
-from twisted.internet.task import Clock
-
-from synapse.logging.context import (
-    SENTINEL_CONTEXT,
-    LoggingContext,
-    PreserveLoggingContext,
-    current_context,
-)
-from synapse.util.async_helpers import timeout_deferred
-
-from tests.unittest import TestCase
-
-
-class TimeoutDeferredTest(TestCase):
-    def setUp(self):
-        self.clock = Clock()
-
-    def test_times_out(self):
-        """Basic test case that checks that the original deferred is cancelled and that
-        the timing-out deferred is errbacked
-        """
-        cancelled = [False]
-
-        def canceller(_d):
-            cancelled[0] = True
-
-        non_completing_d = Deferred(canceller)
-        timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock)
-
-        self.assertNoResult(timing_out_d)
-        self.assertFalse(cancelled[0], "deferred was cancelled prematurely")
-
-        self.clock.pump((1.0,))
-
-        self.assertTrue(cancelled[0], "deferred was not cancelled by timeout")
-        self.failureResultOf(timing_out_d, defer.TimeoutError)
-
-    def test_times_out_when_canceller_throws(self):
-        """Test that we have successfully worked around
-        https://twistedmatrix.com/trac/ticket/9534"""
-
-        def canceller(_d):
-            raise Exception("can't cancel this deferred")
-
-        non_completing_d = Deferred(canceller)
-        timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock)
-
-        self.assertNoResult(timing_out_d)
-
-        self.clock.pump((1.0,))
-
-        self.failureResultOf(timing_out_d, defer.TimeoutError)
-
-    def test_logcontext_is_preserved_on_cancellation(self):
-        blocking_was_cancelled = [False]
-
-        @defer.inlineCallbacks
-        def blocking():
-            non_completing_d = Deferred()
-            with PreserveLoggingContext():
-                try:
-                    yield non_completing_d
-                except CancelledError:
-                    blocking_was_cancelled[0] = True
-                    raise
-
-        with LoggingContext("one") as context_one:
-            # the errbacks should be run in the test logcontext
-            def errback(res, deferred_name):
-                self.assertIs(
-                    current_context(),
-                    context_one,
-                    "errback %s run in unexpected logcontext %s"
-                    % (deferred_name, current_context()),
-                )
-                return res
-
-            original_deferred = blocking()
-            original_deferred.addErrback(errback, "orig")
-            timing_out_d = timeout_deferred(original_deferred, 1.0, self.clock)
-            self.assertNoResult(timing_out_d)
-            self.assertIs(current_context(), SENTINEL_CONTEXT)
-            timing_out_d.addErrback(errback, "timingout")
-
-            self.clock.pump((1.0,))
-
-            self.assertTrue(
-                blocking_was_cancelled[0], "non-completing deferred was not cancelled"
-            )
-            self.failureResultOf(timing_out_d, defer.TimeoutError)
-            self.assertIs(current_context(), context_one)
-- 
cgit 1.5.1


From 753720184042e01bf56478d15bd8c8db11da4b69 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Tue, 2 Nov 2021 11:01:13 +0100
Subject: Add search by room ID and room alias to List Room admin API (#11099)

Fixes: #10874
Signed-off-by: Dirk Klimpel dirk@klimpel.org
---
 changelog.d/11099.feature              |  1 +
 docs/admin_api/rooms.md                | 11 +++--
 synapse/storage/databases/main/room.py | 29 ++++++-----
 tests/rest/admin/test_room.py          | 88 +++++++++++++++++++---------------
 4 files changed, 76 insertions(+), 53 deletions(-)
 create mode 100644 changelog.d/11099.feature

diff --git a/changelog.d/11099.feature b/changelog.d/11099.feature
new file mode 100644
index 0000000000..c9126d4a9d
--- /dev/null
+++ b/changelog.d/11099.feature
@@ -0,0 +1 @@
+Add search by room ID and room alias to List Room admin API.
\ No newline at end of file
diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md
index 62eeff9e1a..1fc3cc3c42 100644
--- a/docs/admin_api/rooms.md
+++ b/docs/admin_api/rooms.md
@@ -38,9 +38,14 @@ The following query parameters are available:
   - `history_visibility` - Rooms are ordered alphabetically by visibility of history of the room.
   - `state_events` - Rooms are ordered by number of state events. Largest to smallest.
 * `dir` - Direction of room order. Either `f` for forwards or `b` for backwards. Setting
-          this value to `b` will reverse the above sort order. Defaults to `f`.
-* `search_term` - Filter rooms by their room name. Search term can be contained in any
-                  part of the room name. Defaults to no filtering.
+  this value to `b` will reverse the above sort order. Defaults to `f`.
+* `search_term` - Filter rooms by their room name, canonical alias and room id.
+  Specifically, rooms are selected if the search term is contained in
+  - the room's name,
+  - the local part of the room's canonical alias, or
+  - the complete (local and server part) room's id (case sensitive).
+
+  Defaults to no filtering.
 
 **Response**
 
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index f879bbe7c7..cefc77fa0f 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -412,22 +412,33 @@ class RoomWorkerStore(SQLBaseStore):
             limit: maximum amount of rooms to retrieve
             order_by: the sort order of the returned list
             reverse_order: whether to reverse the room list
-            search_term: a string to filter room names by
+            search_term: a string to filter room names,
+                canonical alias and room ids by.
+                Room ID must match exactly. Canonical alias must match a substring of the local part.
         Returns:
             A list of room dicts and an integer representing the total number of
             rooms that exist given this query
         """
         # Filter room names by a string
         where_statement = ""
+        search_pattern = []
         if search_term:
-            where_statement = "WHERE LOWER(state.name) LIKE ?"
+            where_statement = """
+                WHERE LOWER(state.name) LIKE ?
+                OR LOWER(state.canonical_alias) LIKE ?
+                OR state.room_id = ?
+            """
 
             # Our postgres db driver converts ? -> %s in SQL strings as that's the
             # placeholder for postgres.
             # HOWEVER, if you put a % into your SQL then everything goes wibbly.
             # To get around this, we're going to surround search_term with %'s
             # before giving it to the database in python instead
-            search_term = "%" + search_term.lower() + "%"
+            search_pattern = [
+                "%" + search_term.lower() + "%",
+                "#%" + search_term.lower() + "%:%",
+                search_term,
+            ]
 
         # Set ordering
         if RoomSortOrder(order_by) == RoomSortOrder.SIZE:
@@ -519,12 +530,9 @@ class RoomWorkerStore(SQLBaseStore):
         )
 
         def _get_rooms_paginate_txn(txn):
-            # Execute the data query
-            sql_values = (limit, start)
-            if search_term:
-                # Add the search term into the WHERE clause
-                sql_values = (search_term,) + sql_values
-            txn.execute(info_sql, sql_values)
+            # Add the search term into the WHERE clause
+            # and execute the data query
+            txn.execute(info_sql, search_pattern + [limit, start])
 
             # Refactor room query data into a structured dictionary
             rooms = []
@@ -551,8 +559,7 @@ class RoomWorkerStore(SQLBaseStore):
             # Execute the count query
 
             # Add the search term into the WHERE clause if present
-            sql_values = (search_term,) if search_term else ()
-            txn.execute(count_sql, sql_values)
+            txn.execute(count_sql, search_pattern)
 
             room_count = txn.fetchone()
             return rooms, room_count[0]
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index b62a7248e8..46116644ce 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -680,36 +680,6 @@ class RoomTestCase(unittest.HomeserverTestCase):
         reversing the order, etc.
         """
 
-        def _set_canonical_alias(room_id: str, test_alias: str, admin_user_tok: str):
-            # Create a new alias to this room
-            url = "/_matrix/client/r0/directory/room/%s" % (
-                urllib.parse.quote(test_alias),
-            )
-            channel = self.make_request(
-                "PUT",
-                url.encode("ascii"),
-                {"room_id": room_id},
-                access_token=admin_user_tok,
-            )
-            self.assertEqual(
-                200, int(channel.result["code"]), msg=channel.result["body"]
-            )
-
-            # Set this new alias as the canonical alias for this room
-            self.helper.send_state(
-                room_id,
-                "m.room.aliases",
-                {"aliases": [test_alias]},
-                tok=admin_user_tok,
-                state_key="test",
-            )
-            self.helper.send_state(
-                room_id,
-                "m.room.canonical_alias",
-                {"alias": test_alias},
-                tok=admin_user_tok,
-            )
-
         def _order_test(
             order_type: str,
             expected_room_list: List[str],
@@ -781,9 +751,9 @@ class RoomTestCase(unittest.HomeserverTestCase):
         )
 
         # Set room canonical room aliases
-        _set_canonical_alias(room_id_1, "#A_alias:test", self.admin_user_tok)
-        _set_canonical_alias(room_id_2, "#B_alias:test", self.admin_user_tok)
-        _set_canonical_alias(room_id_3, "#C_alias:test", self.admin_user_tok)
+        self._set_canonical_alias(room_id_1, "#A_alias:test", self.admin_user_tok)
+        self._set_canonical_alias(room_id_2, "#B_alias:test", self.admin_user_tok)
+        self._set_canonical_alias(room_id_3, "#C_alias:test", self.admin_user_tok)
 
         # Set room member size in the reverse order. room 1 -> 1 member, 2 -> 2, 3 -> 3
         user_1 = self.register_user("bob1", "pass")
@@ -850,7 +820,7 @@ class RoomTestCase(unittest.HomeserverTestCase):
         room_id_2 = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok)
 
         room_name_1 = "something"
-        room_name_2 = "else"
+        room_name_2 = "LoremIpsum"
 
         # Set the name for each room
         self.helper.send_state(
@@ -866,6 +836,8 @@ class RoomTestCase(unittest.HomeserverTestCase):
             tok=self.admin_user_tok,
         )
 
+        self._set_canonical_alias(room_id_1, "#Room_Alias1:test", self.admin_user_tok)
+
         def _search_test(
             expected_room_id: Optional[str],
             search_term: str,
@@ -914,24 +886,36 @@ class RoomTestCase(unittest.HomeserverTestCase):
                 r = rooms[0]
                 self.assertEqual(expected_room_id, r["room_id"])
 
-        # Perform search tests
+        # Test searching by room name
         _search_test(room_id_1, "something")
         _search_test(room_id_1, "thing")
 
-        _search_test(room_id_2, "else")
-        _search_test(room_id_2, "se")
+        _search_test(room_id_2, "LoremIpsum")
+        _search_test(room_id_2, "lorem")
 
         # Test case insensitive
         _search_test(room_id_1, "SOMETHING")
         _search_test(room_id_1, "THING")
 
-        _search_test(room_id_2, "ELSE")
-        _search_test(room_id_2, "SE")
+        _search_test(room_id_2, "LOREMIPSUM")
+        _search_test(room_id_2, "LOREM")
 
         _search_test(None, "foo")
         _search_test(None, "bar")
         _search_test(None, "", expected_http_code=400)
 
+        # Test that the whole room id returns the room
+        _search_test(room_id_1, room_id_1)
+        # Test that the search by room_id is case sensitive
+        _search_test(None, room_id_1.lower())
+        # Test search part of local part of room id do not match
+        _search_test(None, room_id_1[1:10])
+
+        # Test that whole room alias return no result, because of domain
+        _search_test(None, "#Room_Alias1:test")
+        # Test search local part of alias
+        _search_test(room_id_1, "alias1")
+
     def test_search_term_non_ascii(self):
         """Test that searching for a room with non-ASCII characters works correctly"""
 
@@ -1114,6 +1098,32 @@ class RoomTestCase(unittest.HomeserverTestCase):
         # the create_room already does the right thing, so no need to verify that we got
         # the state events it created.
 
+    def _set_canonical_alias(self, room_id: str, test_alias: str, admin_user_tok: str):
+        # Create a new alias to this room
+        url = "/_matrix/client/r0/directory/room/%s" % (urllib.parse.quote(test_alias),)
+        channel = self.make_request(
+            "PUT",
+            url.encode("ascii"),
+            {"room_id": room_id},
+            access_token=admin_user_tok,
+        )
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+
+        # Set this new alias as the canonical alias for this room
+        self.helper.send_state(
+            room_id,
+            "m.room.aliases",
+            {"aliases": [test_alias]},
+            tok=admin_user_tok,
+            state_key="test",
+        )
+        self.helper.send_state(
+            room_id,
+            "m.room.canonical_alias",
+            {"alias": test_alias},
+            tok=admin_user_tok,
+        )
+
 
 class JoinAliasRoomTestCase(unittest.HomeserverTestCase):
 
-- 
cgit 1.5.1


From c9c3aea9b189cb606d7ec2905dad2c87acc039ef Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Tue, 2 Nov 2021 10:39:02 +0000
Subject: Fix providing a `RoomStreamToken` instance to
 `_notify_app_services_ephemeral` (#11137)

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 changelog.d/11137.misc                     |  1 +
 synapse/handlers/appservice.py             | 22 +++++++++++++----
 synapse/notifier.py                        | 38 +++++++-----------------------
 synapse/storage/databases/main/devices.py  |  4 ++--
 synapse/storage/databases/main/presence.py |  2 +-
 5 files changed, 30 insertions(+), 37 deletions(-)
 create mode 100644 changelog.d/11137.misc

diff --git a/changelog.d/11137.misc b/changelog.d/11137.misc
new file mode 100644
index 0000000000..f0d6476f48
--- /dev/null
+++ b/changelog.d/11137.misc
@@ -0,0 +1 @@
+Remove and document unnecessary `RoomStreamToken` checks in application service ephemeral event code.
\ No newline at end of file
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index 36c206dae6..67f8ffcaff 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -182,7 +182,7 @@ class ApplicationServicesHandler:
     def notify_interested_services_ephemeral(
         self,
         stream_key: str,
-        new_token: Optional[int],
+        new_token: Union[int, RoomStreamToken],
         users: Optional[Collection[Union[str, UserID]]] = None,
     ) -> None:
         """
@@ -203,7 +203,7 @@ class ApplicationServicesHandler:
                 Appservices will only receive ephemeral events that fall within their
                 registered user and room namespaces.
 
-            new_token: The latest stream token.
+            new_token: The stream token of the event.
             users: The users that should be informed of the new event, if any.
         """
         if not self.notify_appservices:
@@ -212,6 +212,19 @@ class ApplicationServicesHandler:
         if stream_key not in ("typing_key", "receipt_key", "presence_key"):
             return
 
+        # Assert that new_token is an integer (and not a RoomStreamToken).
+        # All of the supported streams that this function handles use an
+        # integer to track progress (rather than a RoomStreamToken - a
+        # vector clock implementation) as they don't support multiple
+        # stream writers.
+        #
+        # As a result, we simply assert that new_token is an integer.
+        # If we do end up needing to pass a RoomStreamToken down here
+        # in the future, using RoomStreamToken.stream (the minimum stream
+        # position) to convert to an ascending integer value should work.
+        # Additional context: https://github.com/matrix-org/synapse/pull/11137
+        assert isinstance(new_token, int)
+
         services = [
             service
             for service in self.store.get_app_services()
@@ -231,14 +244,13 @@ class ApplicationServicesHandler:
         self,
         services: List[ApplicationService],
         stream_key: str,
-        new_token: Optional[int],
+        new_token: int,
         users: Collection[Union[str, UserID]],
     ) -> None:
         logger.debug("Checking interested services for %s" % (stream_key))
         with Measure(self.clock, "notify_interested_services_ephemeral"):
             for service in services:
-                # Only handle typing if we have the latest token
-                if stream_key == "typing_key" and new_token is not None:
+                if stream_key == "typing_key":
                     # Note that we don't persist the token (via set_type_stream_id_for_appservice)
                     # for typing_key due to performance reasons and due to their highly
                     # ephemeral nature.
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1882fffd2a..60e5409895 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -383,29 +383,6 @@ class Notifier:
         except Exception:
             logger.exception("Error notifying application services of event")
 
-    def _notify_app_services_ephemeral(
-        self,
-        stream_key: str,
-        new_token: Union[int, RoomStreamToken],
-        users: Optional[Collection[Union[str, UserID]]] = None,
-    ) -> None:
-        """Notify application services of ephemeral event activity.
-
-        Args:
-            stream_key: The stream the event came from.
-            new_token: The value of the new stream token.
-            users: The users that should be informed of the new event, if any.
-        """
-        try:
-            stream_token = None
-            if isinstance(new_token, int):
-                stream_token = new_token
-            self.appservice_handler.notify_interested_services_ephemeral(
-                stream_key, stream_token, users or []
-            )
-        except Exception:
-            logger.exception("Error notifying application services of event")
-
     def _notify_pusher_pool(self, max_room_stream_token: RoomStreamToken):
         try:
             self._pusher_pool.on_new_notifications(max_room_stream_token)
@@ -467,12 +444,15 @@ class Notifier:
 
             self.notify_replication()
 
-            # Notify appservices
-            self._notify_app_services_ephemeral(
-                stream_key,
-                new_token,
-                users,
-            )
+            # Notify appservices.
+            try:
+                self.appservice_handler.notify_interested_services_ephemeral(
+                    stream_key,
+                    new_token,
+                    users,
+                )
+            except Exception:
+                logger.exception("Error notifying application services of event")
 
     def on_new_replication_data(self) -> None:
         """Used to inform replication listeners that something has happened
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index b15cd030e0..9ccc66e589 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -427,7 +427,7 @@ class DeviceWorkerStore(SQLBaseStore):
             user_ids: the users who were signed
 
         Returns:
-            THe new stream ID.
+            The new stream ID.
         """
 
         async with self._device_list_id_gen.get_next() as stream_id:
@@ -1322,7 +1322,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore):
 
     async def add_device_change_to_streams(
         self, user_id: str, device_ids: Collection[str], hosts: List[str]
-    ):
+    ) -> int:
         """Persist that a user's devices have been updated, and which hosts
         (if any) should be poked.
         """
diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py
index 12cf6995eb..cc0eebdb46 100644
--- a/synapse/storage/databases/main/presence.py
+++ b/synapse/storage/databases/main/presence.py
@@ -92,7 +92,7 @@ class PresenceStore(PresenceBackgroundUpdateStore):
             prefilled_cache=presence_cache_prefill,
         )
 
-    async def update_presence(self, presence_states):
+    async def update_presence(self, presence_states) -> Tuple[int, int]:
         assert self._can_persist_presence
 
         stream_ordering_manager = self._presence_id_gen.get_next_mult(
-- 
cgit 1.5.1


From 4535532526581834ab798996ffe73f6d19c25123 Mon Sep 17 00:00:00 2001
From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com>
Date: Tue, 2 Nov 2021 14:18:30 +0100
Subject: Delete messages for hidden devices from `device_inbox` (#11199)

---
 changelog.d/11199.bugfix                           |  1 +
 synapse/storage/databases/main/deviceinbox.py      | 89 ++++++++++++++++++++++
 .../03remove_hidden_devices_from_device_inbox.sql  | 22 ++++++
 tests/storage/databases/main/test_deviceinbox.py   | 74 ++++++++++++++++++
 4 files changed, 186 insertions(+)
 create mode 100644 changelog.d/11199.bugfix
 create mode 100644 synapse/storage/schema/main/delta/65/03remove_hidden_devices_from_device_inbox.sql

diff --git a/changelog.d/11199.bugfix b/changelog.d/11199.bugfix
new file mode 100644
index 0000000000..dc3ea8d515
--- /dev/null
+++ b/changelog.d/11199.bugfix
@@ -0,0 +1 @@
+Delete `to_device` messages for hidden devices that will never be read, reducing database size.
\ No newline at end of file
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 25e9c1efe1..264e625bd7 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -561,6 +561,7 @@ class DeviceInboxWorkerStore(SQLBaseStore):
 class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
     DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"
     REMOVE_DELETED_DEVICES = "remove_deleted_devices_from_device_inbox"
+    REMOVE_HIDDEN_DEVICES = "remove_hidden_devices_from_device_inbox"
 
     def __init__(self, database: DatabasePool, db_conn, hs: "HomeServer"):
         super().__init__(database, db_conn, hs)
@@ -581,6 +582,11 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
             self._remove_deleted_devices_from_device_inbox,
         )
 
+        self.db_pool.updates.register_background_update_handler(
+            self.REMOVE_HIDDEN_DEVICES,
+            self._remove_hidden_devices_from_device_inbox,
+        )
+
     async def _background_drop_index_device_inbox(self, progress, batch_size):
         def reindex_txn(conn):
             txn = conn.cursor()
@@ -676,6 +682,89 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
 
         return number_deleted
 
+    async def _remove_hidden_devices_from_device_inbox(
+        self, progress: JsonDict, batch_size: int
+    ) -> int:
+        """A background update that deletes all device_inboxes for hidden devices.
+
+        This should only need to be run once (when users upgrade to v1.47.0)
+
+        Args:
+            progress: JsonDict used to store progress of this background update
+            batch_size: the maximum number of rows to retrieve in a single select query
+
+        Returns:
+            The number of deleted rows
+        """
+
+        def _remove_hidden_devices_from_device_inbox_txn(
+            txn: LoggingTransaction,
+        ) -> int:
+            """stream_id is not unique
+            we need to use an inclusive `stream_id >= ?` clause,
+            since we might not have deleted all hidden device messages for the stream_id
+            returned from the previous query
+
+            Then delete only rows matching the `(user_id, device_id, stream_id)` tuple,
+            to avoid problems of deleting a large number of rows all at once
+            due to a single device having lots of device messages.
+            """
+
+            last_stream_id = progress.get("stream_id", 0)
+
+            sql = """
+                SELECT device_id, user_id, stream_id
+                FROM device_inbox
+                WHERE
+                    stream_id >= ?
+                    AND (device_id, user_id) IN (
+                        SELECT device_id, user_id FROM devices WHERE hidden = ?
+                    )
+                ORDER BY stream_id
+                LIMIT ?
+            """
+
+            txn.execute(sql, (last_stream_id, True, batch_size))
+            rows = txn.fetchall()
+
+            num_deleted = 0
+            for row in rows:
+                num_deleted += self.db_pool.simple_delete_txn(
+                    txn,
+                    "device_inbox",
+                    {"device_id": row[0], "user_id": row[1], "stream_id": row[2]},
+                )
+
+            if rows:
+                # We don't just save the `stream_id` in progress as
+                # otherwise it can happen in large deployments that
+                # no change of status is visible in the log file, as
+                # it may be that the stream_id does not change in several runs
+                self.db_pool.updates._background_update_progress_txn(
+                    txn,
+                    self.REMOVE_HIDDEN_DEVICES,
+                    {
+                        "device_id": rows[-1][0],
+                        "user_id": rows[-1][1],
+                        "stream_id": rows[-1][2],
+                    },
+                )
+
+            return num_deleted
+
+        number_deleted = await self.db_pool.runInteraction(
+            "_remove_hidden_devices_from_device_inbox",
+            _remove_hidden_devices_from_device_inbox_txn,
+        )
+
+        # The task is finished when no more lines are deleted.
+        if not number_deleted:
+            await self.db_pool.updates._end_background_update(
+                self.REMOVE_HIDDEN_DEVICES
+            )
+
+        return number_deleted
+
 
 class DeviceInboxStore(DeviceInboxWorkerStore, DeviceInboxBackgroundUpdateStore):
     pass
diff --git a/synapse/storage/schema/main/delta/65/03remove_hidden_devices_from_device_inbox.sql b/synapse/storage/schema/main/delta/65/03remove_hidden_devices_from_device_inbox.sql
new file mode 100644
index 0000000000..7b3592dcf0
--- /dev/null
+++ b/synapse/storage/schema/main/delta/65/03remove_hidden_devices_from_device_inbox.sql
@@ -0,0 +1,22 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- Remove messages from the device_inbox table which were orphaned
+-- because a device was hidden using Synapse earlier than 1.47.0.
+-- This runs as background task, but may take a bit to finish.
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (6503, 'remove_hidden_devices_from_device_inbox', '{}');
diff --git a/tests/storage/databases/main/test_deviceinbox.py b/tests/storage/databases/main/test_deviceinbox.py
index 4cfd2677f7..4b67bd15b7 100644
--- a/tests/storage/databases/main/test_deviceinbox.py
+++ b/tests/storage/databases/main/test_deviceinbox.py
@@ -88,3 +88,77 @@ class DeviceInboxBackgroundUpdateStoreTestCase(HomeserverTestCase):
         )
         self.assertEqual(1, len(res))
         self.assertEqual(res[0], "cur_device")
+
+    def test_background_remove_hidden_devices_from_device_inbox(self):
+        """Test that the background task to delete hidden devices
+        from device_inboxes works properly."""
+
+        # create a valid device
+        self.get_success(
+            self.store.store_device(self.user_id, "cur_device", "display_name")
+        )
+
+        # create a hidden device
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "devices",
+                values={
+                    "user_id": self.user_id,
+                    "device_id": "hidden_device",
+                    "display_name": "hidden_display_name",
+                    "hidden": True,
+                },
+            )
+        )
+
+        # Add device_inbox to devices
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "device_inbox",
+                {
+                    "user_id": self.user_id,
+                    "device_id": "cur_device",
+                    "stream_id": 1,
+                    "message_json": "{}",
+                },
+            )
+        )
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "device_inbox",
+                {
+                    "user_id": self.user_id,
+                    "device_id": "hidden_device",
+                    "stream_id": 2,
+                    "message_json": "{}",
+                },
+            )
+        )
+
+        # Insert and run the background update.
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "background_updates",
+                {
+                    "update_name": "remove_hidden_devices_from_device_inbox",
+                    "progress_json": "{}",
+                },
+            )
+        )
+
+        # ... and tell the DataStore that it hasn't finished all updates yet
+        self.store.db_pool.updates._all_done = False
+
+        self.wait_for_background_updates()
+
+        # Make sure the background task deleted hidden devices from device_inbox
+        res = self.get_success(
+            self.store.db_pool.simple_select_onecol(
+                table="device_inbox",
+                keyvalues={},
+                retcol="device_id",
+                desc="get_device_id_from_device_inbox",
+            )
+        )
+        self.assertEqual(1, len(res))
+        self.assertEqual(res[0], "cur_device")
-- 
cgit 1.5.1


From df84ad602b21a4cea3a63c9117b5cd7884f1ab05 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Nov 2021 13:23:01 +0000
Subject: 1.46.0

---
 CHANGES.md               | 9 +++++++++
 changelog.d/11196.bugfix | 1 -
 debian/changelog         | 8 ++++++--
 synapse/__init__.py      | 2 +-
 4 files changed, 16 insertions(+), 4 deletions(-)
 delete mode 100644 changelog.d/11196.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index f61d5c706f..124bdf320a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,12 @@
+Synapse 1.46.0 (2021-11-02)
+===========================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in v1.46.0rc1 where URL previews of some XML documents would fail. ([\#11196](https://github.com/matrix-org/synapse/issues/11196))
+
+
 Synapse 1.46.0rc1 (2021-10-27)
 ==============================
 
diff --git a/changelog.d/11196.bugfix b/changelog.d/11196.bugfix
deleted file mode 100644
index 3861eeb908..0000000000
--- a/changelog.d/11196.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in v1.46.0rc1 where URL previews of some XML documents would fail.
diff --git a/debian/changelog b/debian/changelog
index c2ea5d2cfb..06e7a0862d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,12 @@
-matrix-synapse-py3 (1.46.0~rc1ubuntu1) UNRELEASED; urgency=medium
+matrix-synapse-py3 (1.46.0) stable; urgency=medium
 
+  [ Richard van der Hoff ]
   * Compress debs with xz, to fix incompatibility of impish debs with reprepro.
 
- -- Richard van der Hoff <richard@matrix.org>  Wed, 27 Oct 2021 15:32:51 +0100
+  [ Synapse Packaging team ]
+  * New synapse release 1.46.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 02 Nov 2021 13:22:53 +0000
 
 matrix-synapse-py3 (1.46.0~rc1) stable; urgency=medium
 
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 355b36fc63..5ef34bce40 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.46.0rc1"
+__version__ = "1.46.0"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when
-- 
cgit 1.5.1


From 2d44ee6868805d4ff23489a8dd6b4072ff358663 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Nov 2021 13:25:42 +0000
Subject: Update changelog

---
 CHANGES.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 124bdf320a..e74544f489 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,6 +1,8 @@
 Synapse 1.46.0 (2021-11-02)
 ===========================
 
+The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
+
 Bugfixes
 --------
 
@@ -10,8 +12,6 @@ Bugfixes
 Synapse 1.46.0rc1 (2021-10-27)
 ==============================
 
-The cause of the [performance regression affecting Synapse 1.44](https://github.com/matrix-org/synapse/issues/11049) has been identified and fixed. ([\#11177](https://github.com/matrix-org/synapse/issues/11177))
-
 Features
 --------
 
-- 
cgit 1.5.1


From c01bc5f43d1c7d0a25f397b542ced57894395519 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 2 Nov 2021 09:55:52 -0400
Subject: Add remaining type hints to `synapse.events`. (#11098)

---
 changelog.d/11098.misc                       |   1 +
 mypy.ini                                     |   8 +-
 synapse/events/__init__.py                   | 227 +++++++++++++++++----------
 synapse/events/validator.py                  |   2 +-
 synapse/handlers/federation_event.py         |   2 +-
 synapse/handlers/message.py                  |  14 +-
 synapse/handlers/room.py                     |   2 +-
 synapse/handlers/room_batch.py               |   2 +-
 synapse/handlers/room_member.py              |   4 +-
 synapse/push/bulk_push_rule_evaluator.py     |   4 +-
 synapse/push/push_rule_evaluator.py          |  10 +-
 synapse/rest/client/room_batch.py            |   2 +-
 synapse/state/__init__.py                    |   2 +-
 synapse/storage/databases/main/events.py     |   7 +-
 synapse/storage/databases/main/roommember.py |   8 +-
 15 files changed, 185 insertions(+), 110 deletions(-)
 create mode 100644 changelog.d/11098.misc

diff --git a/changelog.d/11098.misc b/changelog.d/11098.misc
new file mode 100644
index 0000000000..1e337bee54
--- /dev/null
+++ b/changelog.d/11098.misc
@@ -0,0 +1 @@
+Add type hints to `synapse.events`.
diff --git a/mypy.ini b/mypy.ini
index 119a7d8c91..600402a5d3 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -22,13 +22,7 @@ files =
   synapse/config,
   synapse/crypto,
   synapse/event_auth.py,
-  synapse/events/builder.py,
-  synapse/events/presence_router.py,
-  synapse/events/snapshot.py,
-  synapse/events/spamcheck.py,
-  synapse/events/third_party_rules.py,
-  synapse/events/utils.py,
-  synapse/events/validator.py,
+  synapse/events,
   synapse/federation,
   synapse/groups,
   synapse/handlers,
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 157669ea88..38f3cf4d33 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -16,8 +16,23 @@
 
 import abc
 import os
-from typing import Dict, Optional, Tuple, Type
-
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    overload,
+)
+
+from typing_extensions import Literal
 from unpaddedbase64 import encode_base64
 
 from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions
@@ -26,6 +41,9 @@ from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 from synapse.util.stringutils import strtobool
 
+if TYPE_CHECKING:
+    from synapse.events.builder import EventBuilder
+
 # Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents
 # bugs where we accidentally share e.g. signature dicts. However, converting a
 # dict to frozen_dicts is expensive.
@@ -37,7 +55,23 @@ from synapse.util.stringutils import strtobool
 USE_FROZEN_DICTS = strtobool(os.environ.get("SYNAPSE_USE_FROZEN_DICTS", "0"))
 
 
-class DictProperty:
+T = TypeVar("T")
+
+
+# DictProperty (and DefaultDictProperty) require the classes they're used with to
+# have a _dict property to pull properties from.
+#
+# TODO _DictPropertyInstance should not include EventBuilder but due to
+# https://github.com/python/mypy/issues/5570 it thinks the DictProperty and
+# DefaultDictProperty get applied to EventBuilder when it is in a Union with
+# EventBase. This is the least invasive hack to get mypy to comply.
+#
+# Note that DictProperty/DefaultDictProperty cannot actually be used with
+# EventBuilder as it lacks a _dict property.
+_DictPropertyInstance = Union["_EventInternalMetadata", "EventBase", "EventBuilder"]
+
+
+class DictProperty(Generic[T]):
     """An object property which delegates to the `_dict` within its parent object."""
 
     __slots__ = ["key"]
@@ -45,12 +79,33 @@ class DictProperty:
     def __init__(self, key: str):
         self.key = key
 
-    def __get__(self, instance, owner=None):
+    @overload
+    def __get__(
+        self,
+        instance: Literal[None],
+        owner: Optional[Type[_DictPropertyInstance]] = None,
+    ) -> "DictProperty":
+        ...
+
+    @overload
+    def __get__(
+        self,
+        instance: _DictPropertyInstance,
+        owner: Optional[Type[_DictPropertyInstance]] = None,
+    ) -> T:
+        ...
+
+    def __get__(
+        self,
+        instance: Optional[_DictPropertyInstance],
+        owner: Optional[Type[_DictPropertyInstance]] = None,
+    ) -> Union[T, "DictProperty"]:
         # if the property is accessed as a class property rather than an instance
         # property, return the property itself rather than the value
         if instance is None:
             return self
         try:
+            assert isinstance(instance, (EventBase, _EventInternalMetadata))
             return instance._dict[self.key]
         except KeyError as e1:
             # We want this to look like a regular attribute error (mostly so that
@@ -65,10 +120,12 @@ class DictProperty:
                 "'%s' has no '%s' property" % (type(instance), self.key)
             ) from e1.__context__
 
-    def __set__(self, instance, v):
+    def __set__(self, instance: _DictPropertyInstance, v: T) -> None:
+        assert isinstance(instance, (EventBase, _EventInternalMetadata))
         instance._dict[self.key] = v
 
-    def __delete__(self, instance):
+    def __delete__(self, instance: _DictPropertyInstance) -> None:
+        assert isinstance(instance, (EventBase, _EventInternalMetadata))
         try:
             del instance._dict[self.key]
         except KeyError as e1:
@@ -77,7 +134,7 @@ class DictProperty:
             ) from e1.__context__
 
 
-class DefaultDictProperty(DictProperty):
+class DefaultDictProperty(DictProperty, Generic[T]):
     """An extension of DictProperty which provides a default if the property is
     not present in the parent's _dict.
 
@@ -86,13 +143,34 @@ class DefaultDictProperty(DictProperty):
 
     __slots__ = ["default"]
 
-    def __init__(self, key, default):
+    def __init__(self, key: str, default: T):
         super().__init__(key)
         self.default = default
 
-    def __get__(self, instance, owner=None):
+    @overload
+    def __get__(
+        self,
+        instance: Literal[None],
+        owner: Optional[Type[_DictPropertyInstance]] = None,
+    ) -> "DefaultDictProperty":
+        ...
+
+    @overload
+    def __get__(
+        self,
+        instance: _DictPropertyInstance,
+        owner: Optional[Type[_DictPropertyInstance]] = None,
+    ) -> T:
+        ...
+
+    def __get__(
+        self,
+        instance: Optional[_DictPropertyInstance],
+        owner: Optional[Type[_DictPropertyInstance]] = None,
+    ) -> Union[T, "DefaultDictProperty"]:
         if instance is None:
             return self
+        assert isinstance(instance, (EventBase, _EventInternalMetadata))
         return instance._dict.get(self.key, self.default)
 
 
@@ -111,22 +189,22 @@ class _EventInternalMetadata:
         # in the DAG)
         self.outlier = False
 
-    out_of_band_membership: bool = DictProperty("out_of_band_membership")
-    send_on_behalf_of: str = DictProperty("send_on_behalf_of")
-    recheck_redaction: bool = DictProperty("recheck_redaction")
-    soft_failed: bool = DictProperty("soft_failed")
-    proactively_send: bool = DictProperty("proactively_send")
-    redacted: bool = DictProperty("redacted")
-    txn_id: str = DictProperty("txn_id")
-    token_id: int = DictProperty("token_id")
-    historical: bool = DictProperty("historical")
+    out_of_band_membership: DictProperty[bool] = DictProperty("out_of_band_membership")
+    send_on_behalf_of: DictProperty[str] = DictProperty("send_on_behalf_of")
+    recheck_redaction: DictProperty[bool] = DictProperty("recheck_redaction")
+    soft_failed: DictProperty[bool] = DictProperty("soft_failed")
+    proactively_send: DictProperty[bool] = DictProperty("proactively_send")
+    redacted: DictProperty[bool] = DictProperty("redacted")
+    txn_id: DictProperty[str] = DictProperty("txn_id")
+    token_id: DictProperty[int] = DictProperty("token_id")
+    historical: DictProperty[bool] = DictProperty("historical")
 
     # XXX: These are set by StreamWorkerStore._set_before_and_after.
     # I'm pretty sure that these are never persisted to the database, so shouldn't
     # be here
-    before: RoomStreamToken = DictProperty("before")
-    after: RoomStreamToken = DictProperty("after")
-    order: Tuple[int, int] = DictProperty("order")
+    before: DictProperty[RoomStreamToken] = DictProperty("before")
+    after: DictProperty[RoomStreamToken] = DictProperty("after")
+    order: DictProperty[Tuple[int, int]] = DictProperty("order")
 
     def get_dict(self) -> JsonDict:
         return dict(self._dict)
@@ -162,9 +240,6 @@ class _EventInternalMetadata:
 
         If the sender of the redaction event is allowed to redact any event
         due to auth rules, then this will always return false.
-
-        Returns:
-            bool
         """
         return self._dict.get("recheck_redaction", False)
 
@@ -176,32 +251,23 @@ class _EventInternalMetadata:
                sent to clients.
             2. They should not be added to the forward extremities (and
                therefore not to current state).
-
-        Returns:
-            bool
         """
         return self._dict.get("soft_failed", False)
 
-    def should_proactively_send(self):
+    def should_proactively_send(self) -> bool:
         """Whether the event, if ours, should be sent to other clients and
         servers.
 
         This is used for sending dummy events internally. Servers and clients
         can still explicitly fetch the event.
-
-        Returns:
-            bool
         """
         return self._dict.get("proactively_send", True)
 
-    def is_redacted(self):
+    def is_redacted(self) -> bool:
         """Whether the event has been redacted.
 
         This is used for efficiently checking whether an event has been
         marked as redacted without needing to make another database call.
-
-        Returns:
-            bool
         """
         return self._dict.get("redacted", False)
 
@@ -241,29 +307,31 @@ class EventBase(metaclass=abc.ABCMeta):
 
         self.internal_metadata = _EventInternalMetadata(internal_metadata_dict)
 
-    auth_events = DictProperty("auth_events")
-    depth = DictProperty("depth")
-    content = DictProperty("content")
-    hashes = DictProperty("hashes")
-    origin = DictProperty("origin")
-    origin_server_ts = DictProperty("origin_server_ts")
-    prev_events = DictProperty("prev_events")
-    redacts = DefaultDictProperty("redacts", None)
-    room_id = DictProperty("room_id")
-    sender = DictProperty("sender")
-    state_key = DictProperty("state_key")
-    type = DictProperty("type")
-    user_id = DictProperty("sender")
+    depth: DictProperty[int] = DictProperty("depth")
+    content: DictProperty[JsonDict] = DictProperty("content")
+    hashes: DictProperty[Dict[str, str]] = DictProperty("hashes")
+    origin: DictProperty[str] = DictProperty("origin")
+    origin_server_ts: DictProperty[int] = DictProperty("origin_server_ts")
+    redacts: DefaultDictProperty[Optional[str]] = DefaultDictProperty("redacts", None)
+    room_id: DictProperty[str] = DictProperty("room_id")
+    sender: DictProperty[str] = DictProperty("sender")
+    # TODO state_key should be Optional[str], this is generally asserted in Synapse
+    # by calling is_state() first (which ensures this), but it is hard (not possible?)
+    # to properly annotate that calling is_state() asserts that state_key exists
+    # and is non-None.
+    state_key: DictProperty[str] = DictProperty("state_key")
+    type: DictProperty[str] = DictProperty("type")
+    user_id: DictProperty[str] = DictProperty("sender")
 
     @property
     def event_id(self) -> str:
         raise NotImplementedError()
 
     @property
-    def membership(self):
+    def membership(self) -> str:
         return self.content["membership"]
 
-    def is_state(self):
+    def is_state(self) -> bool:
         return hasattr(self, "state_key") and self.state_key is not None
 
     def get_dict(self) -> JsonDict:
@@ -272,13 +340,13 @@ class EventBase(metaclass=abc.ABCMeta):
 
         return d
 
-    def get(self, key, default=None):
+    def get(self, key: str, default: Optional[Any] = None) -> Any:
         return self._dict.get(key, default)
 
-    def get_internal_metadata_dict(self):
+    def get_internal_metadata_dict(self) -> JsonDict:
         return self.internal_metadata.get_dict()
 
-    def get_pdu_json(self, time_now=None) -> JsonDict:
+    def get_pdu_json(self, time_now: Optional[int] = None) -> JsonDict:
         pdu_json = self.get_dict()
 
         if time_now is not None and "age_ts" in pdu_json["unsigned"]:
@@ -305,49 +373,46 @@ class EventBase(metaclass=abc.ABCMeta):
 
         return template_json
 
-    def __set__(self, instance, value):
-        raise AttributeError("Unrecognized attribute %s" % (instance,))
-
-    def __getitem__(self, field):
+    def __getitem__(self, field: str) -> Optional[Any]:
         return self._dict[field]
 
-    def __contains__(self, field):
+    def __contains__(self, field: str) -> bool:
         return field in self._dict
 
-    def items(self):
+    def items(self) -> List[Tuple[str, Optional[Any]]]:
         return list(self._dict.items())
 
-    def keys(self):
+    def keys(self) -> Iterable[str]:
         return self._dict.keys()
 
-    def prev_event_ids(self):
+    def prev_event_ids(self) -> Sequence[str]:
         """Returns the list of prev event IDs. The order matches the order
         specified in the event, though there is no meaning to it.
 
         Returns:
-            list[str]: The list of event IDs of this event's prev_events
+            The list of event IDs of this event's prev_events
         """
-        return [e for e, _ in self.prev_events]
+        return [e for e, _ in self._dict["prev_events"]]
 
-    def auth_event_ids(self):
+    def auth_event_ids(self) -> Sequence[str]:
         """Returns the list of auth event IDs. The order matches the order
         specified in the event, though there is no meaning to it.
 
         Returns:
-            list[str]: The list of event IDs of this event's auth_events
+            The list of event IDs of this event's auth_events
         """
-        return [e for e, _ in self.auth_events]
+        return [e for e, _ in self._dict["auth_events"]]
 
-    def freeze(self):
+    def freeze(self) -> None:
         """'Freeze' the event dict, so it cannot be modified by accident"""
 
         # this will be a no-op if the event dict is already frozen.
         self._dict = freeze(self._dict)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.__repr__()
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else ""
 
         return (
@@ -443,7 +508,7 @@ class FrozenEventV2(EventBase):
         else:
             frozen_dict = event_dict
 
-        self._event_id = None
+        self._event_id: Optional[str] = None
 
         super().__init__(
             frozen_dict,
@@ -455,7 +520,7 @@ class FrozenEventV2(EventBase):
         )
 
     @property
-    def event_id(self):
+    def event_id(self) -> str:
         # We have to import this here as otherwise we get an import loop which
         # is hard to break.
         from synapse.crypto.event_signing import compute_event_reference_hash
@@ -465,23 +530,23 @@ class FrozenEventV2(EventBase):
         self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1])
         return self._event_id
 
-    def prev_event_ids(self):
+    def prev_event_ids(self) -> Sequence[str]:
         """Returns the list of prev event IDs. The order matches the order
         specified in the event, though there is no meaning to it.
 
         Returns:
-            list[str]: The list of event IDs of this event's prev_events
+            The list of event IDs of this event's prev_events
         """
-        return self.prev_events
+        return self._dict["prev_events"]
 
-    def auth_event_ids(self):
+    def auth_event_ids(self) -> Sequence[str]:
         """Returns the list of auth event IDs. The order matches the order
         specified in the event, though there is no meaning to it.
 
         Returns:
-            list[str]: The list of event IDs of this event's auth_events
+            The list of event IDs of this event's auth_events
         """
-        return self.auth_events
+        return self._dict["auth_events"]
 
 
 class FrozenEventV3(FrozenEventV2):
@@ -490,7 +555,7 @@ class FrozenEventV3(FrozenEventV2):
     format_version = EventFormatVersions.V3  # All events of this type are V3
 
     @property
-    def event_id(self):
+    def event_id(self) -> str:
         # We have to import this here as otherwise we get an import loop which
         # is hard to break.
         from synapse.crypto.event_signing import compute_event_reference_hash
@@ -503,12 +568,14 @@ class FrozenEventV3(FrozenEventV2):
         return self._event_id
 
 
-def _event_type_from_format_version(format_version: int) -> Type[EventBase]:
+def _event_type_from_format_version(
+    format_version: int,
+) -> Type[Union[FrozenEvent, FrozenEventV2, FrozenEventV3]]:
     """Returns the python type to use to construct an Event object for the
     given event format version.
 
     Args:
-        format_version (int): The event format version
+        format_version: The event format version
 
     Returns:
         type: A type that can be initialized as per the initializer of
diff --git a/synapse/events/validator.py b/synapse/events/validator.py
index 4d459c17f1..cf86934968 100644
--- a/synapse/events/validator.py
+++ b/synapse/events/validator.py
@@ -55,7 +55,7 @@ class EventValidator:
         ]
 
         for k in required:
-            if not hasattr(event, k):
+            if k not in event:
                 raise SynapseError(400, "Event does not have key %s" % (k,))
 
         # Check that the following keys have string values
diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index e617db4c0d..1a1cd93b1a 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -1643,7 +1643,7 @@ class FederationEventHandler:
             event: the event whose auth_events we want
 
         Returns:
-            all of the events in `event.auth_events`, after deduplication
+            all of the events listed in `event.auth_events_ids`, after deduplication
 
         Raises:
             AuthError if we were unable to fetch the auth_events for any reason.
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 4a0fccfcc6..b7bc187169 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -1318,6 +1318,8 @@ class EventCreationHandler:
             # user is actually admin or not).
             is_admin_redaction = False
             if event.type == EventTypes.Redaction:
+                assert event.redacts is not None
+
                 original_event = await self.store.get_event(
                     event.redacts,
                     redact_behaviour=EventRedactBehaviour.AS_IS,
@@ -1413,6 +1415,8 @@ class EventCreationHandler:
                 )
 
         if event.type == EventTypes.Redaction:
+            assert event.redacts is not None
+
             original_event = await self.store.get_event(
                 event.redacts,
                 redact_behaviour=EventRedactBehaviour.AS_IS,
@@ -1500,11 +1504,13 @@ class EventCreationHandler:
                 next_batch_id = event.content.get(
                     EventContentFields.MSC2716_NEXT_BATCH_ID
                 )
-                conflicting_insertion_event_id = (
-                    await self.store.get_insertion_event_by_batch_id(
-                        event.room_id, next_batch_id
+                conflicting_insertion_event_id = None
+                if next_batch_id:
+                    conflicting_insertion_event_id = (
+                        await self.store.get_insertion_event_by_batch_id(
+                            event.room_id, next_batch_id
+                        )
                     )
-                )
                 if conflicting_insertion_event_id is not None:
                     # The current insertion event that we're processing is invalid
                     # because an insertion event already exists in the room with the
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 99e9b37344..969eb3b9b0 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -525,7 +525,7 @@ class RoomCreationHandler:
             ):
                 await self.room_member_handler.update_membership(
                     requester,
-                    UserID.from_string(old_event["state_key"]),
+                    UserID.from_string(old_event.state_key),
                     new_room_id,
                     "ban",
                     ratelimit=False,
diff --git a/synapse/handlers/room_batch.py b/synapse/handlers/room_batch.py
index 2f5a3e4d19..0723286383 100644
--- a/synapse/handlers/room_batch.py
+++ b/synapse/handlers/room_batch.py
@@ -355,7 +355,7 @@ class RoomBatchHandler:
         for (event, context) in reversed(events_to_persist):
             await self.event_creation_handler.handle_new_client_event(
                 await self.create_requester_for_user_id_from_app_service(
-                    event["sender"], app_service_requester.app_service
+                    event.sender, app_service_requester.app_service
                 ),
                 event=event,
                 context=context,
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 74e6c7eca6..08244b690d 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -1669,7 +1669,9 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         #
         # the prev_events consist solely of the previous membership event.
         prev_event_ids = [previous_membership_event.event_id]
-        auth_event_ids = previous_membership_event.auth_event_ids() + prev_event_ids
+        auth_event_ids = (
+            list(previous_membership_event.auth_event_ids()) + prev_event_ids
+        )
 
         event, context = await self.event_creation_handler.create_event(
             requester,
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 0622a37ae8..009d8e77b0 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -232,6 +232,8 @@ class BulkPushRuleEvaluator:
                 # that user, as they might not be already joined.
                 if event.type == EventTypes.Member and event.state_key == uid:
                     display_name = event.content.get("displayname", None)
+                    if not isinstance(display_name, str):
+                        display_name = None
 
             if count_as_unread:
                 # Add an element for the current user if the event needs to be marked as
@@ -268,7 +270,7 @@ def _condition_checker(
     evaluator: PushRuleEvaluatorForEvent,
     conditions: List[dict],
     uid: str,
-    display_name: str,
+    display_name: Optional[str],
     cache: Dict[str, bool],
 ) -> bool:
     for cond in conditions:
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index 7a8dc63976..7f68092ec5 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -18,7 +18,7 @@ import re
 from typing import Any, Dict, List, Optional, Pattern, Tuple, Union
 
 from synapse.events import EventBase
-from synapse.types import UserID
+from synapse.types import JsonDict, UserID
 from synapse.util import glob_to_regex, re_word_boundary
 from synapse.util.caches.lrucache import LruCache
 
@@ -129,7 +129,7 @@ class PushRuleEvaluatorForEvent:
         self._value_cache = _flatten_dict(event)
 
     def matches(
-        self, condition: Dict[str, Any], user_id: str, display_name: str
+        self, condition: Dict[str, Any], user_id: str, display_name: Optional[str]
     ) -> bool:
         if condition["kind"] == "event_match":
             return self._event_match(condition, user_id)
@@ -172,7 +172,7 @@ class PushRuleEvaluatorForEvent:
 
             return _glob_matches(pattern, haystack)
 
-    def _contains_display_name(self, display_name: str) -> bool:
+    def _contains_display_name(self, display_name: Optional[str]) -> bool:
         if not display_name:
             return False
 
@@ -222,7 +222,7 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
 
 
 def _flatten_dict(
-    d: Union[EventBase, dict],
+    d: Union[EventBase, JsonDict],
     prefix: Optional[List[str]] = None,
     result: Optional[Dict[str, str]] = None,
 ) -> Dict[str, str]:
@@ -233,7 +233,7 @@ def _flatten_dict(
     for key, value in d.items():
         if isinstance(value, str):
             result[".".join(prefix + [key])] = value.lower()
-        elif hasattr(value, "items"):
+        elif isinstance(value, dict):
             _flatten_dict(value, prefix=(prefix + [key]), result=result)
 
     return result
diff --git a/synapse/rest/client/room_batch.py b/synapse/rest/client/room_batch.py
index 99f8156ad0..ab9a743bba 100644
--- a/synapse/rest/client/room_batch.py
+++ b/synapse/rest/client/room_batch.py
@@ -191,7 +191,7 @@ class RoomBatchSendEventRestServlet(RestServlet):
                 depth=inherited_depth,
             )
 
-            batch_id_to_connect_to = base_insertion_event["content"][
+            batch_id_to_connect_to = base_insertion_event.content[
                 EventContentFields.MSC2716_NEXT_BATCH_ID
             ]
 
diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py
index 98a0239759..1605411b00 100644
--- a/synapse/state/__init__.py
+++ b/synapse/state/__init__.py
@@ -247,7 +247,7 @@ class StateHandler:
         return await self.get_hosts_in_room_at_events(room_id, event_ids)
 
     async def get_hosts_in_room_at_events(
-        self, room_id: str, event_ids: List[str]
+        self, room_id: str, event_ids: Iterable[str]
     ) -> Set[str]:
         """Get the hosts that were in a room at the given event ids
 
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 8d9086ecf0..596275c23c 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -24,6 +24,7 @@ from typing import (
     Iterable,
     List,
     Optional,
+    Sequence,
     Set,
     Tuple,
 )
@@ -494,7 +495,7 @@ class PersistEventsStore:
         event_chain_id_gen: SequenceGenerator,
         event_to_room_id: Dict[str, str],
         event_to_types: Dict[str, Tuple[str, str]],
-        event_to_auth_chain: Dict[str, List[str]],
+        event_to_auth_chain: Dict[str, Sequence[str]],
     ) -> None:
         """Calculate the chain cover index for the given events.
 
@@ -786,7 +787,7 @@ class PersistEventsStore:
         event_chain_id_gen: SequenceGenerator,
         event_to_room_id: Dict[str, str],
         event_to_types: Dict[str, Tuple[str, str]],
-        event_to_auth_chain: Dict[str, List[str]],
+        event_to_auth_chain: Dict[str, Sequence[str]],
         events_to_calc_chain_id_for: Set[str],
         chain_map: Dict[str, Tuple[int, int]],
     ) -> Dict[str, Tuple[int, int]]:
@@ -1794,7 +1795,7 @@ class PersistEventsStore:
         )
 
         # Insert an edge for every prev_event connection
-        for prev_event_id in event.prev_events:
+        for prev_event_id in event.prev_event_ids():
             self.db_pool.simple_insert_txn(
                 txn,
                 table="insertion_event_edges",
diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py
index 4b288bb2e7..033a9831d6 100644
--- a/synapse/storage/databases/main/roommember.py
+++ b/synapse/storage/databases/main/roommember.py
@@ -570,7 +570,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
 
     async def get_joined_users_from_context(
         self, event: EventBase, context: EventContext
-    ):
+    ) -> Dict[str, ProfileInfo]:
         state_group = context.state_group
         if not state_group:
             # If state_group is None it means it has yet to be assigned a
@@ -584,7 +584,9 @@ class RoomMemberWorkerStore(EventsWorkerStore):
             event.room_id, state_group, current_state_ids, event=event, context=context
         )
 
-    async def get_joined_users_from_state(self, room_id, state_entry):
+    async def get_joined_users_from_state(
+        self, room_id, state_entry
+    ) -> Dict[str, ProfileInfo]:
         state_group = state_entry.state_group
         if not state_group:
             # If state_group is None it means it has yet to be assigned a
@@ -607,7 +609,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
         cache_context,
         event=None,
         context=None,
-    ):
+    ) -> Dict[str, ProfileInfo]:
         # We don't use `state_group`, it's there so that we can cache based
         # on it. However, it's important that it's never None, since two current_states
         # with a state_group of None are likely to be different.
-- 
cgit 1.5.1


From 6250b95efe88385bb3ec2842d5eb76f42ef762ef Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 Nov 2021 15:46:48 +0000
Subject: Add index to `local_group_updates.stream_id` (#11231)

This should speed up startup times and generally increase performance of
groups.
---
 changelog.d/11231.misc                                 |  1 +
 scripts/synapse_port_db                                |  2 ++
 synapse/storage/databases/main/group_server.py         | 17 ++++++++++++++++-
 .../schema/main/delta/65/04_local_group_updates.sql    | 18 ++++++++++++++++++
 4 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11231.misc
 create mode 100644 synapse/storage/schema/main/delta/65/04_local_group_updates.sql

diff --git a/changelog.d/11231.misc b/changelog.d/11231.misc
new file mode 100644
index 0000000000..c7fca7071e
--- /dev/null
+++ b/changelog.d/11231.misc
@@ -0,0 +1 @@
+Minor speed up to start up times and getting updates for groups by adding missing index to `local_group_updates.stream_id`.
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index 349866eb9a..640ff15277 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -43,6 +43,7 @@ from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackground
 from synapse.storage.databases.main.events_bg_updates import (
     EventsBackgroundUpdatesStore,
 )
+from synapse.storage.databases.main.group_server import GroupServerWorkerStore
 from synapse.storage.databases.main.media_repository import (
     MediaRepositoryBackgroundUpdateStore,
 )
@@ -181,6 +182,7 @@ class Store(
     StatsStore,
     PusherWorkerStore,
     PresenceBackgroundUpdateStore,
+    GroupServerWorkerStore,
 ):
     def execute(self, f, *args, **kwargs):
         return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py
index e70d3649ff..bb621df0dd 100644
--- a/synapse/storage/databases/main/group_server.py
+++ b/synapse/storage/databases/main/group_server.py
@@ -13,15 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from typing_extensions import TypedDict
 
 from synapse.api.errors import SynapseError
 from synapse.storage._base import SQLBaseStore, db_to_json
+from synapse.storage.database import DatabasePool
+from synapse.storage.types import Connection
 from synapse.types import JsonDict
 from synapse.util import json_encoder
 
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
 # The category ID for the "default" category. We don't store as null in the
 # database to avoid the fun of null != null
 _DEFAULT_CATEGORY_ID = ""
@@ -35,6 +40,16 @@ class _RoomInGroup(TypedDict):
 
 
 class GroupServerWorkerStore(SQLBaseStore):
+    def __init__(self, database: DatabasePool, db_conn: Connection, hs: "HomeServer"):
+        database.updates.register_background_index_update(
+            update_name="local_group_updates_index",
+            index_name="local_group_updates_stream_id_index",
+            table="local_group_updates",
+            columns=("stream_id",),
+            unique=True,
+        )
+        super().__init__(database, db_conn, hs)
+
     async def get_group(self, group_id: str) -> Optional[Dict[str, Any]]:
         return await self.db_pool.simple_select_one(
             table="groups",
diff --git a/synapse/storage/schema/main/delta/65/04_local_group_updates.sql b/synapse/storage/schema/main/delta/65/04_local_group_updates.sql
new file mode 100644
index 0000000000..a178abfe12
--- /dev/null
+++ b/synapse/storage/schema/main/delta/65/04_local_group_updates.sql
@@ -0,0 +1,18 @@
+/* Copyright 2021 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Check index on `local_group_updates.stream_id`.
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (6504, 'local_group_updates_index', '{}');
-- 
cgit 1.5.1