From 2d42e586a8c54be1a83643148358b1651c1ca666 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Tue, 7 Dec 2021 10:49:39 +0000
Subject: Fix the test breakage introduced by #11435 as a result of concurrent
 PRs (#11522)

---
 changelog.d/11522.feature      | 1 +
 tests/rest/client/test_auth.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11522.feature

diff --git a/changelog.d/11522.feature b/changelog.d/11522.feature
new file mode 100644
index 0000000000..9e127fae3c
--- /dev/null
+++ b/changelog.d/11522.feature
@@ -0,0 +1 @@
+Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification.
\ No newline at end of file
diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py
index aa8ad6d2e1..72bbc87b4a 100644
--- a/tests/rest/client/test_auth.py
+++ b/tests/rest/client/test_auth.py
@@ -703,7 +703,7 @@ class RefreshAuthTests(unittest.HomeserverTestCase):
         login_response1 = self.make_request(
             "POST",
             "/_matrix/client/r0/login",
-            {"org.matrix.msc2918.refresh_token": True, **body},
+            {"refresh_token": True, **body},
         )
         self.assertEqual(login_response1.code, 200, login_response1.result)
         self.assertApproximates(
-- 
cgit 1.5.1


From 9c55dedc8c4484e6269451a8c3c10b3e314aeb4a Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 7 Dec 2021 11:24:31 +0000
Subject: Correctly ignore invites from ignored users (#11511)

---
 changelog.d/11511.bugfix |  1 +
 synapse/handlers/sync.py | 11 ++++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/11511.bugfix

diff --git a/changelog.d/11511.bugfix b/changelog.d/11511.bugfix
new file mode 100644
index 0000000000..9c287357b8
--- /dev/null
+++ b/changelog.d/11511.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where invites from ignored users were included in incremental syncs.
\ No newline at end of file
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 53d4627147..97d5a26e20 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1771,6 +1771,7 @@ class SyncHandler:
 
             if not non_joins:
                 continue
+            last_non_join = non_joins[-1]
 
             # Check if we have left the room. This can either be because we were
             # joined before *or* that we since joined and then left.
@@ -1792,18 +1793,18 @@ class SyncHandler:
                         newly_left_rooms.append(room_id)
 
             # Only bother if we're still currently invited
-            should_invite = non_joins[-1].membership == Membership.INVITE
+            should_invite = last_non_join.membership == Membership.INVITE
             if should_invite:
-                if event.sender not in ignored_users:
-                    invite_room_sync = InvitedSyncResult(room_id, invite=non_joins[-1])
+                if last_non_join.sender not in ignored_users:
+                    invite_room_sync = InvitedSyncResult(room_id, invite=last_non_join)
                     if invite_room_sync:
                         invited.append(invite_room_sync)
 
             # Only bother if our latest membership in the room is knock (and we haven't
             # been accepted/rejected in the meantime).
-            should_knock = non_joins[-1].membership == Membership.KNOCK
+            should_knock = last_non_join.membership == Membership.KNOCK
             if should_knock:
-                knock_room_sync = KnockedSyncResult(room_id, knock=non_joins[-1])
+                knock_room_sync = KnockedSyncResult(room_id, knock=last_non_join)
                 if knock_room_sync:
                     knocked.append(knock_room_sync)
 
-- 
cgit 1.5.1


From b1ecd19c5d19815b69e425d80f442bf2877cab76 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 7 Dec 2021 11:37:54 +0000
Subject: Fix 'delete room' admin api to work on incomplete rooms (#11523)

If, for some reason, we don't have the create event, we should still be able to
purge a room.
---
 changelog.d/11523.feature      |  1 +
 synapse/handlers/pagination.py |  3 ---
 synapse/handlers/room.py       | 21 +++++++--------------
 synapse/rest/admin/rooms.py    |  3 ---
 tests/rest/admin/test_room.py  | 42 +++++++++++++++++++++++++-----------------
 5 files changed, 33 insertions(+), 37 deletions(-)
 create mode 100644 changelog.d/11523.feature

diff --git a/changelog.d/11523.feature b/changelog.d/11523.feature
new file mode 100644
index 0000000000..ecac7f9db9
--- /dev/null
+++ b/changelog.d/11523.feature
@@ -0,0 +1 @@
+Extend the "delete room" admin api to work correctly on rooms which have previously been partially deleted.
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index cd64142735..4f42438053 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -406,9 +406,6 @@ class PaginationHandler:
             force: set true to skip checking for joined users.
         """
         with await self.pagination_lock.write(room_id):
-            # check we know about the room
-            await self.store.get_room_version_id(room_id)
-
             # first check that we have no users in this room
             if not force:
                 joined = await self.store.is_host_joined(room_id, self._server_name)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 2bcdf32dcc..ead2198e14 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -1535,20 +1535,13 @@ class RoomShutdownHandler:
             await self.store.block_room(room_id, requester_user_id)
 
         if not await self.store.get_room(room_id):
-            if block:
-                # We allow you to block an unknown room.
-                return {
-                    "kicked_users": [],
-                    "failed_to_kick_users": [],
-                    "local_aliases": [],
-                    "new_room_id": None,
-                }
-            else:
-                # But if you don't want to preventatively block another room,
-                # this function can't do anything useful.
-                raise NotFoundError(
-                    "Cannot shut down room: unknown room id %s" % (room_id,)
-                )
+            # if we don't know about the room, there is nothing left to do.
+            return {
+                "kicked_users": [],
+                "failed_to_kick_users": [],
+                "local_aliases": [],
+                "new_room_id": None,
+            }
 
         if new_room_user_id is not None:
             if not self.hs.is_mine_id(new_room_user_id):
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index 669ab44a45..829e86675a 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -106,9 +106,6 @@ class RoomRestV2Servlet(RestServlet):
                 HTTPStatus.BAD_REQUEST, "%s is not a legal room ID" % (room_id,)
             )
 
-        if not await self._store.get_room(room_id):
-            raise NotFoundError("Unknown room id %s" % (room_id,))
-
         delete_id = self._pagination_handler.start_shutdown_and_purge_room(
             room_id=room_id,
             new_room_user_id=content.get("new_room_user_id"),
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index d3858e460d..22f9aa6234 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -83,7 +83,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
 
     def test_room_does_not_exist(self):
         """
-        Check that unknown rooms/server return error HTTPStatus.NOT_FOUND.
+        Check that unknown rooms/server return 200
         """
         url = "/_synapse/admin/v1/rooms/%s" % "!unknown:test"
 
@@ -94,8 +94,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase):
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, msg=channel.json_body)
-        self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+        self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
 
     def test_room_is_not_valid(self):
         """
@@ -508,27 +507,36 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase):
         self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body)
         self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
 
-    @parameterized.expand(
-        [
-            ("DELETE", "/_synapse/admin/v2/rooms/%s"),
-            ("GET", "/_synapse/admin/v2/rooms/%s/delete_status"),
-            ("GET", "/_synapse/admin/v2/rooms/delete_status/%s"),
-        ]
-    )
-    def test_room_does_not_exist(self, method: str, url: str):
-        """
-        Check that unknown rooms/server return error HTTPStatus.NOT_FOUND.
+    def test_room_does_not_exist(self):
         """
+        Check that unknown rooms/server return 200
 
+        This is important, as it allows incomplete vestiges of rooms to be cleared up
+        even if the create event/etc is missing.
+        """
+        room_id = "!unknown:test"
         channel = self.make_request(
-            method,
-            url % "!unknown:test",
+            "DELETE",
+            f"/_synapse/admin/v2/rooms/{room_id}",
             content={},
             access_token=self.admin_user_tok,
         )
 
-        self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, msg=channel.json_body)
-        self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
+        self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
+        self.assertIn("delete_id", channel.json_body)
+        delete_id = channel.json_body["delete_id"]
+
+        # get status
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v2/rooms/{room_id}/delete_status",
+            access_token=self.admin_user_tok,
+        )
+
+        self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
+        self.assertEqual(1, len(channel.json_body["results"]))
+        self.assertEqual("complete", channel.json_body["results"][0]["status"])
+        self.assertEqual(delete_id, channel.json_body["results"][0]["delete_id"])
 
     @parameterized.expand(
         [
-- 
cgit 1.5.1


From eccc49d7554d1fab001e1fefb0fda8ffb254b630 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 7 Dec 2021 11:41:31 +0000
Subject: Fix `ModuleApi.looping_background_call` for non-async functions
 (#11524)

After #10847, `looping_background_call` would print an error in the logs
every time a non-async function was called. Since the error would be
caught and ignored immediately, there were no other side effects.
---
 changelog.d/11524.bugfix       | 1 +
 synapse/module_api/__init__.py | 5 ++---
 2 files changed, 3 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/11524.bugfix

diff --git a/changelog.d/11524.bugfix b/changelog.d/11524.bugfix
new file mode 100644
index 0000000000..6e1e4bd44c
--- /dev/null
+++ b/changelog.d/11524.bugfix
@@ -0,0 +1 @@
+Fix a regression in Synapse 1.48.0 where the module API's `looping_background_call` method would spam errors to the logs when given a non-async function.
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
index 6bfb4b8d1b..662e60bc33 100644
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -108,6 +108,7 @@ from synapse.types import (
     create_requester,
 )
 from synapse.util import Clock
+from synapse.util.async_helpers import maybe_awaitable
 from synapse.util.caches.descriptors import cached
 
 if TYPE_CHECKING:
@@ -1014,9 +1015,7 @@ class ModuleApi:
                 run_as_background_process,
                 msec,
                 desc,
-                f,
-                *args,
-                **kwargs,
+                lambda: maybe_awaitable(f(*args, **kwargs)),
             )
         else:
             logger.warning(
-- 
cgit 1.5.1


From 2a3ec6facf79f6aae011d9fb6f9ed5e43c7b6bec Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 7 Dec 2021 12:34:38 +0000
Subject: Correctly register shutdown handler for presence workers (#11518)

Fixes #11517
---
 changelog.d/11518.bugfix     | 1 +
 synapse/handlers/presence.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/11518.bugfix

diff --git a/changelog.d/11518.bugfix b/changelog.d/11518.bugfix
new file mode 100644
index 0000000000..3c27382c7a
--- /dev/null
+++ b/changelog.d/11518.bugfix
@@ -0,0 +1 @@
+Fix a regression in Synapse 1.48.0 where presence workers would not clear their presence updates over replication on shutdown.
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 3df872c578..454d06c973 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -421,7 +421,7 @@ class WorkerPresenceHandler(BasePresenceHandler):
             self._on_shutdown,
         )
 
-    def _on_shutdown(self) -> None:
+    async def _on_shutdown(self) -> None:
         if self._presence_enabled:
             self.hs.get_tcp_replication().send_command(
                 ClearUserSyncsCommand(self.instance_id)
-- 
cgit 1.5.1


From 14d593f72d10b4d8cb67e3288bb3131ee30ccf59 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 7 Dec 2021 12:42:05 +0000
Subject: Refactors in `_generate_sync_entry_for_rooms` (#11515)

* Move sync_token up to the top
* Pull out _get_ignored_users
* Try to signpost the body of `_generate_sync_entry_for_rooms`
* Pull out _calculate_user_changes

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
---
 changelog.d/11494.misc   |   2 +-
 changelog.d/11515.misc   |   1 +
 synapse/handlers/sync.py | 122 ++++++++++++++++++++++++++++++-----------------
 3 files changed, 79 insertions(+), 46 deletions(-)
 create mode 100644 changelog.d/11515.misc

diff --git a/changelog.d/11494.misc b/changelog.d/11494.misc
index 7afd7d3a07..52cf26a4b6 100644
--- a/changelog.d/11494.misc
+++ b/changelog.d/11494.misc
@@ -1 +1 @@
-Add comments to various parts of the `/sync` handler.
\ No newline at end of file
+Refactor various parts of the `/sync` handler.
\ No newline at end of file
diff --git a/changelog.d/11515.misc b/changelog.d/11515.misc
new file mode 100644
index 0000000000..7f9d8cd57f
--- /dev/null
+++ b/changelog.d/11515.misc
@@ -0,0 +1 @@
+Refactor various parts of the `/sync` handler.
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 97d5a26e20..f3039c3c3f 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1506,16 +1506,22 @@ class SyncHandler:
             account_data_by_room: Dictionary of per room account data
 
         Returns:
-            Returns a 4-tuple whose entries are:
+            Returns a 4-tuple describing rooms the user has joined or left, and users who've
+            joined or left rooms any rooms the user is in. This gets used later in
+            `_generate_sync_entry_for_device_list`.
+
+            Its entries are:
             - newly_joined_rooms
             - newly_joined_or_invited_or_knocked_users
             - newly_left_rooms
             - newly_left_users
         """
-        # Start by fetching all ephemeral events in rooms we've joined (if required).
+        since_token = sync_result_builder.since_token
+
+        # 1. Start by fetching all ephemeral events in rooms we've joined (if required).
         user_id = sync_result_builder.sync_config.user.to_string()
         block_all_room_ephemeral = (
-            sync_result_builder.since_token is None
+            since_token is None
             and sync_result_builder.sync_config.filter_collection.blocks_all_room_ephemeral()
         )
 
@@ -1529,9 +1535,8 @@ class SyncHandler:
             )
             sync_result_builder.now_token = now_token
 
-        # We check up front if anything has changed, if it hasn't then there is
+        # 2. We check up front if anything has changed, if it hasn't then there is
         # no point in going further.
-        since_token = sync_result_builder.since_token
         if not sync_result_builder.full_state:
             if since_token and not ephemeral_by_room and not account_data_by_room:
                 have_changed = await self._have_rooms_changed(sync_result_builder)
@@ -1544,20 +1549,8 @@ class SyncHandler:
                         logger.debug("no-oping sync")
                         return set(), set(), set(), set()
 
-        ignored_account_data = (
-            await self.store.get_global_account_data_by_type_for_user(
-                AccountDataTypes.IGNORED_USER_LIST, user_id=user_id
-            )
-        )
-
-        # If there is ignored users account data and it matches the proper type,
-        # then use it.
-        ignored_users: FrozenSet[str] = frozenset()
-        if ignored_account_data:
-            ignored_users_data = ignored_account_data.get("ignored_users", {})
-            if isinstance(ignored_users_data, dict):
-                ignored_users = frozenset(ignored_users_data.keys())
-
+        # 3. Work out which rooms need reporting in the sync response.
+        ignored_users = await self._get_ignored_users(user_id)
         if since_token:
             room_changes = await self._get_rooms_changed(
                 sync_result_builder, ignored_users
@@ -1567,7 +1560,6 @@ class SyncHandler:
             )
         else:
             room_changes = await self._get_all_rooms(sync_result_builder, ignored_users)
-
             tags_by_room = await self.store.get_tags_for_user(user_id)
 
         log_kv({"rooms_changed": len(room_changes.room_entries)})
@@ -1578,6 +1570,8 @@ class SyncHandler:
         newly_joined_rooms = room_changes.newly_joined_rooms
         newly_left_rooms = room_changes.newly_left_rooms
 
+        # 4. We need to apply further processing to `room_entries` (rooms considered
+        # joined or archived).
         async def handle_room_entries(room_entry: "RoomSyncResultBuilder") -> None:
             logger.debug("Generating room entry for %s", room_entry.room_id)
             await self._generate_room_entry(
@@ -1596,31 +1590,13 @@ class SyncHandler:
         sync_result_builder.invited.extend(invited)
         sync_result_builder.knocked.extend(knocked)
 
-        # Now we want to get any newly joined, invited or knocking users
-        newly_joined_or_invited_or_knocked_users = set()
-        newly_left_users = set()
-        if since_token:
-            for joined_sync in sync_result_builder.joined:
-                it = itertools.chain(
-                    joined_sync.timeline.events, joined_sync.state.values()
-                )
-                for event in it:
-                    if event.type == EventTypes.Member:
-                        if (
-                            event.membership == Membership.JOIN
-                            or event.membership == Membership.INVITE
-                            or event.membership == Membership.KNOCK
-                        ):
-                            newly_joined_or_invited_or_knocked_users.add(
-                                event.state_key
-                            )
-                        else:
-                            prev_content = event.unsigned.get("prev_content", {})
-                            prev_membership = prev_content.get("membership", None)
-                            if prev_membership == Membership.JOIN:
-                                newly_left_users.add(event.state_key)
-
-        newly_left_users -= newly_joined_or_invited_or_knocked_users
+        # 5. Work out which users have joined or left rooms we're in. We use this
+        # to build the device_list part of the sync response in
+        # `_generate_sync_entry_for_device_list`.
+        (
+            newly_joined_or_invited_or_knocked_users,
+            newly_left_users,
+        ) = sync_result_builder.calculate_user_changes()
 
         return (
             set(newly_joined_rooms),
@@ -1629,6 +1605,29 @@ class SyncHandler:
             newly_left_users,
         )
 
+    async def _get_ignored_users(self, user_id: str) -> FrozenSet[str]:
+        """Retrieve the users ignored by the given user from their global account_data.
+
+        Returns an empty set if
+        - there is no global account_data entry for ignored_users
+        - there is such an entry, but it's not a JSON object.
+        """
+        # TODO: Can we `SELECT ignored_user_id FROM ignored_users WHERE ignorer_user_id=?;` instead?
+        ignored_account_data = (
+            await self.store.get_global_account_data_by_type_for_user(
+                AccountDataTypes.IGNORED_USER_LIST, user_id=user_id
+            )
+        )
+
+        # If there is ignored users account data and it matches the proper type,
+        # then use it.
+        ignored_users: FrozenSet[str] = frozenset()
+        if ignored_account_data:
+            ignored_users_data = ignored_account_data.get("ignored_users", {})
+            if isinstance(ignored_users_data, dict):
+                ignored_users = frozenset(ignored_users_data.keys())
+        return ignored_users
+
     async def _have_rooms_changed(
         self, sync_result_builder: "SyncResultBuilder"
     ) -> bool:
@@ -2341,6 +2340,39 @@ class SyncResultBuilder:
     groups: Optional[GroupsSyncResult] = None
     to_device: List[JsonDict] = attr.Factory(list)
 
+    def calculate_user_changes(self) -> Tuple[Set[str], Set[str]]:
+        """Work out which other users have joined or left rooms we are joined to.
+
+        This data only is only useful for an incremental sync.
+
+        The SyncResultBuilder is not modified by this function.
+        """
+        newly_joined_or_invited_or_knocked_users = set()
+        newly_left_users = set()
+        if self.since_token:
+            for joined_sync in self.joined:
+                it = itertools.chain(
+                    joined_sync.timeline.events, joined_sync.state.values()
+                )
+                for event in it:
+                    if event.type == EventTypes.Member:
+                        if (
+                            event.membership == Membership.JOIN
+                            or event.membership == Membership.INVITE
+                            or event.membership == Membership.KNOCK
+                        ):
+                            newly_joined_or_invited_or_knocked_users.add(
+                                event.state_key
+                            )
+                        else:
+                            prev_content = event.unsigned.get("prev_content", {})
+                            prev_membership = prev_content.get("membership", None)
+                            if prev_membership == Membership.JOIN:
+                                newly_left_users.add(event.state_key)
+
+        newly_left_users -= newly_joined_or_invited_or_knocked_users
+        return newly_joined_or_invited_or_knocked_users, newly_left_users
+
 
 @attr.s(slots=True, auto_attribs=True)
 class RoomSyncResultBuilder:
-- 
cgit 1.5.1


From 088d748f2cb51f03f3bcacc0fb3af1e0f9607737 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 7 Dec 2021 13:51:11 +0000
Subject: Revert "Move `glob_to_regex` and `re_word_boundary` to
 `matrix-python-common` (#11505) (#11527)

This reverts commit a77c36989785c0d5565ab9a1169f4f88e512ce8a.
---
 changelog.d/11527.misc                  |  1 +
 synapse/config/room_directory.py        |  3 +-
 synapse/config/tls.py                   |  3 +-
 synapse/federation/federation_server.py |  3 +-
 synapse/push/push_rule_evaluator.py     |  7 ++--
 synapse/python_dependencies.py          |  1 -
 synapse/util/__init__.py                | 59 ++++++++++++++++++++++++++++++++-
 tests/util/test_glob_to_regex.py        | 59 +++++++++++++++++++++++++++++++++
 8 files changed, 124 insertions(+), 12 deletions(-)
 create mode 100644 changelog.d/11527.misc
 create mode 100644 tests/util/test_glob_to_regex.py

diff --git a/changelog.d/11527.misc b/changelog.d/11527.misc
new file mode 100644
index 0000000000..081eae317c
--- /dev/null
+++ b/changelog.d/11527.misc
@@ -0,0 +1 @@
+Temporarily revert usage of `matrix-python-common`.
diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py
index 3c5e0f7ce7..57316c59b6 100644
--- a/synapse/config/room_directory.py
+++ b/synapse/config/room_directory.py
@@ -15,9 +15,8 @@
 
 from typing import List
 
-from matrix_common.regex import glob_to_regex
-
 from synapse.types import JsonDict
+from synapse.util import glob_to_regex
 
 from ._base import Config, ConfigError
 
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 3e235b57a7..4ca111618f 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -16,12 +16,11 @@ import logging
 import os
 from typing import List, Optional, Pattern
 
-from matrix_common.regex import glob_to_regex
-
 from OpenSSL import SSL, crypto
 from twisted.internet._sslverify import Certificate, trustRootFromCertificates
 
 from synapse.config._base import Config, ConfigError
+from synapse.util import glob_to_regex
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 4697a62c18..8e37e76206 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -28,7 +28,6 @@ from typing import (
     Union,
 )
 
-from matrix_common.regex import glob_to_regex
 from prometheus_client import Counter, Gauge, Histogram
 
 from twisted.internet import defer
@@ -67,7 +66,7 @@ from synapse.replication.http.federation import (
 )
 from synapse.storage.databases.main.lock import Lock
 from synapse.types import JsonDict, get_domain_from_id
-from synapse.util import json_decoder, unwrapFirstError
+from synapse.util import glob_to_regex, json_decoder, unwrapFirstError
 from synapse.util.async_helpers import Linearizer, concurrently_execute
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.stringutils import parse_server_name
diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py
index 659a53805d..7f68092ec5 100644
--- a/synapse/push/push_rule_evaluator.py
+++ b/synapse/push/push_rule_evaluator.py
@@ -17,10 +17,9 @@ import logging
 import re
 from typing import Any, Dict, List, Optional, Pattern, Tuple, Union
 
-from matrix_common.regex import glob_to_regex, to_word_pattern
-
 from synapse.events import EventBase
 from synapse.types import JsonDict, UserID
+from synapse.util import glob_to_regex, re_word_boundary
 from synapse.util.caches.lrucache import LruCache
 
 logger = logging.getLogger(__name__)
@@ -185,7 +184,7 @@ class PushRuleEvaluatorForEvent:
         r = regex_cache.get((display_name, False, True), None)
         if not r:
             r1 = re.escape(display_name)
-            r1 = to_word_pattern(r1)
+            r1 = re_word_boundary(r1)
             r = re.compile(r1, flags=re.IGNORECASE)
             regex_cache[(display_name, False, True)] = r
 
@@ -214,7 +213,7 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool:
     try:
         r = regex_cache.get((glob, True, word_boundary), None)
         if not r:
-            r = glob_to_regex(glob, word_boundary=word_boundary)
+            r = glob_to_regex(glob, word_boundary)
             regex_cache[(glob, True, word_boundary)] = r
         return bool(r.search(value))
     except re.error:
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 386debd7db..7d26954244 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -87,7 +87,6 @@ REQUIREMENTS = [
     # with the latest security patches.
     "cryptography>=3.4.7",
     "ijson>=3.1",
-    "matrix-common==1.0.0",
 ]
 
 CONDITIONAL_REQUIREMENTS = {
diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index f157132210..95f23e27b6 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -14,8 +14,9 @@
 
 import json
 import logging
+import re
 import typing
-from typing import Any, Callable, Dict, Generator, Optional
+from typing import Any, Callable, Dict, Generator, Optional, Pattern
 
 import attr
 from frozendict import frozendict
@@ -34,6 +35,9 @@ if typing.TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+_WILDCARD_RUN = re.compile(r"([\?\*]+)")
+
+
 def _reject_invalid_json(val: Any) -> None:
     """Do not allow Infinity, -Infinity, or NaN values in JSON."""
     raise ValueError("Invalid JSON value: '%s'" % val)
@@ -181,3 +185,56 @@ def log_failure(
     if not consumeErrors:
         return failure
     return None
+
+
+def glob_to_regex(glob: str, word_boundary: bool = False) -> Pattern:
+    """Converts a glob to a compiled regex object.
+
+    Args:
+        glob: pattern to match
+        word_boundary: If True, the pattern will be allowed to match at word boundaries
+           anywhere in the string. Otherwise, the pattern is anchored at the start and
+           end of the string.
+
+    Returns:
+        compiled regex pattern
+    """
+
+    # Patterns with wildcards must be simplified to avoid performance cliffs
+    # - The glob `?**?**?` is equivalent to the glob `???*`
+    # - The glob `???*` is equivalent to the regex `.{3,}`
+    chunks = []
+    for chunk in _WILDCARD_RUN.split(glob):
+        # No wildcards? re.escape()
+        if not _WILDCARD_RUN.match(chunk):
+            chunks.append(re.escape(chunk))
+            continue
+
+        # Wildcards? Simplify.
+        qmarks = chunk.count("?")
+        if "*" in chunk:
+            chunks.append(".{%d,}" % qmarks)
+        else:
+            chunks.append(".{%d}" % qmarks)
+
+    res = "".join(chunks)
+
+    if word_boundary:
+        res = re_word_boundary(res)
+    else:
+        # \A anchors at start of string, \Z at end of string
+        res = r"\A" + res + r"\Z"
+
+    return re.compile(res, re.IGNORECASE)
+
+
+def re_word_boundary(r: str) -> str:
+    """
+    Adds word boundary characters to the start and end of an
+    expression to require that the match occur as a whole word,
+    but do so respecting the fact that strings starting or ending
+    with non-word characters will change word boundaries.
+    """
+    # we can't use \b as it chokes on unicode. however \W seems to be okay
+    # as shorthand for [^0-9A-Za-z_].
+    return r"(^|\W)%s(\W|$)" % (r,)
diff --git a/tests/util/test_glob_to_regex.py b/tests/util/test_glob_to_regex.py
new file mode 100644
index 0000000000..220accb92b
--- /dev/null
+++ b/tests/util/test_glob_to_regex.py
@@ -0,0 +1,59 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from synapse.util import glob_to_regex
+
+from tests.unittest import TestCase
+
+
+class GlobToRegexTestCase(TestCase):
+    def test_literal_match(self):
+        """patterns without wildcards should match"""
+        pat = glob_to_regex("foobaz")
+        self.assertTrue(
+            pat.match("FoobaZ"), "patterns should match and be case-insensitive"
+        )
+        self.assertFalse(
+            pat.match("x foobaz"), "pattern should not match at word boundaries"
+        )
+
+    def test_wildcard_match(self):
+        pat = glob_to_regex("f?o*baz")
+
+        self.assertTrue(
+            pat.match("FoobarbaZ"),
+            "* should match string and pattern should be case-insensitive",
+        )
+        self.assertTrue(pat.match("foobaz"), "* should match 0 characters")
+        self.assertFalse(pat.match("fooxaz"), "the character after * must match")
+        self.assertFalse(pat.match("fobbaz"), "? should not match 0 characters")
+        self.assertFalse(pat.match("fiiobaz"), "? should not match 2 characters")
+
+    def test_multi_wildcard(self):
+        """patterns with multiple wildcards in a row should match"""
+        pat = glob_to_regex("**baz")
+        self.assertTrue(pat.match("agsgsbaz"), "** should match any string")
+        self.assertTrue(pat.match("baz"), "** should match the empty string")
+        self.assertEqual(pat.pattern, r"\A.{0,}baz\Z")
+
+        pat = glob_to_regex("*?baz")
+        self.assertTrue(pat.match("agsgsbaz"), "*? should match any string")
+        self.assertTrue(pat.match("abaz"), "*? should match a single char")
+        self.assertFalse(pat.match("baz"), "*? should not match the empty string")
+        self.assertEqual(pat.pattern, r"\A.{1,}baz\Z")
+
+        pat = glob_to_regex("a?*?*?baz")
+        self.assertTrue(pat.match("a g baz"), "?*?*? should match 3 chars")
+        self.assertFalse(pat.match("a..baz"), "?*?*? should not match 2 chars")
+        self.assertTrue(pat.match("a.gg.baz"), "?*?*? should match 4 chars")
+        self.assertEqual(pat.pattern, r"\Aa.{3,}baz\Z")
-- 
cgit 1.5.1


From 966b5d0fa0893c3b628c942dfc232e285417f46d Mon Sep 17 00:00:00 2001
From: Sean Quah <seanq@element.io>
Date: Tue, 7 Dec 2021 13:56:09 +0000
Subject: 1.49.0rc1

---
 CHANGES.md                | 80 +++++++++++++++++++++++++++++++++++++++++++++++
 changelog.d/11029.misc    |  1 -
 changelog.d/11220.bugfix  |  1 -
 changelog.d/11284.feature |  1 -
 changelog.d/11306.feature |  1 -
 changelog.d/11329.feature |  1 -
 changelog.d/11356.misc    |  1 -
 changelog.d/11376.bugfix  |  1 -
 changelog.d/11402.misc    |  1 -
 changelog.d/11407.feature |  1 -
 changelog.d/11409.misc    |  1 -
 changelog.d/11411.misc    |  1 -
 changelog.d/11413.bugfix  |  1 -
 changelog.d/11415.doc     |  1 -
 changelog.d/11416.misc    |  1 -
 changelog.d/11417.misc    |  1 -
 changelog.d/11425.feature |  1 -
 changelog.d/11428.misc    |  1 -
 changelog.d/11429.docker  |  1 -
 changelog.d/11430.misc    |  1 -
 changelog.d/11435.feature |  1 -
 changelog.d/11439.bugfix  |  1 -
 changelog.d/11440.bugfix  |  1 -
 changelog.d/11441.bugfix  |  1 -
 changelog.d/11445.feature |  1 -
 changelog.d/11446.bugfix  |  1 -
 changelog.d/11449.feature |  1 -
 changelog.d/11451.bugfix  |  1 -
 changelog.d/11452.misc    |  1 -
 changelog.d/11453.misc    |  1 -
 changelog.d/11454.bugfix  |  1 -
 changelog.d/11455.misc    |  1 -
 changelog.d/11459.feature |  1 -
 changelog.d/11460.misc    |  1 -
 changelog.d/11461.misc    |  1 -
 changelog.d/11465.misc    |  1 -
 changelog.d/11466.misc    |  1 -
 changelog.d/11467.misc    |  1 -
 changelog.d/11468.misc    |  1 -
 changelog.d/11469.doc     |  1 -
 changelog.d/11475.feature |  1 -
 changelog.d/11478.bugfix  |  1 -
 changelog.d/11479.feature |  1 -
 changelog.d/11482.misc    |  1 -
 changelog.d/11483.misc    |  1 -
 changelog.d/11486.misc    |  1 -
 changelog.d/11488.misc    |  1 -
 changelog.d/11490.feature |  1 -
 changelog.d/11492.misc    |  1 -
 changelog.d/11493.misc    |  1 -
 changelog.d/11494.misc    |  1 -
 changelog.d/11495.misc    |  1 -
 changelog.d/11497.misc    |  1 -
 changelog.d/11501.misc    |  1 -
 changelog.d/11505.misc    |  1 -
 changelog.d/11511.bugfix  |  1 -
 changelog.d/11515.misc    |  1 -
 changelog.d/11518.bugfix  |  1 -
 changelog.d/11522.feature |  1 -
 changelog.d/11523.feature |  1 -
 changelog.d/11524.bugfix  |  1 -
 changelog.d/11527.misc    |  1 -
 changelog.d/9445.feature  |  1 -
 debian/changelog          |  6 ++++
 synapse/__init__.py       |  2 +-
 65 files changed, 87 insertions(+), 63 deletions(-)
 delete mode 100644 changelog.d/11029.misc
 delete mode 100644 changelog.d/11220.bugfix
 delete mode 100644 changelog.d/11284.feature
 delete mode 100644 changelog.d/11306.feature
 delete mode 100644 changelog.d/11329.feature
 delete mode 100644 changelog.d/11356.misc
 delete mode 100644 changelog.d/11376.bugfix
 delete mode 100644 changelog.d/11402.misc
 delete mode 100644 changelog.d/11407.feature
 delete mode 100644 changelog.d/11409.misc
 delete mode 100644 changelog.d/11411.misc
 delete mode 100644 changelog.d/11413.bugfix
 delete mode 100644 changelog.d/11415.doc
 delete mode 100644 changelog.d/11416.misc
 delete mode 100644 changelog.d/11417.misc
 delete mode 100644 changelog.d/11425.feature
 delete mode 100644 changelog.d/11428.misc
 delete mode 100644 changelog.d/11429.docker
 delete mode 100644 changelog.d/11430.misc
 delete mode 100644 changelog.d/11435.feature
 delete mode 100644 changelog.d/11439.bugfix
 delete mode 100644 changelog.d/11440.bugfix
 delete mode 100644 changelog.d/11441.bugfix
 delete mode 100644 changelog.d/11445.feature
 delete mode 100644 changelog.d/11446.bugfix
 delete mode 100644 changelog.d/11449.feature
 delete mode 100644 changelog.d/11451.bugfix
 delete mode 100644 changelog.d/11452.misc
 delete mode 100644 changelog.d/11453.misc
 delete mode 100644 changelog.d/11454.bugfix
 delete mode 100644 changelog.d/11455.misc
 delete mode 100644 changelog.d/11459.feature
 delete mode 100644 changelog.d/11460.misc
 delete mode 100644 changelog.d/11461.misc
 delete mode 100644 changelog.d/11465.misc
 delete mode 100644 changelog.d/11466.misc
 delete mode 100644 changelog.d/11467.misc
 delete mode 100644 changelog.d/11468.misc
 delete mode 100644 changelog.d/11469.doc
 delete mode 100644 changelog.d/11475.feature
 delete mode 100644 changelog.d/11478.bugfix
 delete mode 100644 changelog.d/11479.feature
 delete mode 100644 changelog.d/11482.misc
 delete mode 100644 changelog.d/11483.misc
 delete mode 100644 changelog.d/11486.misc
 delete mode 100644 changelog.d/11488.misc
 delete mode 100644 changelog.d/11490.feature
 delete mode 100644 changelog.d/11492.misc
 delete mode 100644 changelog.d/11493.misc
 delete mode 100644 changelog.d/11494.misc
 delete mode 100644 changelog.d/11495.misc
 delete mode 100644 changelog.d/11497.misc
 delete mode 100644 changelog.d/11501.misc
 delete mode 100644 changelog.d/11505.misc
 delete mode 100644 changelog.d/11511.bugfix
 delete mode 100644 changelog.d/11515.misc
 delete mode 100644 changelog.d/11518.bugfix
 delete mode 100644 changelog.d/11522.feature
 delete mode 100644 changelog.d/11523.feature
 delete mode 100644 changelog.d/11524.bugfix
 delete mode 100644 changelog.d/11527.misc
 delete mode 100644 changelog.d/9445.feature

diff --git a/CHANGES.md b/CHANGES.md
index f398b8e9c5..66ec691af0 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,83 @@
+Synapse 1.49.0rc1 (2021-12-07)
+==============================
+
+Features
+--------
+
+- Add [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) experimental client and federation API endpoints to get the closest event to a given timestamp. ([\#9445](https://github.com/matrix-org/synapse/issues/9445))
+- When returning relation events from the `/relations` API, bundle any relations of those relations into the result, per updates to [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11284](https://github.com/matrix-org/synapse/issues/11284))
+- Add plugin support for controlling database background updates. ([\#11306](https://github.com/matrix-org/synapse/issues/11306), [\#11475](https://github.com/matrix-org/synapse/issues/11475), [\#11479](https://github.com/matrix-org/synapse/issues/11479))
+- Support the stable API endpoints for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): the room `/hierarchy` endpoint. ([\#11329](https://github.com/matrix-org/synapse/issues/11329))
+- Add admin API to get some information about federation status with remote servers. ([\#11407](https://github.com/matrix-org/synapse/issues/11407))
+- Support expiry of refresh tokens and expiry of the overall session when refresh tokens are in use. ([\#11425](https://github.com/matrix-org/synapse/issues/11425))
+- Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification. ([\#11435](https://github.com/matrix-org/synapse/issues/11435), [\#11522](https://github.com/matrix-org/synapse/issues/11522))
+- Support configuring the lifetime of non-refreshable access tokens separately to refreshable access tokens. ([\#11445](https://github.com/matrix-org/synapse/issues/11445))
+- Expose synapse_homeserver and synapse_worker commands as entry points to run Synapse's main process and worker processes, respectively. Contributed by @Ma27. ([\#11449](https://github.com/matrix-org/synapse/issues/11449))
+- `synctl stop` will now wait for Synapse to exit before returning. ([\#11459](https://github.com/matrix-org/synapse/issues/11459), [\#11490](https://github.com/matrix-org/synapse/issues/11490))
+- Extend the "delete room" admin api to work correctly on rooms which have previously been partially deleted. ([\#11523](https://github.com/matrix-org/synapse/issues/11523))
+
+
+Bugfixes
+--------
+
+- Fix using MSC2716 batch sending in combination with event persistence workers. Contributed by @tulir at Beeper. ([\#11220](https://github.com/matrix-org/synapse/issues/11220))
+- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection, for real this time. Also fix a race condition introduced in the previous insufficient fix in 1.47.0. ([\#11376](https://github.com/matrix-org/synapse/issues/11376))
+- The `/send_join` response now includes the stable `event` field instead of the unstable field from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#11413](https://github.com/matrix-org/synapse/issues/11413))
+- Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11439](https://github.com/matrix-org/synapse/issues/11439), [\#11441](https://github.com/matrix-org/synapse/issues/11441))
+- Fix a bug introduced in Synapse 1.36 which could cause problems fetching event-signing keys from trusted key servers. ([\#11440](https://github.com/matrix-org/synapse/issues/11440))
+- Fix a bug introduced in 1.47.1 where the media repository would fail to work if the media store path contained any symbolic links. ([\#11446](https://github.com/matrix-org/synapse/issues/11446))
+- Add support for the `/_matrix/client/v3/login/sso/redirect/{idpId}` API from Matrix v1.1. This endpoint was overlooked when support for v3 endpoints was added in v1.48.0rc1. ([\#11451](https://github.com/matrix-org/synapse/issues/11451))
+- Fix an `LruCache` corruption bug, introduced in 1.38.0, that would cause certain requests to fail until the next Synapse restart. ([\#11454](https://github.com/matrix-org/synapse/issues/11454))
+- Include bundled relation aggregations during a limited `/sync` request, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11478](https://github.com/matrix-org/synapse/issues/11478))
+- Fix a long-standing bug where invites from ignored users were included in incremental syncs. ([\#11511](https://github.com/matrix-org/synapse/issues/11511))
+- Fix a regression in Synapse 1.48.0 where presence workers would not clear their presence updates over replication on shutdown. ([\#11518](https://github.com/matrix-org/synapse/issues/11518))
+- Fix a regression in Synapse 1.48.0 where the module API's `looping_background_call` method would spam errors to the logs when given a non-async function. ([\#11524](https://github.com/matrix-org/synapse/issues/11524))
+
+
+Updates to the Docker image
+---------------------------
+
+- Update `Dockerfile-workers` to healthcheck all workers in container. ([\#11429](https://github.com/matrix-org/synapse/issues/11429))
+
+
+Improved Documentation
+----------------------
+
+- Update the media repository documentation. ([\#11415](https://github.com/matrix-org/synapse/issues/11415))
+- Update section about backward extremities in the room DAG concepts doc to correct the misconception about backward extremities indicating whether we have fetched an events' `prev_events`. ([\#11469](https://github.com/matrix-org/synapse/issues/11469))
+
+
+Internal Changes
+----------------
+
+- Improve type annotations in `synapse.module_api`. ([\#11029](https://github.com/matrix-org/synapse/issues/11029))
+- Add `Final` annotation to string constants in `synapse.api.constants` so that they get typed as `Literal`s. ([\#11356](https://github.com/matrix-org/synapse/issues/11356))
+- Add wiki pages to documentation website. ([\#11402](https://github.com/matrix-org/synapse/issues/11402))
+- Improve internal types in push code. ([\#11409](https://github.com/matrix-org/synapse/issues/11409))
+- Add type hints to storage classes. ([\#11411](https://github.com/matrix-org/synapse/issues/11411))
+- Add a check to ensure that users cannot start the Synapse master process when `worker_app` is set. ([\#11416](https://github.com/matrix-org/synapse/issues/11416))
+- Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls). ([\#11417](https://github.com/matrix-org/synapse/issues/11417))
+- Add type annotations to some of the configuration surrounding refresh tokens. ([\#11428](https://github.com/matrix-org/synapse/issues/11428))
+- Update [MSC2918 refresh token](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) support to confirm with the latest revision: accept the `refresh_tokens` parameter in the request body rather than in the URL parameters. ([\#11430](https://github.com/matrix-org/synapse/issues/11430))
+- Convert status codes to `HTTPStatus` in `synapse.rest.admin`. ([\#11452](https://github.com/matrix-org/synapse/issues/11452), [\#11455](https://github.com/matrix-org/synapse/issues/11455))
+- Improve type hints for `LruCache`. ([\#11453](https://github.com/matrix-org/synapse/issues/11453))
+- Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11460](https://github.com/matrix-org/synapse/issues/11460))
+- Remove unnecessary `json.dumps` from `tests.rest.admin`. ([\#11461](https://github.com/matrix-org/synapse/issues/11461))
+- Add missing type hints to `synapse.config` module. ([\#11465](https://github.com/matrix-org/synapse/issues/11465))
+- Update and clean up recently ported documentation pages. ([\#11466](https://github.com/matrix-org/synapse/issues/11466))
+- Add a note about postgres memory management and hugepages to postgres doc. ([\#11467](https://github.com/matrix-org/synapse/issues/11467))
+- Refactor `get_version_string` to fix-up types and duplicated code. ([\#11468](https://github.com/matrix-org/synapse/issues/11468))
+- Save the OpenID Connect session ID on login. ([\#11482](https://github.com/matrix-org/synapse/issues/11482))
+- Add missing type hints to `synapse.federation`. ([\#11483](https://github.com/matrix-org/synapse/issues/11483))
+- Extend the `scripts-dev/sign_json` script to support signing events. ([\#11486](https://github.com/matrix-org/synapse/issues/11486))
+- Add type annotations to `tests.storage.test_appservice`. ([\#11488](https://github.com/matrix-org/synapse/issues/11488), [\#11492](https://github.com/matrix-org/synapse/issues/11492))
+- Clean up `tests.storage.test_main` to remove use of legacy code. ([\#11493](https://github.com/matrix-org/synapse/issues/11493))
+- Refactor various parts of the `/sync` handler. ([\#11494](https://github.com/matrix-org/synapse/issues/11494), [\#11515](https://github.com/matrix-org/synapse/issues/11515))
+- Clean up `tests.test_visibility` to remove legacy code. ([\#11495](https://github.com/matrix-org/synapse/issues/11495))
+- Preparation for database schema simplifications: disambiguate queries on `state_key`. ([\#11497](https://github.com/matrix-org/synapse/issues/11497))
+- Add type hints to `synapse/tests/rest/admin`. ([\#11501](https://github.com/matrix-org/synapse/issues/11501))
+
+
 UNRELEASED
 ==========
 
diff --git a/changelog.d/11029.misc b/changelog.d/11029.misc
deleted file mode 100644
index 111de5fc7a..0000000000
--- a/changelog.d/11029.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type annotations in `synapse.module_api`.
\ No newline at end of file
diff --git a/changelog.d/11220.bugfix b/changelog.d/11220.bugfix
deleted file mode 100644
index 8baae28d5b..0000000000
--- a/changelog.d/11220.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix using MSC2716 batch sending in combination with event persistence workers. Contributed by @tulir at Beeper.
diff --git a/changelog.d/11284.feature b/changelog.d/11284.feature
deleted file mode 100644
index cbaa5a988c..0000000000
--- a/changelog.d/11284.feature
+++ /dev/null
@@ -1 +0,0 @@
-When returning relation events from the `/relations` API, bundle any relations of those relations into the result, per updates to [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675).
diff --git a/changelog.d/11306.feature b/changelog.d/11306.feature
deleted file mode 100644
index aba3292015..0000000000
--- a/changelog.d/11306.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add plugin support for controlling database background updates.
diff --git a/changelog.d/11329.feature b/changelog.d/11329.feature
deleted file mode 100644
index 7e0efb3b00..0000000000
--- a/changelog.d/11329.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support the stable API endpoints for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): the room `/hierarchy` endpoint.
diff --git a/changelog.d/11356.misc b/changelog.d/11356.misc
deleted file mode 100644
index 01ce6a306c..0000000000
--- a/changelog.d/11356.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add `Final` annotation to string constants in `synapse.api.constants` so that they get typed as `Literal`s.
diff --git a/changelog.d/11376.bugfix b/changelog.d/11376.bugfix
deleted file mode 100644
index 639e48b59b..0000000000
--- a/changelog.d/11376.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection, for real this time. Also fix a race condition introduced in the previous insufficient fix in 1.47.0.
diff --git a/changelog.d/11402.misc b/changelog.d/11402.misc
deleted file mode 100644
index c956338744..0000000000
--- a/changelog.d/11402.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add wiki pages to documentation website.
\ No newline at end of file
diff --git a/changelog.d/11407.feature b/changelog.d/11407.feature
deleted file mode 100644
index 1d21bde98f..0000000000
--- a/changelog.d/11407.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add admin API to get some information about federation status with remote servers.
\ No newline at end of file
diff --git a/changelog.d/11409.misc b/changelog.d/11409.misc
deleted file mode 100644
index f9e8ae9e3a..0000000000
--- a/changelog.d/11409.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve internal types in push code.
diff --git a/changelog.d/11411.misc b/changelog.d/11411.misc
deleted file mode 100644
index 86594a332d..0000000000
--- a/changelog.d/11411.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to storage classes.
diff --git a/changelog.d/11413.bugfix b/changelog.d/11413.bugfix
deleted file mode 100644
index 44111d8152..0000000000
--- a/changelog.d/11413.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-The `/send_join` response now includes the stable `event` field instead of the unstable field from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083).
diff --git a/changelog.d/11415.doc b/changelog.d/11415.doc
deleted file mode 100644
index e405531867..0000000000
--- a/changelog.d/11415.doc
+++ /dev/null
@@ -1 +0,0 @@
-Update the media repository documentation.
diff --git a/changelog.d/11416.misc b/changelog.d/11416.misc
deleted file mode 100644
index a5c3aeda83..0000000000
--- a/changelog.d/11416.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a check to ensure that users cannot start the Synapse master process when `worker_app` is set.
\ No newline at end of file
diff --git a/changelog.d/11417.misc b/changelog.d/11417.misc
deleted file mode 100644
index 88dc4722da..0000000000
--- a/changelog.d/11417.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls).
diff --git a/changelog.d/11425.feature b/changelog.d/11425.feature
deleted file mode 100644
index 806dd5d91c..0000000000
--- a/changelog.d/11425.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support expiry of refresh tokens and expiry of the overall session when refresh tokens are in use.
\ No newline at end of file
diff --git a/changelog.d/11428.misc b/changelog.d/11428.misc
deleted file mode 100644
index 2f814fa5fb..0000000000
--- a/changelog.d/11428.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type annotations to some of the configuration surrounding refresh tokens.
\ No newline at end of file
diff --git a/changelog.d/11429.docker b/changelog.d/11429.docker
deleted file mode 100644
index 81db719ed6..0000000000
--- a/changelog.d/11429.docker
+++ /dev/null
@@ -1 +0,0 @@
-Update `Dockerfile-workers` to healthcheck all workers in container.
diff --git a/changelog.d/11430.misc b/changelog.d/11430.misc
deleted file mode 100644
index 28f06f4c4e..0000000000
--- a/changelog.d/11430.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update [MSC2918 refresh token](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) support to confirm with the latest revision: accept the `refresh_tokens` parameter in the request body rather than in the URL parameters.
\ No newline at end of file
diff --git a/changelog.d/11435.feature b/changelog.d/11435.feature
deleted file mode 100644
index 9e127fae3c..0000000000
--- a/changelog.d/11435.feature
+++ /dev/null
@@ -1 +0,0 @@
-Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification.
\ No newline at end of file
diff --git a/changelog.d/11439.bugfix b/changelog.d/11439.bugfix
deleted file mode 100644
index fc6bc82b36..0000000000
--- a/changelog.d/11439.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version.
diff --git a/changelog.d/11440.bugfix b/changelog.d/11440.bugfix
deleted file mode 100644
index 02ce2e428f..0000000000
--- a/changelog.d/11440.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.36 which could cause problems fetching event-signing keys from trusted key servers.
diff --git a/changelog.d/11441.bugfix b/changelog.d/11441.bugfix
deleted file mode 100644
index 1baef41d70..0000000000
--- a/changelog.d/11441.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version.
\ No newline at end of file
diff --git a/changelog.d/11445.feature b/changelog.d/11445.feature
deleted file mode 100644
index 211a722b65..0000000000
--- a/changelog.d/11445.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support configuring the lifetime of non-refreshable access tokens separately to refreshable access tokens.
\ No newline at end of file
diff --git a/changelog.d/11446.bugfix b/changelog.d/11446.bugfix
deleted file mode 100644
index fa5e055d50..0000000000
--- a/changelog.d/11446.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in 1.47.1 where the media repository would fail to work if the media store path contained any symbolic links.
diff --git a/changelog.d/11449.feature b/changelog.d/11449.feature
deleted file mode 100644
index 94bc175db9..0000000000
--- a/changelog.d/11449.feature
+++ /dev/null
@@ -1 +0,0 @@
-Expose synapse_homeserver and synapse_worker commands as entry points to run Synapse's main process and worker processes, respectively. Contributed by @Ma27.
diff --git a/changelog.d/11451.bugfix b/changelog.d/11451.bugfix
deleted file mode 100644
index 960714d0f9..0000000000
--- a/changelog.d/11451.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Add support for the `/_matrix/client/v3/login/sso/redirect/{idpId}` API from Matrix v1.1. This endpoint was overlooked when support for v3 endpoints was added in v1.48.0rc1.
diff --git a/changelog.d/11452.misc b/changelog.d/11452.misc
deleted file mode 100644
index 7c83f62e3f..0000000000
--- a/changelog.d/11452.misc
+++ /dev/null
@@ -1 +0,0 @@
-Convert status codes to `HTTPStatus` in `synapse.rest.admin`.
\ No newline at end of file
diff --git a/changelog.d/11453.misc b/changelog.d/11453.misc
deleted file mode 100644
index a31ab826e5..0000000000
--- a/changelog.d/11453.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve type hints for `LruCache`.
diff --git a/changelog.d/11454.bugfix b/changelog.d/11454.bugfix
deleted file mode 100644
index 096265cbc9..0000000000
--- a/changelog.d/11454.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix an `LruCache` corruption bug, introduced in 1.38.0, that would cause certain requests to fail until the next Synapse restart.
diff --git a/changelog.d/11455.misc b/changelog.d/11455.misc
deleted file mode 100644
index 7c83f62e3f..0000000000
--- a/changelog.d/11455.misc
+++ /dev/null
@@ -1 +0,0 @@
-Convert status codes to `HTTPStatus` in `synapse.rest.admin`.
\ No newline at end of file
diff --git a/changelog.d/11459.feature b/changelog.d/11459.feature
deleted file mode 100644
index 4cb97dc1d0..0000000000
--- a/changelog.d/11459.feature
+++ /dev/null
@@ -1 +0,0 @@
-`synctl stop` will now wait for Synapse to exit before returning.
diff --git a/changelog.d/11460.misc b/changelog.d/11460.misc
deleted file mode 100644
index fc6bc82b36..0000000000
--- a/changelog.d/11460.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version.
diff --git a/changelog.d/11461.misc b/changelog.d/11461.misc
deleted file mode 100644
index 92133f9eaa..0000000000
--- a/changelog.d/11461.misc
+++ /dev/null
@@ -1 +0,0 @@
-Remove unnecessary `json.dumps` from `tests.rest.admin`.
\ No newline at end of file
diff --git a/changelog.d/11465.misc b/changelog.d/11465.misc
deleted file mode 100644
index aadc938b2b..0000000000
--- a/changelog.d/11465.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add missing type hints to `synapse.config` module.
diff --git a/changelog.d/11466.misc b/changelog.d/11466.misc
deleted file mode 100644
index 4317d017d7..0000000000
--- a/changelog.d/11466.misc
+++ /dev/null
@@ -1 +0,0 @@
-Update and clean up recently ported documentation pages.
\ No newline at end of file
diff --git a/changelog.d/11467.misc b/changelog.d/11467.misc
deleted file mode 100644
index 84e8a5ab4e..0000000000
--- a/changelog.d/11467.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a note about postgres memory management and hugepages to postgres doc.
\ No newline at end of file
diff --git a/changelog.d/11468.misc b/changelog.d/11468.misc
deleted file mode 100644
index 6fc0b5bcab..0000000000
--- a/changelog.d/11468.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor `get_version_string` to fix-up types and duplicated code.
diff --git a/changelog.d/11469.doc b/changelog.d/11469.doc
deleted file mode 100644
index 51602aa968..0000000000
--- a/changelog.d/11469.doc
+++ /dev/null
@@ -1 +0,0 @@
-Update section about backward extremities in the room DAG concepts doc to correct the misconception about backward extremities indicating whether we have fetched an events' `prev_events`.
diff --git a/changelog.d/11475.feature b/changelog.d/11475.feature
deleted file mode 100644
index aba3292015..0000000000
--- a/changelog.d/11475.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add plugin support for controlling database background updates.
diff --git a/changelog.d/11478.bugfix b/changelog.d/11478.bugfix
deleted file mode 100644
index 5f02636f50..0000000000
--- a/changelog.d/11478.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Include bundled relation aggregations during a limited `/sync` request, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675).
diff --git a/changelog.d/11479.feature b/changelog.d/11479.feature
deleted file mode 100644
index aba3292015..0000000000
--- a/changelog.d/11479.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add plugin support for controlling database background updates.
diff --git a/changelog.d/11482.misc b/changelog.d/11482.misc
deleted file mode 100644
index e78662988f..0000000000
--- a/changelog.d/11482.misc
+++ /dev/null
@@ -1 +0,0 @@
-Save the OpenID Connect session ID on login.
diff --git a/changelog.d/11483.misc b/changelog.d/11483.misc
deleted file mode 100644
index 4cc47fcab9..0000000000
--- a/changelog.d/11483.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add missing type hints to `synapse.federation`.
diff --git a/changelog.d/11486.misc b/changelog.d/11486.misc
deleted file mode 100644
index 966f04d185..0000000000
--- a/changelog.d/11486.misc
+++ /dev/null
@@ -1 +0,0 @@
-Extend the `scripts-dev/sign_json` script to support signing events.
diff --git a/changelog.d/11488.misc b/changelog.d/11488.misc
deleted file mode 100644
index c14a7d2e98..0000000000
--- a/changelog.d/11488.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type annotations to `tests.storage.test_appservice`.
\ No newline at end of file
diff --git a/changelog.d/11490.feature b/changelog.d/11490.feature
deleted file mode 100644
index 4cb97dc1d0..0000000000
--- a/changelog.d/11490.feature
+++ /dev/null
@@ -1 +0,0 @@
-`synctl stop` will now wait for Synapse to exit before returning.
diff --git a/changelog.d/11492.misc b/changelog.d/11492.misc
deleted file mode 100644
index c14a7d2e98..0000000000
--- a/changelog.d/11492.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type annotations to `tests.storage.test_appservice`.
\ No newline at end of file
diff --git a/changelog.d/11493.misc b/changelog.d/11493.misc
deleted file mode 100644
index 646584a0d1..0000000000
--- a/changelog.d/11493.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up `tests.storage.test_main` to remove use of legacy code.
\ No newline at end of file
diff --git a/changelog.d/11494.misc b/changelog.d/11494.misc
deleted file mode 100644
index 52cf26a4b6..0000000000
--- a/changelog.d/11494.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor various parts of the `/sync` handler.
\ No newline at end of file
diff --git a/changelog.d/11495.misc b/changelog.d/11495.misc
deleted file mode 100644
index 5b52697fb4..0000000000
--- a/changelog.d/11495.misc
+++ /dev/null
@@ -1 +0,0 @@
-Clean up `tests.test_visibility` to remove legacy code.
\ No newline at end of file
diff --git a/changelog.d/11497.misc b/changelog.d/11497.misc
deleted file mode 100644
index c4393f6193..0000000000
--- a/changelog.d/11497.misc
+++ /dev/null
@@ -1 +0,0 @@
-Preparation for database schema simplifications: disambiguate queries on `state_key`.
diff --git a/changelog.d/11501.misc b/changelog.d/11501.misc
deleted file mode 100644
index 40e01194df..0000000000
--- a/changelog.d/11501.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to `synapse/tests/rest/admin`.
\ No newline at end of file
diff --git a/changelog.d/11505.misc b/changelog.d/11505.misc
deleted file mode 100644
index 926b562fad..0000000000
--- a/changelog.d/11505.misc
+++ /dev/null
@@ -1 +0,0 @@
-Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`.
diff --git a/changelog.d/11511.bugfix b/changelog.d/11511.bugfix
deleted file mode 100644
index 9c287357b8..0000000000
--- a/changelog.d/11511.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a long-standing bug where invites from ignored users were included in incremental syncs.
\ No newline at end of file
diff --git a/changelog.d/11515.misc b/changelog.d/11515.misc
deleted file mode 100644
index 7f9d8cd57f..0000000000
--- a/changelog.d/11515.misc
+++ /dev/null
@@ -1 +0,0 @@
-Refactor various parts of the `/sync` handler.
diff --git a/changelog.d/11518.bugfix b/changelog.d/11518.bugfix
deleted file mode 100644
index 3c27382c7a..0000000000
--- a/changelog.d/11518.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a regression in Synapse 1.48.0 where presence workers would not clear their presence updates over replication on shutdown.
diff --git a/changelog.d/11522.feature b/changelog.d/11522.feature
deleted file mode 100644
index 9e127fae3c..0000000000
--- a/changelog.d/11522.feature
+++ /dev/null
@@ -1 +0,0 @@
-Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification.
\ No newline at end of file
diff --git a/changelog.d/11523.feature b/changelog.d/11523.feature
deleted file mode 100644
index ecac7f9db9..0000000000
--- a/changelog.d/11523.feature
+++ /dev/null
@@ -1 +0,0 @@
-Extend the "delete room" admin api to work correctly on rooms which have previously been partially deleted.
diff --git a/changelog.d/11524.bugfix b/changelog.d/11524.bugfix
deleted file mode 100644
index 6e1e4bd44c..0000000000
--- a/changelog.d/11524.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a regression in Synapse 1.48.0 where the module API's `looping_background_call` method would spam errors to the logs when given a non-async function.
diff --git a/changelog.d/11527.misc b/changelog.d/11527.misc
deleted file mode 100644
index 081eae317c..0000000000
--- a/changelog.d/11527.misc
+++ /dev/null
@@ -1 +0,0 @@
-Temporarily revert usage of `matrix-python-common`.
diff --git a/changelog.d/9445.feature b/changelog.d/9445.feature
deleted file mode 100644
index 6d12eea71f..0000000000
--- a/changelog.d/9445.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) experimental client and federation API endpoints to get the closest event to a given timestamp.
diff --git a/debian/changelog b/debian/changelog
index 7deab5936e..acc9f6049e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.49.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.49.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Dec 2021 13:52:21 +0000
+
 matrix-synapse-py3 (1.48.0) stable; urgency=medium
 
   * New synapse release 1.48.0.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 3cd1ce6070..6369f18a53 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.48.0"
+__version__ = "1.49.0rc1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when
-- 
cgit 1.5.1


From bce4220f387bf5448387f0ed7d14ed1e41e40747 Mon Sep 17 00:00:00 2001
From: Sean Quah <seanq@element.io>
Date: Tue, 7 Dec 2021 14:41:06 +0000
Subject: Update CHANGES.md

---
 CHANGES.md | 56 ++++++++++++++++++++++++++------------------------------
 1 file changed, 26 insertions(+), 30 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 66ec691af0..a875aec00e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,34 +1,49 @@
 Synapse 1.49.0rc1 (2021-12-07)
 ==============================
 
+We've decided to move the existing, somewhat stagnant pages from the GitHub wiki
+to the [documentation website](https://matrix-org.github.io/synapse/latest/).
+
+This was done for two reasons. The first was to ensure that changes are checked by
+multiple authors before being committed (everyone makes mistakes!) and the second
+was visibility of the documentation. Not everyone knows that Synapse has some very
+useful information hidden away in its GitHub wiki pages. Bringing them to the
+documentation website should help with visibility, as well as keep all Synapse documentation
+in one, easily-searchable location.
+
+Note that contributions to the documentation website happen through [GitHub pull
+requests](https://github.com/matrix-org/synapse/pulls). Please visit [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)
+if you need help with the process!
+
+
 Features
 --------
 
 - Add [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) experimental client and federation API endpoints to get the closest event to a given timestamp. ([\#9445](https://github.com/matrix-org/synapse/issues/9445))
-- When returning relation events from the `/relations` API, bundle any relations of those relations into the result, per updates to [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11284](https://github.com/matrix-org/synapse/issues/11284))
+- Include bundled relation aggregations during a limited `/sync` request and `/relations` request, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11284](https://github.com/matrix-org/synapse/issues/11284), [\#11478](https://github.com/matrix-org/synapse/issues/11478))
 - Add plugin support for controlling database background updates. ([\#11306](https://github.com/matrix-org/synapse/issues/11306), [\#11475](https://github.com/matrix-org/synapse/issues/11475), [\#11479](https://github.com/matrix-org/synapse/issues/11479))
 - Support the stable API endpoints for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): the room `/hierarchy` endpoint. ([\#11329](https://github.com/matrix-org/synapse/issues/11329))
 - Add admin API to get some information about federation status with remote servers. ([\#11407](https://github.com/matrix-org/synapse/issues/11407))
 - Support expiry of refresh tokens and expiry of the overall session when refresh tokens are in use. ([\#11425](https://github.com/matrix-org/synapse/issues/11425))
 - Stabilise support for [MSC2918](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) refresh tokens as they have now been merged into the Matrix specification. ([\#11435](https://github.com/matrix-org/synapse/issues/11435), [\#11522](https://github.com/matrix-org/synapse/issues/11522))
+- Update [MSC2918 refresh token](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) support to confirm with the latest revision: accept the `refresh_tokens` parameter in the request body rather than in the URL parameters. ([\#11430](https://github.com/matrix-org/synapse/issues/11430))
 - Support configuring the lifetime of non-refreshable access tokens separately to refreshable access tokens. ([\#11445](https://github.com/matrix-org/synapse/issues/11445))
-- Expose synapse_homeserver and synapse_worker commands as entry points to run Synapse's main process and worker processes, respectively. Contributed by @Ma27. ([\#11449](https://github.com/matrix-org/synapse/issues/11449))
+- Expose `synapse_homeserver` and `synapse_worker` commands as entry points to run Synapse's main process and worker processes, respectively. Contributed by @Ma27. ([\#11449](https://github.com/matrix-org/synapse/issues/11449))
 - `synctl stop` will now wait for Synapse to exit before returning. ([\#11459](https://github.com/matrix-org/synapse/issues/11459), [\#11490](https://github.com/matrix-org/synapse/issues/11490))
 - Extend the "delete room" admin api to work correctly on rooms which have previously been partially deleted. ([\#11523](https://github.com/matrix-org/synapse/issues/11523))
+- Add support for the `/_matrix/client/v3/login/sso/redirect/{idpId}` API from Matrix v1.1. This endpoint was overlooked when support for v3 endpoints was added in Synapse 1.48.0rc1. ([\#11451](https://github.com/matrix-org/synapse/issues/11451))
 
 
 Bugfixes
 --------
 
-- Fix using MSC2716 batch sending in combination with event persistence workers. Contributed by @tulir at Beeper. ([\#11220](https://github.com/matrix-org/synapse/issues/11220))
-- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection, for real this time. Also fix a race condition introduced in the previous insufficient fix in 1.47.0. ([\#11376](https://github.com/matrix-org/synapse/issues/11376))
+- Fix using [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) batch sending in combination with event persistence workers. Contributed by @tulir at Beeper. ([\#11220](https://github.com/matrix-org/synapse/issues/11220))
+- Fix a long-standing bug where all requests that read events from the database could get stuck as a result of losing the database connection, properly this time. Also fix a race condition introduced in the previous insufficient fix in Synapse 1.47.0. ([\#11376](https://github.com/matrix-org/synapse/issues/11376))
 - The `/send_join` response now includes the stable `event` field instead of the unstable field from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083). ([\#11413](https://github.com/matrix-org/synapse/issues/11413))
-- Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11439](https://github.com/matrix-org/synapse/issues/11439), [\#11441](https://github.com/matrix-org/synapse/issues/11441))
-- Fix a bug introduced in Synapse 1.36 which could cause problems fetching event-signing keys from trusted key servers. ([\#11440](https://github.com/matrix-org/synapse/issues/11440))
-- Fix a bug introduced in 1.47.1 where the media repository would fail to work if the media store path contained any symbolic links. ([\#11446](https://github.com/matrix-org/synapse/issues/11446))
-- Add support for the `/_matrix/client/v3/login/sso/redirect/{idpId}` API from Matrix v1.1. This endpoint was overlooked when support for v3 endpoints was added in v1.48.0rc1. ([\#11451](https://github.com/matrix-org/synapse/issues/11451))
-- Fix an `LruCache` corruption bug, introduced in 1.38.0, that would cause certain requests to fail until the next Synapse restart. ([\#11454](https://github.com/matrix-org/synapse/issues/11454))
-- Include bundled relation aggregations during a limited `/sync` request, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11478](https://github.com/matrix-org/synapse/issues/11478))
+- Fix a bug introduced in Synapse 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11439](https://github.com/matrix-org/synapse/issues/11439), [\#11441](https://github.com/matrix-org/synapse/issues/11441), [\#11460](https://github.com/matrix-org/synapse/issues/11460))
+- Fix a bug introduced in Synapse 1.36.0 which could cause problems fetching event-signing keys from trusted key servers. ([\#11440](https://github.com/matrix-org/synapse/issues/11440))
+- Fix a bug introduced in Synapse 1.47.1 where the media repository would fail to work if the media store path contained any symbolic links. ([\#11446](https://github.com/matrix-org/synapse/issues/11446))
+- Fix an `LruCache` corruption bug, introduced in Synapse 1.38.0, that would cause certain requests to fail until the next Synapse restart. ([\#11454](https://github.com/matrix-org/synapse/issues/11454))
 - Fix a long-standing bug where invites from ignored users were included in incremental syncs. ([\#11511](https://github.com/matrix-org/synapse/issues/11511))
 - Fix a regression in Synapse 1.48.0 where presence workers would not clear their presence updates over replication on shutdown. ([\#11518](https://github.com/matrix-org/synapse/issues/11518))
 - Fix a regression in Synapse 1.48.0 where the module API's `looping_background_call` method would spam errors to the logs when given a non-async function. ([\#11524](https://github.com/matrix-org/synapse/issues/11524))
@@ -37,7 +52,7 @@ Bugfixes
 Updates to the Docker image
 ---------------------------
 
-- Update `Dockerfile-workers` to healthcheck all workers in container. ([\#11429](https://github.com/matrix-org/synapse/issues/11429))
+- Update `Dockerfile-workers` to healthcheck all workers in the container. ([\#11429](https://github.com/matrix-org/synapse/issues/11429))
 
 
 Improved Documentation
@@ -58,10 +73,8 @@ Internal Changes
 - Add a check to ensure that users cannot start the Synapse master process when `worker_app` is set. ([\#11416](https://github.com/matrix-org/synapse/issues/11416))
 - Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls). ([\#11417](https://github.com/matrix-org/synapse/issues/11417))
 - Add type annotations to some of the configuration surrounding refresh tokens. ([\#11428](https://github.com/matrix-org/synapse/issues/11428))
-- Update [MSC2918 refresh token](https://github.com/matrix-org/matrix-doc/blob/main/proposals/2918-refreshtokens.md#msc2918-refresh-tokens) support to confirm with the latest revision: accept the `refresh_tokens` parameter in the request body rather than in the URL parameters. ([\#11430](https://github.com/matrix-org/synapse/issues/11430))
 - Convert status codes to `HTTPStatus` in `synapse.rest.admin`. ([\#11452](https://github.com/matrix-org/synapse/issues/11452), [\#11455](https://github.com/matrix-org/synapse/issues/11455))
 - Improve type hints for `LruCache`. ([\#11453](https://github.com/matrix-org/synapse/issues/11453))
-- Fix a bug introduced in 1.47.0 where `send_join` could fail due to an outdated `ijson` version. ([\#11460](https://github.com/matrix-org/synapse/issues/11460))
 - Remove unnecessary `json.dumps` from `tests.rest.admin`. ([\#11461](https://github.com/matrix-org/synapse/issues/11461))
 - Add missing type hints to `synapse.config` module. ([\#11465](https://github.com/matrix-org/synapse/issues/11465))
 - Update and clean up recently ported documentation pages. ([\#11466](https://github.com/matrix-org/synapse/issues/11466))
@@ -78,23 +91,6 @@ Internal Changes
 - Add type hints to `synapse/tests/rest/admin`. ([\#11501](https://github.com/matrix-org/synapse/issues/11501))
 
 
-UNRELEASED
-==========
-
-We've decided to move the existing, somewhat stagnant pages from the GitHub wiki
-to the [documentation website](https://matrix-org.github.io/synapse/latest/).
-
-This was done for two reasons. The first was to ensure that changes are checked by 
-multiple authors before being committed (everyone makes mistakes!) and the second
-was visibility of the documentation. Not everyone knows that Synapse has some very
-useful information hidden away in its GitHub wiki pages. Bringing them to the 
-documentation website should help with visibility, as well as keep all Synapse documentation
-in one, easily-searchable location.
-
-Note that contributions to the documentation website happen through [GitHub pull
-requests](https://github.com/matrix-org/synapse/pulls). Please visit [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)
-if you need help with the process!
-
 Synapse 1.48.0 (2021-11-30)
 ===========================
 
-- 
cgit 1.5.1


From 26b5d2320f62b5eb6262c7614fbdfc364a4dfc02 Mon Sep 17 00:00:00 2001
From: Sean Quah <seanq@element.io>
Date: Tue, 7 Dec 2021 14:46:33 +0000
Subject: Sort internal changes in changelog

---
 CHANGES.md | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index a875aec00e..72e8d64cf7 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -65,30 +65,30 @@ Improved Documentation
 Internal Changes
 ----------------
 
-- Improve type annotations in `synapse.module_api`. ([\#11029](https://github.com/matrix-org/synapse/issues/11029))
 - Add `Final` annotation to string constants in `synapse.api.constants` so that they get typed as `Literal`s. ([\#11356](https://github.com/matrix-org/synapse/issues/11356))
-- Add wiki pages to documentation website. ([\#11402](https://github.com/matrix-org/synapse/issues/11402))
-- Improve internal types in push code. ([\#11409](https://github.com/matrix-org/synapse/issues/11409))
-- Add type hints to storage classes. ([\#11411](https://github.com/matrix-org/synapse/issues/11411))
 - Add a check to ensure that users cannot start the Synapse master process when `worker_app` is set. ([\#11416](https://github.com/matrix-org/synapse/issues/11416))
-- Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls). ([\#11417](https://github.com/matrix-org/synapse/issues/11417))
-- Add type annotations to some of the configuration surrounding refresh tokens. ([\#11428](https://github.com/matrix-org/synapse/issues/11428))
-- Convert status codes to `HTTPStatus` in `synapse.rest.admin`. ([\#11452](https://github.com/matrix-org/synapse/issues/11452), [\#11455](https://github.com/matrix-org/synapse/issues/11455))
-- Improve type hints for `LruCache`. ([\#11453](https://github.com/matrix-org/synapse/issues/11453))
-- Remove unnecessary `json.dumps` from `tests.rest.admin`. ([\#11461](https://github.com/matrix-org/synapse/issues/11461))
-- Add missing type hints to `synapse.config` module. ([\#11465](https://github.com/matrix-org/synapse/issues/11465))
-- Update and clean up recently ported documentation pages. ([\#11466](https://github.com/matrix-org/synapse/issues/11466))
 - Add a note about postgres memory management and hugepages to postgres doc. ([\#11467](https://github.com/matrix-org/synapse/issues/11467))
-- Refactor `get_version_string` to fix-up types and duplicated code. ([\#11468](https://github.com/matrix-org/synapse/issues/11468))
-- Save the OpenID Connect session ID on login. ([\#11482](https://github.com/matrix-org/synapse/issues/11482))
+- Add missing type hints to `synapse.config` module. ([\#11465](https://github.com/matrix-org/synapse/issues/11465))
 - Add missing type hints to `synapse.federation`. ([\#11483](https://github.com/matrix-org/synapse/issues/11483))
-- Extend the `scripts-dev/sign_json` script to support signing events. ([\#11486](https://github.com/matrix-org/synapse/issues/11486))
 - Add type annotations to `tests.storage.test_appservice`. ([\#11488](https://github.com/matrix-org/synapse/issues/11488), [\#11492](https://github.com/matrix-org/synapse/issues/11492))
+- Add type annotations to some of the configuration surrounding refresh tokens. ([\#11428](https://github.com/matrix-org/synapse/issues/11428))
+- Add type hints to `synapse/tests/rest/admin`. ([\#11501](https://github.com/matrix-org/synapse/issues/11501))
+- Add type hints to storage classes. ([\#11411](https://github.com/matrix-org/synapse/issues/11411))
+- Add wiki pages to documentation website. ([\#11402](https://github.com/matrix-org/synapse/issues/11402))
 - Clean up `tests.storage.test_main` to remove use of legacy code. ([\#11493](https://github.com/matrix-org/synapse/issues/11493))
-- Refactor various parts of the `/sync` handler. ([\#11494](https://github.com/matrix-org/synapse/issues/11494), [\#11515](https://github.com/matrix-org/synapse/issues/11515))
 - Clean up `tests.test_visibility` to remove legacy code. ([\#11495](https://github.com/matrix-org/synapse/issues/11495))
+- Convert status codes to `HTTPStatus` in `synapse.rest.admin`. ([\#11452](https://github.com/matrix-org/synapse/issues/11452), [\#11455](https://github.com/matrix-org/synapse/issues/11455))
+- Extend the `scripts-dev/sign_json` script to support signing events. ([\#11486](https://github.com/matrix-org/synapse/issues/11486))
+- Improve internal types in push code. ([\#11409](https://github.com/matrix-org/synapse/issues/11409))
+- Improve type annotations in `synapse.module_api`. ([\#11029](https://github.com/matrix-org/synapse/issues/11029))
+- Improve type hints for `LruCache`. ([\#11453](https://github.com/matrix-org/synapse/issues/11453))
 - Preparation for database schema simplifications: disambiguate queries on `state_key`. ([\#11497](https://github.com/matrix-org/synapse/issues/11497))
-- Add type hints to `synapse/tests/rest/admin`. ([\#11501](https://github.com/matrix-org/synapse/issues/11501))
+- Refactor `backfilled` into specific behavior function arguments (`_persist_events_and_state_updates` and downstream calls). ([\#11417](https://github.com/matrix-org/synapse/issues/11417))
+- Refactor `get_version_string` to fix-up types and duplicated code. ([\#11468](https://github.com/matrix-org/synapse/issues/11468))
+- Refactor various parts of the `/sync` handler. ([\#11494](https://github.com/matrix-org/synapse/issues/11494), [\#11515](https://github.com/matrix-org/synapse/issues/11515))
+- Remove unnecessary `json.dumps` from `tests.rest.admin`. ([\#11461](https://github.com/matrix-org/synapse/issues/11461))
+- Save the OpenID Connect session ID on login. ([\#11482](https://github.com/matrix-org/synapse/issues/11482))
+- Update and clean up recently ported documentation pages. ([\#11466](https://github.com/matrix-org/synapse/issues/11466))
 
 
 Synapse 1.48.0 (2021-11-30)
-- 
cgit 1.5.1


From d6fb96e056f79de220d8d59429d89a61498e9af3 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Tue, 7 Dec 2021 16:51:53 +0000
Subject: Fix case in `wait_for_background_updates` where `self.store` does not
 exist (#11331)

Pull the DataStore from the HomeServer instance, which
always exists.
---
 changelog.d/11331.misc |  1 +
 tests/unittest.py      | 11 ++++-------
 2 files changed, 5 insertions(+), 7 deletions(-)
 create mode 100644 changelog.d/11331.misc

diff --git a/changelog.d/11331.misc b/changelog.d/11331.misc
new file mode 100644
index 0000000000..1ab3a6a975
--- /dev/null
+++ b/changelog.d/11331.misc
@@ -0,0 +1 @@
+A test helper (`wait_for_background_updates`) no longer depends on classes defining a `store` property.
diff --git a/tests/unittest.py b/tests/unittest.py
index eea0903f05..1431848367 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -331,16 +331,13 @@ class HomeserverTestCase(TestCase):
             time.sleep(0.01)
 
     def wait_for_background_updates(self) -> None:
-        """Block until all background database updates have completed.
-
-        Note that callers must ensure there's a store property created on the
-        testcase.
-        """
+        """Block until all background database updates have completed."""
+        store = self.hs.get_datastore()
         while not self.get_success(
-            self.store.db_pool.updates.has_completed_background_updates()
+            store.db_pool.updates.has_completed_background_updates()
         ):
             self.get_success(
-                self.store.db_pool.updates.do_next_background_update(False), by=0.1
+                store.db_pool.updates.do_next_background_update(False), by=0.1
             )
 
     def make_homeserver(self, reactor, clock):
-- 
cgit 1.5.1


From 8541809cb952ebf0da2a95dd93eccd5644dab49d Mon Sep 17 00:00:00 2001
From: Hubert Chathi <hubert@uhoreg.ca>
Date: Wed, 8 Dec 2021 05:01:38 -0500
Subject: Send and handle cross-signing messages using the stable prefix.
 (#10520)

---
 changelog.d/10520.misc                     | 1 +
 synapse/handlers/e2e_keys.py               | 8 ++++++--
 synapse/storage/databases/main/devices.py  | 4 +++-
 tests/federation/test_federation_sender.py | 5 +++--
 4 files changed, 13 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/10520.misc

diff --git a/changelog.d/10520.misc b/changelog.d/10520.misc
new file mode 100644
index 0000000000..a911e165da
--- /dev/null
+++ b/changelog.d/10520.misc
@@ -0,0 +1 @@
+Send and handle cross-signing messages using the stable prefix.
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 60c11e3d21..b2554bda04 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -65,8 +65,12 @@ class E2eKeysHandler:
         else:
             # Only register this edu handler on master as it requires writing
             # device updates to the db
-            #
-            # FIXME: switch to m.signing_key_update when MSC1756 is merged into the spec
+            federation_registry.register_edu_handler(
+                "m.signing_key_update",
+                self._edu_updater.incoming_signing_key_update,
+            )
+            # also handle the unstable version
+            # FIXME: remove this when enough servers have upgraded
             federation_registry.register_edu_handler(
                 "org.matrix.signing_key_update",
                 self._edu_updater.incoming_signing_key_update,
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index d5a4a661cd..838a2a6a3d 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -274,7 +274,9 @@ class DeviceWorkerStore(SQLBaseStore):
         # add the updated cross-signing keys to the results list
         for user_id, result in cross_signing_keys_by_user.items():
             result["user_id"] = user_id
-            # FIXME: switch to m.signing_key_update when MSC1756 is merged into the spec
+            results.append(("m.signing_key_update", result))
+            # also send the unstable version
+            # FIXME: remove this when enough servers have upgraded
             results.append(("org.matrix.signing_key_update", result))
 
         return now_stream_id, results
diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py
index b457dad6d2..b2376e2db9 100644
--- a/tests/federation/test_federation_sender.py
+++ b/tests/federation/test_federation_sender.py
@@ -266,7 +266,8 @@ class FederationSenderDevicesTestCases(HomeserverTestCase):
         )
 
         # expect signing key update edu
-        self.assertEqual(len(self.edus), 1)
+        self.assertEqual(len(self.edus), 2)
+        self.assertEqual(self.edus.pop(0)["edu_type"], "m.signing_key_update")
         self.assertEqual(self.edus.pop(0)["edu_type"], "org.matrix.signing_key_update")
 
         # sign the devices
@@ -491,7 +492,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase):
     ) -> None:
         """Check that the txn has an EDU with a signing key update."""
         edus = txn["edus"]
-        self.assertEqual(len(edus), 1)
+        self.assertEqual(len(edus), 2)
 
     def generate_and_upload_device_signing_key(
         self, user_id: str, device_id: str
-- 
cgit 1.5.1


From ff7cc17b5706ff3f386b15fda668511c0502ab9c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 8 Dec 2021 14:15:14 +0000
Subject: Improve log messages for stream ids (#11536)

Somehow I'd managed to get my database in a pickle with stream ids. These
changes were useful to debug.
---
 changelog.d/11536.misc                         | 1 +
 synapse/storage/databases/main/state_deltas.py | 4 +++-
 synapse/storage/util/id_generators.py          | 6 +++---
 3 files changed, 7 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/11536.misc

diff --git a/changelog.d/11536.misc b/changelog.d/11536.misc
new file mode 100644
index 0000000000..b9191c111b
--- /dev/null
+++ b/changelog.d/11536.misc
@@ -0,0 +1 @@
+Improvements to log messages around handling stream ids.
diff --git a/synapse/storage/databases/main/state_deltas.py b/synapse/storage/databases/main/state_deltas.py
index 7f3624b128..188afec332 100644
--- a/synapse/storage/databases/main/state_deltas.py
+++ b/synapse/storage/databases/main/state_deltas.py
@@ -56,7 +56,9 @@ class StateDeltasStore(SQLBaseStore):
         prev_stream_id = int(prev_stream_id)
 
         # check we're not going backwards
-        assert prev_stream_id <= max_stream_id
+        assert (
+            prev_stream_id <= max_stream_id
+        ), f"New stream id {max_stream_id} is smaller than prev stream id {prev_stream_id}"
 
         if not self._curr_state_delta_stream_cache.has_any_entity_changed(
             prev_stream_id
diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py
index 4ff3013908..b8112e1c05 100644
--- a/synapse/storage/util/id_generators.py
+++ b/synapse/storage/util/id_generators.py
@@ -74,8 +74,6 @@ class IdGenerator:
 def _load_current_id(
     db_conn: LoggingDatabaseConnection, table: str, column: str, step: int = 1
 ) -> int:
-    # debug logging for https://github.com/matrix-org/synapse/issues/7968
-    logger.info("initialising stream generator for %s(%s)", table, column)
     cur = db_conn.cursor(txn_name="_load_current_id")
     if step == 1:
         cur.execute("SELECT MAX(%s) FROM %s" % (column, table))
@@ -86,7 +84,9 @@ def _load_current_id(
     (val,) = result
     cur.close()
     current_id = int(val) if val else step
-    return (max if step > 0 else min)(current_id, step)
+    res = (max if step > 0 else min)(current_id, step)
+    logger.info("Initialising stream generator for %s(%s): %i", table, column, res)
+    return res
 
 
 class AbstractStreamIdTracker(metaclass=abc.ABCMeta):
-- 
cgit 1.5.1


From a39ccfc769c30f53ee200059d54e79f42d980705 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 14:47:43 +0000
Subject: Expand return type of get_appservice_user_id to allow returning a
 device ID to masquerade as

---
 synapse/api/auth.py | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 44883c6663..1a245c8b5c 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -155,7 +155,9 @@ class Auth:
 
             access_token = self.get_access_token_from_request(request)
 
-            user_id, app_service = await self._get_appservice_user_id(request)
+            user_id, _, app_service = await self._get_appservice_user_id_and_device_id(
+                request
+            )
             if user_id and app_service:
                 if ip_addr and self._track_appservice_user_ips:
                     await self.store.insert_client_ip(
@@ -274,33 +276,59 @@ class Auth:
                 403, "Application service has not registered this user (%s)" % user_id
             )
 
-    async def _get_appservice_user_id(
+    async def _get_appservice_user_id_and_device_id(
         self, request: Request
-    ) -> Tuple[Optional[str], Optional[ApplicationService]]:
+    ) -> Tuple[Optional[str], Optional[str], Optional[ApplicationService]]:
+        """
+        Given a request, reads the request parameters to determine:
+        - whether it's an application service that's making this request
+        - what user the application service should be treated as controlling
+          (the user_id URI parameter allows an application service to masquerade
+          any applicable user in its namespace)
+        - what device the application service should be treated as controlling
+          (the device_id[^1] URI parameter allows an application service to masquerade
+          as any device that exists for the relevant user)
+
+        [^1] Unstable and provided by MSC3202.
+             Must use `org.matrix.msc3202.device_id` in place of `device_id` for now.
+
+        Returns:
+            3-tuple of
+            (user ID?, device ID?, application service?)
+
+        Postconditions:
+        - If an application service is returned, so is a user ID
+        - A user ID is never returned without an application service
+        - A device ID is never returned without a user ID or an application service
+        - The returned application service, if present, is permitted to control the
+          returned user ID.
+        - The returned device ID, if present, has been checked to be a valid device ID
+          for the returned user ID.
+        """
         app_service = self.store.get_app_service_by_token(
             self.get_access_token_from_request(request)
         )
         if app_service is None:
-            return None, None
+            return None, None, None
 
         if app_service.ip_range_whitelist:
             ip_address = IPAddress(request.getClientIP())
             if ip_address not in app_service.ip_range_whitelist:
-                return None, None
+                return None, None, None
 
         # This will always be set by the time Twisted calls us.
         assert request.args is not None
 
         if b"user_id" not in request.args:
-            return app_service.sender, app_service
+            return app_service.sender, None, app_service
 
         user_id = request.args[b"user_id"][0].decode("utf8")
         await self.validate_appservice_can_control_user_id(app_service, user_id)
 
         if app_service.sender == user_id:
-            return app_service.sender, app_service
+            return app_service.sender, None, app_service
 
-        return user_id, app_service
+        return user_id, None, app_service
 
     async def get_user_by_access_token(
         self,
-- 
cgit 1.5.1


From be8814fcaa2b354a2563fc7fbcf93d98cdc03216 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 14:57:12 +0000
Subject: Expand get_user_by_req to support handling a device ID

---
 synapse/api/auth.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 1a245c8b5c..26d6007a04 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -155,9 +155,11 @@ class Auth:
 
             access_token = self.get_access_token_from_request(request)
 
-            user_id, _, app_service = await self._get_appservice_user_id_and_device_id(
-                request
-            )
+            (
+                user_id,
+                device_id,
+                app_service,
+            ) = await self._get_appservice_user_id_and_device_id(request)
             if user_id and app_service:
                 if ip_addr and self._track_appservice_user_ips:
                     await self.store.insert_client_ip(
@@ -165,16 +167,22 @@ class Auth:
                         access_token=access_token,
                         ip=ip_addr,
                         user_agent=user_agent,
-                        device_id="dummy-device",  # stubbed
+                        device_id="dummy-device"
+                        if device_id is None
+                        else device_id,  # stubbed
                     )
 
-                requester = create_requester(user_id, app_service=app_service)
+                requester = create_requester(
+                    user_id, app_service=app_service, device_id=device_id
+                )
 
                 request.requester = user_id
                 if user_id in self._force_tracing_for_users:
                     opentracing.force_tracing()
                 opentracing.set_tag("authenticated_entity", user_id)
                 opentracing.set_tag("user_id", user_id)
+                if device_id is not None:
+                    opentracing.set_tag("device_id", device_id)
                 opentracing.set_tag("appservice_id", app_service.id)
 
                 return requester
-- 
cgit 1.5.1


From 7ea5022be8b7891dd8a01dc85ab31501bc2c212c Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 15:00:13 +0000
Subject: Remove superfluous lines

---
 synapse/api/auth.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 26d6007a04..5f7e9163c6 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -333,8 +333,6 @@ class Auth:
         user_id = request.args[b"user_id"][0].decode("utf8")
         await self.validate_appservice_can_control_user_id(app_service, user_id)
 
-        if app_service.sender == user_id:
-            return app_service.sender, None, app_service
 
         return user_id, None, app_service
 
-- 
cgit 1.5.1


From 9551a3ed678e729062194e615074500f44b73ba1 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 15:01:12 +0000
Subject: Remove early return because we need more logic here

---
 synapse/api/auth.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 5f7e9163c6..d34d9f8abe 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -327,14 +327,15 @@ class Auth:
         # This will always be set by the time Twisted calls us.
         assert request.args is not None
 
-        if b"user_id" not in request.args:
-            return app_service.sender, None, app_service
-
-        user_id = request.args[b"user_id"][0].decode("utf8")
-        await self.validate_appservice_can_control_user_id(app_service, user_id)
-
+        if b"user_id" in request.args:
+            effective_user_id = request.args[b"user_id"][0].decode("utf8")
+            await self.validate_appservice_can_control_user_id(
+                app_service, effective_user_id
+            )
+        else:
+            effective_user_id = app_service.sender
 
-        return user_id, None, app_service
+        return effective_user_id, None, app_service
 
     async def get_user_by_access_token(
         self,
-- 
cgit 1.5.1


From 86ef692d5a36ed58a0f82a72b81f85853074ce90 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 15:11:31 +0000
Subject: Add get_device_opt which returns None instead of raising if it
 doesn't exist

---
 synapse/storage/databases/main/devices.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index 838a2a6a3d..afc516a978 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -112,6 +112,8 @@ class DeviceWorkerStore(SQLBaseStore):
             A dict containing the device information
         Raises:
             StoreError: if the device is not found
+        See also:
+            `get_device_opt` which returns None instead if the device is not found
         """
         return await self.db_pool.simple_select_one(
             table="devices",
@@ -120,6 +122,26 @@ class DeviceWorkerStore(SQLBaseStore):
             desc="get_device",
         )
 
+    async def get_device_opt(
+        self, user_id: str, device_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """Retrieve a device. Only returns devices that are not marked as
+        hidden.
+
+        Args:
+            user_id: The ID of the user which owns the device
+            device_id: The ID of the device to retrieve
+        Returns:
+            A dict containing the device information, or None if the device does not exist.
+        """
+        return await self.db_pool.simple_select_one(
+            table="devices",
+            keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
+            retcols=("user_id", "device_id", "display_name"),
+            desc="get_device",
+            allow_none=True,
+        )
+
     async def get_devices_by_user(self, user_id: str) -> Dict[str, Dict[str, str]]:
         """Retrieve all of a user's registered devices. Only returns devices
         that are not marked as hidden.
-- 
cgit 1.5.1


From d3b0be57f961f5860d71e3e89daf93b493d40bd7 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 15:12:32 +0000
Subject: Allow masquerading as a device by specifying the device_id URI
 parameter

---
 synapse/api/auth.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index d34d9f8abe..65369afd13 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -313,6 +313,8 @@ class Auth:
         - The returned device ID, if present, has been checked to be a valid device ID
           for the returned user ID.
         """
+        DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id"
+
         app_service = self.store.get_app_service_by_token(
             self.get_access_token_from_request(request)
         )
@@ -335,7 +337,22 @@ class Auth:
         else:
             effective_user_id = app_service.sender
 
-        return effective_user_id, None, app_service
+        effective_device_id: Optional[str] = None
+
+        if DEVICE_ID_ARG_NAME in request.args:
+            effective_device_id = request.args[DEVICE_ID_ARG_NAME][0].decode("utf8")
+            # We only just set this so it can't be None!
+            assert effective_device_id is not None
+            device_opt = await self.store.get_device_opt(
+                effective_user_id, effective_device_id
+            )
+            if device_opt is None:
+                raise AuthError(
+                    403,
+                    f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
+                )
+
+        return effective_user_id, effective_device_id, app_service
 
     async def get_user_by_access_token(
         self,
-- 
cgit 1.5.1


From 8a078ce3720eab6996e709cf15728bda11584ec4 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Wed, 8 Dec 2021 15:59:53 +0000
Subject: Newsfile

Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>
---
 changelog.d/11538.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/11538.feature

diff --git a/changelog.d/11538.feature b/changelog.d/11538.feature
new file mode 100644
index 0000000000..b6229e2b45
--- /dev/null
+++ b/changelog.d/11538.feature
@@ -0,0 +1 @@
+Add experimental support for MSC3202: allowing application services to masquerade as specific devices.
\ No newline at end of file
-- 
cgit 1.5.1


From cc2bbcd4dce1f53dc057390cc740e25ba5e7fe64 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 9 Dec 2021 12:29:08 +0000
Subject: Switch to the 400 M_EXCLUSIVE error code for non-existent device IDs

This is as a result of a discussion on the MSC
---
 synapse/api/auth.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 65369afd13..fd48735e1c 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -347,9 +347,12 @@ class Auth:
                 effective_user_id, effective_device_id
             )
             if device_opt is None:
+                # For now, use 400 M_EXCLUSIVE if the device doesn't exist.
+                # This is an open thread of discussion on MSC3202 as of 2021-12-09.
                 raise AuthError(
-                    403,
+                    400,
                     f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
+                    Codes.EXCLUSIVE,
                 )
 
         return effective_user_id, effective_device_id, app_service
-- 
cgit 1.5.1


From 7e398067f19037d41f98d98a2dce619ed10f891e Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 9 Dec 2021 12:48:36 +0000
Subject: Add a pair of tests for the ?device_id parameter for AS device
 masquerading

---
 tests/api/test_auth.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index 3aa9ba3c43..fe98c6d4dc 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -210,6 +210,67 @@ class AuthTestCase(unittest.HomeserverTestCase):
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         self.get_failure(self.auth.get_user_by_req(request), AuthError)
 
+    def test_get_user_by_req_appservice_valid_token_valid_device_id(self):
+        """
+        Tests that when an application service passes the device_id URL parameter
+        with the ID of a valid device for the user in question,
+        the requester instance tracks that device ID.
+        """
+        masquerading_user_id = b"@doppelganger:matrix.org"
+        masquerading_device_id = b"DOPPELDEVICE"
+        app_service = Mock(
+            token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None
+        )
+        app_service.is_interested_in_user = Mock(return_value=True)
+        self.store.get_app_service_by_token = Mock(return_value=app_service)
+        # This just needs to return a truth-y value.
+        self.store.get_user_by_id = simple_async_mock({"is_guest": False})
+        self.store.get_user_by_access_token = simple_async_mock(None)
+        # This also needs to just return a truth-y value
+        self.store.get_device_opt = simple_async_mock({"hidden": False})
+
+        request = Mock(args={})
+        request.getClientIP.return_value = "127.0.0.1"
+        request.args[b"access_token"] = [self.test_token]
+        request.args[b"user_id"] = [masquerading_user_id]
+        request.args[b"org.matrix.msc3202.device_id"] = [masquerading_device_id]
+        request.requestHeaders.getRawHeaders = mock_getRawHeaders()
+        requester = self.get_success(self.auth.get_user_by_req(request))
+        self.assertEquals(
+            requester.user.to_string(), masquerading_user_id.decode("utf8")
+        )
+        self.assertEquals(requester.device_id, masquerading_device_id.decode("utf8"))
+
+    def test_get_user_by_req_appservice_valid_token_invalid_device_id(self):
+        """
+        Tests that when an application service passes the device_id URL parameter
+        with an ID that is not a valid device ID for the user in question,
+        the request fails with the appropriate error code.
+        """
+        masquerading_user_id = b"@doppelganger:matrix.org"
+        masquerading_device_id = b"NOT_A_REAL_DEVICE_ID"
+        app_service = Mock(
+            token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None
+        )
+        app_service.is_interested_in_user = Mock(return_value=True)
+        self.store.get_app_service_by_token = Mock(return_value=app_service)
+        # This just needs to return a truth-y value.
+        self.store.get_user_by_id = simple_async_mock({"is_guest": False})
+        self.store.get_user_by_access_token = simple_async_mock(None)
+        # This also needs to just return a truth-y value
+        self.store.get_device_opt = simple_async_mock(None)
+
+        request = Mock(args={})
+        request.getClientIP.return_value = "127.0.0.1"
+        request.args[b"access_token"] = [self.test_token]
+        request.args[b"user_id"] = [masquerading_user_id]
+        request.args[b"org.matrix.msc3202.device_id"] = [masquerading_device_id]
+        request.requestHeaders.getRawHeaders = mock_getRawHeaders()
+
+        failure = self.get_failure(self.auth.get_user_by_req(request), AuthError)
+        self.assertEquals(failure.value.code, 400)
+        self.assertEquals(failure.value.errcode, Codes.EXCLUSIVE)
+
     def test_get_user_from_macaroon(self):
         self.store.get_user_by_access_token = simple_async_mock(
             TokenLookupResult(user_id="@baldrick:matrix.org", device_id="device")
-- 
cgit 1.5.1


From ae968eaa936513455b77768c92dd22ef0bf847a7 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 9 Dec 2021 13:10:18 +0000
Subject: Add an experimental flag to control device masquerading

---
 synapse/api/auth.py            | 5 ++++-
 synapse/config/experimental.py | 5 +++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index fd48735e1c..0bb4b77f84 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -339,7 +339,10 @@ class Auth:
 
         effective_device_id: Optional[str] = None
 
-        if DEVICE_ID_ARG_NAME in request.args:
+        if (
+            self.hs.config.experimental.msc3202_device_masquerading_enabled
+            and DEVICE_ID_ARG_NAME in request.args
+        ):
             effective_device_id = request.args[DEVICE_ID_ARG_NAME][0].decode("utf8")
             # We only just set this so it can't be None!
             assert effective_device_id is not None
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index d78a15097c..678c78d565 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -49,3 +49,8 @@ class ExperimentalConfig(Config):
 
         # MSC3030 (Jump to date API endpoint)
         self.msc3030_enabled: bool = experimental.get("msc3030_enabled", False)
+
+        # The portion of MSC3202 which is related to device masquerading.
+        self.msc3202_device_masquerading_enabled: bool = experimental.get(
+            "msc3202_device_masquerading", False
+        )
-- 
cgit 1.5.1


From 11e2192b3224816578f054a325b2c16ed4a70580 Mon Sep 17 00:00:00 2001
From: "Olivier Wilkinson (reivilibre)" <oliverw@matrix.org>
Date: Thu, 9 Dec 2021 13:13:32 +0000
Subject: Update tests to enable experimental features

---
 tests/api/test_auth.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py
index fe98c6d4dc..3c9ca52922 100644
--- a/tests/api/test_auth.py
+++ b/tests/api/test_auth.py
@@ -31,6 +31,7 @@ from synapse.types import Requester
 
 from tests import unittest
 from tests.test_utils import simple_async_mock
+from tests.unittest import override_config
 from tests.utils import mock_getRawHeaders
 
 
@@ -210,6 +211,7 @@ class AuthTestCase(unittest.HomeserverTestCase):
         request.requestHeaders.getRawHeaders = mock_getRawHeaders()
         self.get_failure(self.auth.get_user_by_req(request), AuthError)
 
+    @override_config({"experimental_features": {"msc3202_device_masquerading": True}})
     def test_get_user_by_req_appservice_valid_token_valid_device_id(self):
         """
         Tests that when an application service passes the device_id URL parameter
@@ -241,6 +243,7 @@ class AuthTestCase(unittest.HomeserverTestCase):
         )
         self.assertEquals(requester.device_id, masquerading_device_id.decode("utf8"))
 
+    @override_config({"experimental_features": {"msc3202_device_masquerading": True}})
     def test_get_user_by_req_appservice_valid_token_invalid_device_id(self):
         """
         Tests that when an application service passes the device_id URL parameter
-- 
cgit 1.5.1