From c75c91f54f9a485e4e27bf57d5bce22d6c1e083d Mon Sep 17 00:00:00 2001 From: Rory& Date: Sun, 12 Oct 2025 21:03:39 +0200 Subject: Update synapse patches --- .../patches/0001-Fast-auth-links.patch | 12 +- ...h-logging-to-room-summary-over-federation.patch | 77 ++++++++ ...-ignore-rejected-events-in-delayed_events.patch | 31 ---- ...h-logging-to-room-summary-over-federation.patch | 77 -------- ...-entire-room-if-accessibility-check-fails.patch | 28 +++ ...-entire-room-if-accessibility-check-fails.patch | 28 --- .../0004-Log-policy-server-rejected-events.patch | 30 +++ .../0005-Log-policy-server-rejected-events.patch | 30 --- ...-Use-parse_boolean-for-unredacted-content.patch | 29 +++ .../0006-Expose-tombstone-in-room-admin-api.patch | 114 ++++++++++++ ...-Use-parse_boolean-for-unredacted-content.patch | 29 --- .../0007-Expose-tombstone-in-room-admin-api.patch | 114 ------------ ...recheck-messages-pagination-data-if-a-bac.patch | 204 +++++++++++++++++++++ ...nation-with-large-gaps-of-rejected-events.patch | 50 +++++ ...recheck-messages-pagination-data-if-a-bac.patch | 204 --------------------- .../patches/0009-Fix-nix-flake.patch | 189 +++++++++++++++++++ ...nation-with-large-gaps-of-rejected-events.patch | 50 ----- .../0010-Fix-gitignore-to-ignore-.venv.patch | 24 +++ .../patches/0010-Fix-nix-flake.patch | 189 ------------------- .../patches/0011-Devenv-use-postgres-17.patch | 24 +++ .../0011-Fix-gitignore-to-ignore-.venv.patch | 24 --- .../patches/0012-Devenv-use-postgres-17.patch | 24 --- ...2-RequestRatelimiter-expose-can_do_action.patch | 95 ++++++++++ .../0013-Add-bulk-send-events-endpoint.patch | 187 +++++++++++++++++++ ...3-RequestRatelimiter-expose-can_do_action.patch | 95 ---------- .../0014-Add-bulk-send-events-endpoint.patch | 187 ------------------- .../0014-Temporarily-disable-go-in-flake.patch | 25 +++ .../patches/0015-Clarify-asserts.patch | 25 +++ .../0015-Temporarily-disable-go-in-flake.patch | 25 --- .../patches/0016-Clarify-asserts.patch | 25 --- ...16-Try-to-optimise-bulk-sending-some-more.patch | 117 ++++++++++++ .../0017-Further-optimisation-attempts.patch | 90 +++++++++ ...17-Try-to-optimise-bulk-sending-some-more.patch | 117 ------------ .../0018-Further-optimisation-attempts.patch | 90 --------- ...and_send_new_client_events-for-bulk-sendi.patch | 144 +++++++++++++++ ...and_send_new_client_events-for-bulk-sendi.patch | 144 --------------- 36 files changed, 1458 insertions(+), 1489 deletions(-) create mode 100644 packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch create mode 100644 packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch create mode 100644 packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch create mode 100644 packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch create mode 100644 packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch create mode 100644 packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch create mode 100644 packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch create mode 100644 packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch create mode 100644 packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch create mode 100644 packages/overlays/matrix-synapse/patches/0011-Devenv-use-postgres-17.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch create mode 100644 packages/overlays/matrix-synapse/patches/0012-RequestRatelimiter-expose-can_do_action.patch create mode 100644 packages/overlays/matrix-synapse/patches/0013-Add-bulk-send-events-endpoint.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch create mode 100644 packages/overlays/matrix-synapse/patches/0014-Temporarily-disable-go-in-flake.patch create mode 100644 packages/overlays/matrix-synapse/patches/0015-Clarify-asserts.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch create mode 100644 packages/overlays/matrix-synapse/patches/0016-Try-to-optimise-bulk-sending-some-more.patch create mode 100644 packages/overlays/matrix-synapse/patches/0017-Further-optimisation-attempts.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch create mode 100644 packages/overlays/matrix-synapse/patches/0018-Use-create_and_send_new_client_events-for-bulk-sendi.patch delete mode 100644 packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch diff --git a/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch b/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch index 2b971cf..edbd5da 100644 --- a/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch +++ b/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch @@ -1,7 +1,7 @@ -From a4c542d11e60728426b85b8a59a7c02062930ede Mon Sep 17 00:00:00 2001 +From ea2e053dd1b578e49ad066992388143db629c390 Mon Sep 17 00:00:00 2001 From: Rory& Date: Tue, 22 Jul 2025 05:07:01 +0200 -Subject: [PATCH 01/19] Fast auth links +Subject: [PATCH 01/18] Fast auth links --- synapse/storage/database.py | 43 +++++++++++++++++++ @@ -9,10 +9,10 @@ Subject: [PATCH 01/19] Fast auth links 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py -index f7aec16c96..fdc7572e97 100644 +index aae029f910..35948416a4 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py -@@ -2615,6 +2615,49 @@ class DatabasePool: +@@ -2609,6 +2609,49 @@ class DatabasePool: return txn.fetchall() @@ -63,7 +63,7 @@ index f7aec16c96..fdc7572e97 100644 def make_in_list_sql_clause( database_engine: BaseDatabaseEngine, diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py -index 26a91109df..2244e47cf5 100644 +index 5c9bd2e848..bf4d29728f 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -53,6 +53,7 @@ from synapse.storage.database import ( @@ -96,5 +96,5 @@ index 26a91109df..2244e47cf5 100644 txn.execute(sql % (clause,), args) -- -2.50.1 +2.51.0 diff --git a/packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch b/packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch new file mode 100644 index 0000000..39cf8d6 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0002-Add-too-much-logging-to-room-summary-over-federation.patch @@ -0,0 +1,77 @@ +From 6a68a2a94a714bf4eec4dcfabaea3be64171e7af Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Wed, 23 Apr 2025 17:53:52 +0200 +Subject: [PATCH 02/18] Add too much logging to room summary over federation + +Signed-off-by: Rory& +--- + synapse/handlers/room_summary.py | 40 ++++++++++++++++++++++++++++---- + 1 file changed, 36 insertions(+), 4 deletions(-) + +diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py +index 838fee6a30..e99e43b3bf 100644 +--- a/synapse/handlers/room_summary.py ++++ b/synapse/handlers/room_summary.py +@@ -717,23 +717,55 @@ class RoomSummaryHandler: + """ + # The API doesn't return the room version so assume that a + # join rule of knock is valid. ++ join_rule = room.get("join_rule") ++ world_readable = room.get("world_readable") ++ ++ logger.warning( ++ "[EMMA] Checking if room %s is accessible to %s: join_rule=%s, world_readable=%s", ++ room_id, requester, join_rule, world_readable ++ ) ++ + if ( +- room.get("join_rule", JoinRules.PUBLIC) +- in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) +- or room.get("world_readable") is True ++ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) ++ or world_readable is True + ): + return True +- elif not requester: ++ else: ++ logger.warning( ++ "[EMMA] Room %s is not accessible to %s: join_rule=%s, world_readable=%s, join_rule result=%s, world_readable result=%s", ++ room_id, requester, join_rule, world_readable, ++ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED), ++ world_readable is True ++ ) ++ ++ if not requester: ++ logger.warning( ++ "[EMMA] No requester, so room %s is not accessible", ++ room_id ++ ) + return False ++ + + # Check if the user is a member of any of the allowed rooms from the response. + allowed_rooms = room.get("allowed_room_ids") ++ logger.warning( ++ "[EMMA] Checking if room %s is in allowed rooms for %s: join_rule=%s, allowed_rooms=%s", ++ requester, ++ room_id, ++ join_rule, ++ allowed_rooms ++ ) + if allowed_rooms and isinstance(allowed_rooms, list): + if await self._event_auth_handler.is_user_in_rooms( + allowed_rooms, requester + ): + return True + ++ logger.warning( ++ "[EMMA] Checking if room %s is accessble to %s via local state", ++ room_id, ++ requester ++ ) + # Finally, check locally if we can access the room. The user might + # already be in the room (if it was a child room), or there might be a + # pending invite, etc. +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch b/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch deleted file mode 100644 index b44a6ea..0000000 --- a/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 46e8ebbb8253470489be0c5a9a481eea86507f79 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sun, 20 Apr 2025 00:30:29 +0200 -Subject: [PATCH 02/19] Hotfix: ignore rejected events in delayed_events - ---- - synapse/handlers/delayed_events.py | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py -index ce13dcc737..588cc35632 100644 ---- a/synapse/handlers/delayed_events.py -+++ b/synapse/handlers/delayed_events.py -@@ -216,8 +216,13 @@ class DelayedEventsHandler: - ) - - event = await self._store.get_event( -- delta.event_id, check_room_id=delta.room_id -+ delta.event_id, check_room_id=delta.room_id, allow_rejected=True, allow_none=True - ) -+ -+ if event is None or event.rejected_reason is not None: -+ # This event has been rejected, so we don't want to cancel any delayed events for it. -+ continue -+ - sender = UserID.from_string(event.sender) - - next_send_ts = await self._store.cancel_delayed_state_events( --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch b/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch deleted file mode 100644 index 88b0db4..0000000 --- a/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch +++ /dev/null @@ -1,77 +0,0 @@ -From 370ea67df812fc3174bf480ebe12068b14922d90 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Wed, 23 Apr 2025 17:53:52 +0200 -Subject: [PATCH 03/19] Add too much logging to room summary over federation - -Signed-off-by: Rory& ---- - synapse/handlers/room_summary.py | 40 ++++++++++++++++++++++++++++---- - 1 file changed, 36 insertions(+), 4 deletions(-) - -diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py -index 838fee6a30..e99e43b3bf 100644 ---- a/synapse/handlers/room_summary.py -+++ b/synapse/handlers/room_summary.py -@@ -717,23 +717,55 @@ class RoomSummaryHandler: - """ - # The API doesn't return the room version so assume that a - # join rule of knock is valid. -+ join_rule = room.get("join_rule") -+ world_readable = room.get("world_readable") -+ -+ logger.warning( -+ "[EMMA] Checking if room %s is accessible to %s: join_rule=%s, world_readable=%s", -+ room_id, requester, join_rule, world_readable -+ ) -+ - if ( -- room.get("join_rule", JoinRules.PUBLIC) -- in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) -- or room.get("world_readable") is True -+ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED) -+ or world_readable is True - ): - return True -- elif not requester: -+ else: -+ logger.warning( -+ "[EMMA] Room %s is not accessible to %s: join_rule=%s, world_readable=%s, join_rule result=%s, world_readable result=%s", -+ room_id, requester, join_rule, world_readable, -+ join_rule in (JoinRules.PUBLIC, JoinRules.KNOCK, JoinRules.KNOCK_RESTRICTED), -+ world_readable is True -+ ) -+ -+ if not requester: -+ logger.warning( -+ "[EMMA] No requester, so room %s is not accessible", -+ room_id -+ ) - return False -+ - - # Check if the user is a member of any of the allowed rooms from the response. - allowed_rooms = room.get("allowed_room_ids") -+ logger.warning( -+ "[EMMA] Checking if room %s is in allowed rooms for %s: join_rule=%s, allowed_rooms=%s", -+ requester, -+ room_id, -+ join_rule, -+ allowed_rooms -+ ) - if allowed_rooms and isinstance(allowed_rooms, list): - if await self._event_auth_handler.is_user_in_rooms( - allowed_rooms, requester - ): - return True - -+ logger.warning( -+ "[EMMA] Checking if room %s is accessble to %s via local state", -+ room_id, -+ requester -+ ) - # Finally, check locally if we can access the room. The user might - # already be in the room (if it was a child room), or there might be a - # pending invite, etc. --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch b/packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch new file mode 100644 index 0000000..29380ca --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0003-Log-entire-room-if-accessibility-check-fails.patch @@ -0,0 +1,28 @@ +From 4fbb4cf40da97c4d65b0b6a1c8f123fb84582395 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Wed, 23 Apr 2025 18:24:57 +0200 +Subject: [PATCH 03/18] Log entire room if accessibility check fails + +Signed-off-by: Rory& +--- + synapse/handlers/room_summary.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py +index e99e43b3bf..daf543bd43 100644 +--- a/synapse/handlers/room_summary.py ++++ b/synapse/handlers/room_summary.py +@@ -933,6 +933,10 @@ class RoomSummaryHandler: + if not room_entry or not await self._is_remote_room_accessible( + requester, room_entry.room_id, room_entry.room + ): ++ logger.warning( ++ "[Emma] Room entry contents: %s", ++ room_entry.room if room_entry else None ++ ) + raise NotFoundError("Room not found or is not accessible") + + room = dict(room_entry.room) +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch b/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch deleted file mode 100644 index 4741c42..0000000 --- a/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch +++ /dev/null @@ -1,28 +0,0 @@ -From 5c0e74cb20848a62e9921e697249d0cb1d6df035 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Wed, 23 Apr 2025 18:24:57 +0200 -Subject: [PATCH 04/19] Log entire room if accessibility check fails - -Signed-off-by: Rory& ---- - synapse/handlers/room_summary.py | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py -index e99e43b3bf..daf543bd43 100644 ---- a/synapse/handlers/room_summary.py -+++ b/synapse/handlers/room_summary.py -@@ -933,6 +933,10 @@ class RoomSummaryHandler: - if not room_entry or not await self._is_remote_room_accessible( - requester, room_entry.room_id, room_entry.room - ): -+ logger.warning( -+ "[Emma] Room entry contents: %s", -+ room_entry.room if room_entry else None -+ ) - raise NotFoundError("Room not found or is not accessible") - - room = dict(room_entry.room) --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch new file mode 100644 index 0000000..2f7dd0c --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0004-Log-policy-server-rejected-events.patch @@ -0,0 +1,30 @@ +From ca9bb9a0ec8f66751dd53be9c5933621a72eddae Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 27 May 2025 05:21:46 +0200 +Subject: [PATCH 04/18] Log policy server rejected events + +--- + synapse/handlers/room_policy.py | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py +index 170c477d6f..df5585e4ee 100644 +--- a/synapse/handlers/room_policy.py ++++ b/synapse/handlers/room_policy.py +@@ -87,6 +87,13 @@ class RoomPolicyHandler: + policy_server, event + ) + if recommendation != RECOMMENDATION_OK: ++ logger.info( ++ "[POLICY] Policy server %s recommended not to allow event %s in room %s: %s", ++ policy_server, ++ event.event_id, ++ event.room_id, ++ recommendation, ++ ) + return False + + return True # default allow +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch deleted file mode 100644 index 9158bff..0000000 --- a/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 34afe28e5c91d72f449cd453944a7763b6cc5c6b Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Tue, 27 May 2025 05:21:46 +0200 -Subject: [PATCH 05/19] Log policy server rejected events - ---- - synapse/handlers/room_policy.py | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py -index 170c477d6f..df5585e4ee 100644 ---- a/synapse/handlers/room_policy.py -+++ b/synapse/handlers/room_policy.py -@@ -87,6 +87,13 @@ class RoomPolicyHandler: - policy_server, event - ) - if recommendation != RECOMMENDATION_OK: -+ logger.info( -+ "[POLICY] Policy server %s recommended not to allow event %s in room %s: %s", -+ policy_server, -+ event.event_id, -+ event.room_id, -+ recommendation, -+ ) - return False - - return True # default allow --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch b/packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch new file mode 100644 index 0000000..01c8bc5 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0005-Use-parse_boolean-for-unredacted-content.patch @@ -0,0 +1,29 @@ +From 0ea324778e957164fc74b17e504e71d1018119f6 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 27 May 2025 06:14:26 +0200 +Subject: [PATCH 05/18] Use parse_boolean for unredacted content + +--- + synapse/rest/client/room.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index 64deae7650..eadfd721b3 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -930,10 +930,9 @@ class RoomEventServlet(RestServlet): + requester = await self.auth.get_user_by_req(request, allow_guest=True) + + include_unredacted_content = self.msc2815_enabled and ( +- parse_string( ++ parse_boolean( + request, +- "fi.mau.msc2815.include_unredacted_content", +- allowed_values=("true", "false"), ++ "fi.mau.msc2815.include_unredacted_content" + ) + == "true" + ) +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch b/packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch new file mode 100644 index 0000000..63832a7 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0006-Expose-tombstone-in-room-admin-api.patch @@ -0,0 +1,114 @@ +From e89b8e5fcf731e87ca28c5f2f65b8e33f39c9df5 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Tue, 27 May 2025 06:37:52 +0200 +Subject: [PATCH 06/18] Expose tombstone in room admin api + +--- + synapse/rest/admin/rooms.py | 5 ++++ + synapse/rest/client/room.py | 1 - + synapse/storage/databases/main/room.py | 36 +++++++++++++++++++++++++- + 3 files changed, 40 insertions(+), 2 deletions(-) + +diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py +index 5bed89c2c4..87a851bf64 100644 +--- a/synapse/rest/admin/rooms.py ++++ b/synapse/rest/admin/rooms.py +@@ -251,6 +251,10 @@ class ListRoomRestServlet(RestServlet): + direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS) + reverse_order = True if direction == Direction.BACKWARDS else False + ++ emma_include_tombstone = parse_boolean( ++ request, "emma_include_tombstone", default=False ++ ) ++ + # Return list of rooms according to parameters + rooms, total_rooms = await self.store.get_rooms_paginate( + start, +@@ -260,6 +264,7 @@ class ListRoomRestServlet(RestServlet): + search_term, + public_rooms, + empty_rooms, ++ emma_include_tombstone = emma_include_tombstone + ) + + response = { +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index eadfd721b3..e7f41f0079 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -934,7 +934,6 @@ class RoomEventServlet(RestServlet): + request, + "fi.mau.msc2815.include_unredacted_content" + ) +- == "true" + ) + if include_unredacted_content and not await self.auth.is_server_admin( + requester +diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py +index 9f03c084a5..b084a3f076 100644 +--- a/synapse/storage/databases/main/room.py ++++ b/synapse/storage/databases/main/room.py +@@ -611,6 +611,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): + search_term: Optional[str], + public_rooms: Optional[bool], + empty_rooms: Optional[bool], ++ emma_include_tombstone: bool = False, + ) -> Tuple[List[Dict[str, Any]], int]: + """Function to retrieve a paginated list of rooms as json. + +@@ -630,6 +631,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): + If true, empty rooms are queried. + if false, empty rooms are excluded from the query. When it is + none (the default), both empty rooms and none-empty rooms are queried. ++ emma_include_tombstone: If true, include tombstone events in the results. + Returns: + A list of room dicts and an integer representing the total number of + rooms that exist given this query +@@ -798,11 +800,43 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): + room_count = cast(Tuple[int], txn.fetchone()) + return rooms, room_count[0] + +- return await self.db_pool.runInteraction( ++ result = await self.db_pool.runInteraction( + "get_rooms_paginate", + _get_rooms_paginate_txn, + ) + ++ if emma_include_tombstone: ++ room_id_sql, room_id_args = make_in_list_sql_clause( ++ self.database_engine, "cse.room_id", [r["room_id"] for r in result[0]] ++ ) ++ ++ tombstone_sql = """ ++ SELECT cse.room_id, cse.event_id, ej.json ++ FROM current_state_events cse ++ JOIN event_json ej USING (event_id) ++ WHERE cse.type = 'm.room.tombstone' ++ AND {room_id_sql} ++ """.format( ++ room_id_sql=room_id_sql ++ ) ++ ++ def _get_tombstones_txn( ++ txn: LoggingTransaction, ++ ) -> Dict[str, JsonDict]: ++ txn.execute(tombstone_sql, room_id_args) ++ for room_id, event_id, json in txn: ++ for result_room in result[0]: ++ if result_room["room_id"] == room_id: ++ result_room["gay.rory.synapse_admin_extensions.tombstone"] = db_to_json(json) ++ break ++ return result[0], result[1] ++ ++ result = await self.db_pool.runInteraction( ++ "get_rooms_tombstones", _get_tombstones_txn, ++ ) ++ ++ return result ++ + @cached(max_entries=10000) + async def get_ratelimit_for_user(self, user_id: str) -> Optional[RatelimitOverride]: + """Check if there are any overrides for ratelimiting for the given user +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch b/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch deleted file mode 100644 index a2f52f7..0000000 --- a/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch +++ /dev/null @@ -1,29 +0,0 @@ -From b8ac66b50a38d38a34e4d5f003d39674eaffe77f Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Tue, 27 May 2025 06:14:26 +0200 -Subject: [PATCH 06/19] Use parse_boolean for unredacted content - ---- - synapse/rest/client/room.py | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index 64deae7650..eadfd721b3 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -930,10 +930,9 @@ class RoomEventServlet(RestServlet): - requester = await self.auth.get_user_by_req(request, allow_guest=True) - - include_unredacted_content = self.msc2815_enabled and ( -- parse_string( -+ parse_boolean( - request, -- "fi.mau.msc2815.include_unredacted_content", -- allowed_values=("true", "false"), -+ "fi.mau.msc2815.include_unredacted_content" - ) - == "true" - ) --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch b/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch deleted file mode 100644 index 74195eb..0000000 --- a/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch +++ /dev/null @@ -1,114 +0,0 @@ -From 09eebe99f37c523fe1de000fbc7fb3dc692faee7 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Tue, 27 May 2025 06:37:52 +0200 -Subject: [PATCH 07/19] Expose tombstone in room admin api - ---- - synapse/rest/admin/rooms.py | 5 ++++ - synapse/rest/client/room.py | 1 - - synapse/storage/databases/main/room.py | 36 +++++++++++++++++++++++++- - 3 files changed, 40 insertions(+), 2 deletions(-) - -diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py -index 5bed89c2c4..87a851bf64 100644 ---- a/synapse/rest/admin/rooms.py -+++ b/synapse/rest/admin/rooms.py -@@ -251,6 +251,10 @@ class ListRoomRestServlet(RestServlet): - direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS) - reverse_order = True if direction == Direction.BACKWARDS else False - -+ emma_include_tombstone = parse_boolean( -+ request, "emma_include_tombstone", default=False -+ ) -+ - # Return list of rooms according to parameters - rooms, total_rooms = await self.store.get_rooms_paginate( - start, -@@ -260,6 +264,7 @@ class ListRoomRestServlet(RestServlet): - search_term, - public_rooms, - empty_rooms, -+ emma_include_tombstone = emma_include_tombstone - ) - - response = { -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index eadfd721b3..e7f41f0079 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -934,7 +934,6 @@ class RoomEventServlet(RestServlet): - request, - "fi.mau.msc2815.include_unredacted_content" - ) -- == "true" - ) - if include_unredacted_content and not await self.auth.is_server_admin( - requester -diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py -index 6ffc3aed34..0cb87a811c 100644 ---- a/synapse/storage/databases/main/room.py -+++ b/synapse/storage/databases/main/room.py -@@ -611,6 +611,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): - search_term: Optional[str], - public_rooms: Optional[bool], - empty_rooms: Optional[bool], -+ emma_include_tombstone: bool = False, - ) -> Tuple[List[Dict[str, Any]], int]: - """Function to retrieve a paginated list of rooms as json. - -@@ -630,6 +631,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): - If true, empty rooms are queried. - if false, empty rooms are excluded from the query. When it is - none (the default), both empty rooms and none-empty rooms are queried. -+ emma_include_tombstone: If true, include tombstone events in the results. - Returns: - A list of room dicts and an integer representing the total number of - rooms that exist given this query -@@ -798,11 +800,43 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): - room_count = cast(Tuple[int], txn.fetchone()) - return rooms, room_count[0] - -- return await self.db_pool.runInteraction( -+ result = await self.db_pool.runInteraction( - "get_rooms_paginate", - _get_rooms_paginate_txn, - ) - -+ if emma_include_tombstone: -+ room_id_sql, room_id_args = make_in_list_sql_clause( -+ self.database_engine, "cse.room_id", [r["room_id"] for r in result[0]] -+ ) -+ -+ tombstone_sql = """ -+ SELECT cse.room_id, cse.event_id, ej.json -+ FROM current_state_events cse -+ JOIN event_json ej USING (event_id) -+ WHERE cse.type = 'm.room.tombstone' -+ AND {room_id_sql} -+ """.format( -+ room_id_sql=room_id_sql -+ ) -+ -+ def _get_tombstones_txn( -+ txn: LoggingTransaction, -+ ) -> Dict[str, JsonDict]: -+ txn.execute(tombstone_sql, room_id_args) -+ for room_id, event_id, json in txn: -+ for result_room in result[0]: -+ if result_room["room_id"] == room_id: -+ result_room["gay.rory.synapse_admin_extensions.tombstone"] = db_to_json(json) -+ break -+ return result[0], result[1] -+ -+ result = await self.db_pool.runInteraction( -+ "get_rooms_tombstones", _get_tombstones_txn, -+ ) -+ -+ return result -+ - @cached(max_entries=10000) - async def get_ratelimit_for_user(self, user_id: str) -> Optional[RatelimitOverride]: - """Check if there are any overrides for ratelimiting for the given user --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch b/packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch new file mode 100644 index 0000000..b59e1ee --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0007-fix-Always-recheck-messages-pagination-data-if-a-bac.patch @@ -0,0 +1,204 @@ +From 79676cabd12b566c6e80549f1a0b423eacb230dc Mon Sep 17 00:00:00 2001 +From: Jason Little +Date: Wed, 30 Apr 2025 09:29:42 -0500 +Subject: [PATCH 07/18] fix: Always recheck `/messages` pagination data if a + backfill might have been needed (#28) + +--- + synapse/handlers/federation.py | 35 +++++++++++++-------------------- + synapse/handlers/pagination.py | 36 +++++++++++++++++++--------------- + 2 files changed, 34 insertions(+), 37 deletions(-) + +diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py +index 34aae7ef3c..b3226de999 100644 +--- a/synapse/handlers/federation.py ++++ b/synapse/handlers/federation.py +@@ -197,7 +197,7 @@ class FederationHandler: + @tag_args + async def maybe_backfill( + self, room_id: str, current_depth: int, limit: int, record_time: bool = True +- ) -> bool: ++ ) -> None: + """Checks the database to see if we should backfill before paginating, + and if so do. + +@@ -211,8 +211,6 @@ class FederationHandler: + should back paginate. + record_time: Whether to record the time it takes to backfill. + +- Returns: +- True if we actually tried to backfill something, otherwise False. + """ + # Starting the processing time here so we can include the room backfill + # linearizer lock queue in the timing +@@ -238,7 +236,7 @@ class FederationHandler: + limit: int, + *, + processing_start_time: Optional[int], +- ) -> bool: ++ ) -> None: + """ + Checks whether the `current_depth` is at or approaching any backfill + points in the room and if so, will backfill. We only care about +@@ -312,7 +310,7 @@ class FederationHandler: + limit=1, + ) + if not have_later_backfill_points: +- return False ++ return None + + logger.debug( + "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points." +@@ -333,15 +331,15 @@ class FederationHandler: + ) + # We return `False` because we're backfilling in the background and there is + # no new events immediately for the caller to know about yet. +- return False ++ return None + + # Even after recursing with `MAX_DEPTH`, we didn't find any + # backward extremities to backfill from. + if not sorted_backfill_points: + logger.debug( +- "_maybe_backfill_inner: Not backfilling as no backward extremeties found." ++ "_maybe_backfill_inner: Not backfilling as no backward extremities found." + ) +- return False ++ return None + + # If we're approaching an extremity we trigger a backfill, otherwise we + # no-op. +@@ -360,7 +358,7 @@ class FederationHandler: + current_depth, + limit, + ) +- return False ++ return None + + # For performance's sake, we only want to paginate from a particular extremity + # if we can actually see the events we'll get. Otherwise, we'd just spend a lot +@@ -428,7 +426,7 @@ class FederationHandler: + logger.debug( + "_maybe_backfill_inner: found no extremities which would be visible" + ) +- return False ++ return None + + logger.debug( + "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request +@@ -451,7 +449,7 @@ class FederationHandler: + ) + ) + +- async def try_backfill(domains: StrCollection) -> bool: ++ async def try_backfill(domains: StrCollection) -> None: + # TODO: Should we try multiple of these at a time? + + # Number of contacted remote homeservers that have denied our backfill +@@ -474,7 +472,7 @@ class FederationHandler: + # If this succeeded then we probably already have the + # appropriate stuff. + # TODO: We can probably do something more intelligent here. +- return True ++ return None + except NotRetryingDestination as e: + logger.info("_maybe_backfill_inner: %s", e) + continue +@@ -498,7 +496,7 @@ class FederationHandler: + ) + denied_count += 1 + if denied_count >= max_denied_count: +- return False ++ return None + continue + + logger.info("Failed to backfill from %s because %s", dom, e) +@@ -514,7 +512,7 @@ class FederationHandler: + ) + denied_count += 1 + if denied_count >= max_denied_count: +- return False ++ return None + continue + + logger.info("Failed to backfill from %s because %s", dom, e) +@@ -526,7 +524,7 @@ class FederationHandler: + logger.exception("Failed to backfill from %s because %s", dom, e) + continue + +- return False ++ return None + + # If we have the `processing_start_time`, then we can make an + # observation. We wouldn't have the `processing_start_time` in the case +@@ -538,14 +536,9 @@ class FederationHandler: + **{SERVER_NAME_LABEL: self.server_name} + ).observe((processing_end_time - processing_start_time) / 1000) + +- success = await try_backfill(likely_domains) +- if success: +- return True +- + # TODO: we could also try servers which were previously in the room, but + # are no longer. +- +- return False ++ return await try_backfill(likely_domains) + + async def send_invite(self, target_host: str, event: EventBase) -> EventBase: + """Sends the invite to the remote server for signing. +diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py +index df1a7e714c..503c58e32a 100644 +--- a/synapse/handlers/pagination.py ++++ b/synapse/handlers/pagination.py +@@ -579,27 +579,31 @@ class PaginationHandler: + or missing_too_many_events + or not_enough_events_to_fill_response + ): +- did_backfill = await self.hs.get_federation_handler().maybe_backfill( ++ # Historical Note: There used to be a check here for if backfill was ++ # successful or not ++ await self.hs.get_federation_handler().maybe_backfill( + room_id, + curr_topo, + limit=pagin_config.limit, + ) + +- # If we did backfill something, refetch the events from the database to +- # catch anything new that might have been added since we last fetched. +- if did_backfill: +- ( +- events, +- next_key, +- _, +- ) = await self.store.paginate_room_events_by_topological_ordering( +- room_id=room_id, +- from_key=from_token.room_key, +- to_key=to_room_key, +- direction=pagin_config.direction, +- limit=pagin_config.limit, +- event_filter=event_filter, +- ) ++ # Regardless if we backfilled or not, another worker or even a ++ # simultaneous request may have backfilled for us while we were held ++ # behind the linearizer. This should not have too much additional ++ # database load as it will only be triggered if a backfill *might* have ++ # been needed ++ ( ++ events, ++ next_key, ++ _, ++ ) = await self.store.paginate_room_events_by_topological_ordering( ++ room_id=room_id, ++ from_key=from_token.room_key, ++ to_key=to_room_key, ++ direction=pagin_config.direction, ++ limit=pagin_config.limit, ++ event_filter=event_filter, ++ ) + else: + # Otherwise, we can backfill in the background for eventual + # consistency's sake but we don't need to block the client waiting +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch new file mode 100644 index 0000000..93ca05a --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0008-Fix-pagination-with-large-gaps-of-rejected-events.patch @@ -0,0 +1,50 @@ +From d626f4da5162ba49ed8dbac9a90c847be5f9ac81 Mon Sep 17 00:00:00 2001 +From: Nicolas Werner +Date: Sun, 8 Jun 2025 23:14:31 +0200 +Subject: [PATCH 08/18] Fix pagination with large gaps of rejected events + +--- + synapse/handlers/pagination.py | 13 +++++++++++-- + 1 file changed, 11 insertions(+), 2 deletions(-) + +diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py +index 503c58e32a..43fdbc2e83 100644 +--- a/synapse/handlers/pagination.py ++++ b/synapse/handlers/pagination.py +@@ -512,7 +512,7 @@ class PaginationHandler: + ( + events, + next_key, +- _, ++ limited, + ) = await self.store.paginate_room_events_by_topological_ordering( + room_id=room_id, + from_key=from_token.room_key, +@@ -595,7 +595,7 @@ class PaginationHandler: + ( + events, + next_key, +- _, ++ limited, + ) = await self.store.paginate_room_events_by_topological_ordering( + room_id=room_id, + from_key=from_token.room_key, +@@ -619,6 +619,15 @@ class PaginationHandler: + + next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) + ++ # We might have hit some internal filtering first, for example rejected ++ # events. Ensure we return a pagination token then. ++ if not events and limited: ++ return { ++ "chunk": [], ++ "start": await from_token.to_string(self.store), ++ "end": await next_token.to_string(self.store), ++ } ++ + # if no events are returned from pagination, that implies + # we have reached the end of the available events. + # In that case we do not return end, to tell the client +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch b/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch deleted file mode 100644 index d607b7c..0000000 --- a/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch +++ /dev/null @@ -1,204 +0,0 @@ -From 0ed11113c980e5edfd79c8c0f78b245adf823ac2 Mon Sep 17 00:00:00 2001 -From: Jason Little -Date: Wed, 30 Apr 2025 09:29:42 -0500 -Subject: [PATCH 08/19] fix: Always recheck `/messages` pagination data if a - backfill might have been needed (#28) - ---- - synapse/handlers/federation.py | 35 +++++++++++++-------------------- - synapse/handlers/pagination.py | 36 +++++++++++++++++++--------------- - 2 files changed, 34 insertions(+), 37 deletions(-) - -diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py -index 34aae7ef3c..b3226de999 100644 ---- a/synapse/handlers/federation.py -+++ b/synapse/handlers/federation.py -@@ -197,7 +197,7 @@ class FederationHandler: - @tag_args - async def maybe_backfill( - self, room_id: str, current_depth: int, limit: int, record_time: bool = True -- ) -> bool: -+ ) -> None: - """Checks the database to see if we should backfill before paginating, - and if so do. - -@@ -211,8 +211,6 @@ class FederationHandler: - should back paginate. - record_time: Whether to record the time it takes to backfill. - -- Returns: -- True if we actually tried to backfill something, otherwise False. - """ - # Starting the processing time here so we can include the room backfill - # linearizer lock queue in the timing -@@ -238,7 +236,7 @@ class FederationHandler: - limit: int, - *, - processing_start_time: Optional[int], -- ) -> bool: -+ ) -> None: - """ - Checks whether the `current_depth` is at or approaching any backfill - points in the room and if so, will backfill. We only care about -@@ -312,7 +310,7 @@ class FederationHandler: - limit=1, - ) - if not have_later_backfill_points: -- return False -+ return None - - logger.debug( - "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points." -@@ -333,15 +331,15 @@ class FederationHandler: - ) - # We return `False` because we're backfilling in the background and there is - # no new events immediately for the caller to know about yet. -- return False -+ return None - - # Even after recursing with `MAX_DEPTH`, we didn't find any - # backward extremities to backfill from. - if not sorted_backfill_points: - logger.debug( -- "_maybe_backfill_inner: Not backfilling as no backward extremeties found." -+ "_maybe_backfill_inner: Not backfilling as no backward extremities found." - ) -- return False -+ return None - - # If we're approaching an extremity we trigger a backfill, otherwise we - # no-op. -@@ -360,7 +358,7 @@ class FederationHandler: - current_depth, - limit, - ) -- return False -+ return None - - # For performance's sake, we only want to paginate from a particular extremity - # if we can actually see the events we'll get. Otherwise, we'd just spend a lot -@@ -428,7 +426,7 @@ class FederationHandler: - logger.debug( - "_maybe_backfill_inner: found no extremities which would be visible" - ) -- return False -+ return None - - logger.debug( - "_maybe_backfill_inner: extremities_to_request %s", extremities_to_request -@@ -451,7 +449,7 @@ class FederationHandler: - ) - ) - -- async def try_backfill(domains: StrCollection) -> bool: -+ async def try_backfill(domains: StrCollection) -> None: - # TODO: Should we try multiple of these at a time? - - # Number of contacted remote homeservers that have denied our backfill -@@ -474,7 +472,7 @@ class FederationHandler: - # If this succeeded then we probably already have the - # appropriate stuff. - # TODO: We can probably do something more intelligent here. -- return True -+ return None - except NotRetryingDestination as e: - logger.info("_maybe_backfill_inner: %s", e) - continue -@@ -498,7 +496,7 @@ class FederationHandler: - ) - denied_count += 1 - if denied_count >= max_denied_count: -- return False -+ return None - continue - - logger.info("Failed to backfill from %s because %s", dom, e) -@@ -514,7 +512,7 @@ class FederationHandler: - ) - denied_count += 1 - if denied_count >= max_denied_count: -- return False -+ return None - continue - - logger.info("Failed to backfill from %s because %s", dom, e) -@@ -526,7 +524,7 @@ class FederationHandler: - logger.exception("Failed to backfill from %s because %s", dom, e) - continue - -- return False -+ return None - - # If we have the `processing_start_time`, then we can make an - # observation. We wouldn't have the `processing_start_time` in the case -@@ -538,14 +536,9 @@ class FederationHandler: - **{SERVER_NAME_LABEL: self.server_name} - ).observe((processing_end_time - processing_start_time) / 1000) - -- success = await try_backfill(likely_domains) -- if success: -- return True -- - # TODO: we could also try servers which were previously in the room, but - # are no longer. -- -- return False -+ return await try_backfill(likely_domains) - - async def send_invite(self, target_host: str, event: EventBase) -> EventBase: - """Sends the invite to the remote server for signing. -diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py -index df1a7e714c..503c58e32a 100644 ---- a/synapse/handlers/pagination.py -+++ b/synapse/handlers/pagination.py -@@ -579,27 +579,31 @@ class PaginationHandler: - or missing_too_many_events - or not_enough_events_to_fill_response - ): -- did_backfill = await self.hs.get_federation_handler().maybe_backfill( -+ # Historical Note: There used to be a check here for if backfill was -+ # successful or not -+ await self.hs.get_federation_handler().maybe_backfill( - room_id, - curr_topo, - limit=pagin_config.limit, - ) - -- # If we did backfill something, refetch the events from the database to -- # catch anything new that might have been added since we last fetched. -- if did_backfill: -- ( -- events, -- next_key, -- _, -- ) = await self.store.paginate_room_events_by_topological_ordering( -- room_id=room_id, -- from_key=from_token.room_key, -- to_key=to_room_key, -- direction=pagin_config.direction, -- limit=pagin_config.limit, -- event_filter=event_filter, -- ) -+ # Regardless if we backfilled or not, another worker or even a -+ # simultaneous request may have backfilled for us while we were held -+ # behind the linearizer. This should not have too much additional -+ # database load as it will only be triggered if a backfill *might* have -+ # been needed -+ ( -+ events, -+ next_key, -+ _, -+ ) = await self.store.paginate_room_events_by_topological_ordering( -+ room_id=room_id, -+ from_key=from_token.room_key, -+ to_key=to_room_key, -+ direction=pagin_config.direction, -+ limit=pagin_config.limit, -+ event_filter=event_filter, -+ ) - else: - # Otherwise, we can backfill in the background for eventual - # consistency's sake but we don't need to block the client waiting --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch b/packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch new file mode 100644 index 0000000..0b31c1e --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0009-Fix-nix-flake.patch @@ -0,0 +1,189 @@ +From 0273b1059a90ed162cfd7d4754151fce9c173d6a Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Mon, 9 Jun 2025 17:38:34 +0200 +Subject: [PATCH 09/18] Fix nix flake + +--- + flake.lock | 58 +++++++++++++++++++----------------------------------- + flake.nix | 10 +++++++++- + 2 files changed, 29 insertions(+), 39 deletions(-) + +diff --git a/flake.lock b/flake.lock +index a6a2aea328..4e2f01153b 100644 +--- a/flake.lock ++++ b/flake.lock +@@ -39,15 +39,12 @@ + } + }, + "flake-utils": { +- "inputs": { +- "systems": "systems" +- }, + "locked": { +- "lastModified": 1685518550, +- "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", ++ "lastModified": 1667395993, ++ "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", +- "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", ++ "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { +@@ -152,27 +149,27 @@ + }, + "nixpkgs-stable": { + "locked": { +- "lastModified": 1685801374, +- "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", ++ "lastModified": 1678872516, ++ "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=", + "owner": "NixOS", + "repo": "nixpkgs", +- "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", ++ "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8", + "type": "github" + }, + "original": { + "owner": "NixOS", +- "ref": "nixos-23.05", ++ "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { +- "lastModified": 1729265718, +- "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=", ++ "lastModified": 1748217807, ++ "narHash": "sha256-P3u2PXxMlo49PutQLnk2PhI/imC69hFl1yY4aT5Nax8=", + "owner": "NixOS", + "repo": "nixpkgs", +- "rev": "ccc0c2126893dd20963580b6478d1a10a4512185", ++ "rev": "3108eaa516ae22c2360928589731a4f1581526ef", + "type": "github" + }, + "original": { +@@ -184,11 +181,11 @@ + }, + "nixpkgs_3": { + "locked": { +- "lastModified": 1728538411, +- "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=", ++ "lastModified": 1744536153, ++ "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", + "owner": "NixOS", + "repo": "nixpkgs", +- "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221", ++ "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "type": "github" + }, + "original": { +@@ -213,11 +210,11 @@ + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { +- "lastModified": 1688056373, +- "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", ++ "lastModified": 1686050334, ++ "narHash": "sha256-R0mczWjDzBpIvM3XXhO908X5e2CQqjyh/gFbwZk/7/Q=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", +- "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", ++ "rev": "6881eb2ae5d8a3516e34714e7a90d9d95914c4dc", + "type": "github" + }, + "original": { +@@ -231,7 +228,7 @@ + "devenv": "devenv", + "nixpkgs": "nixpkgs_2", + "rust-overlay": "rust-overlay", +- "systems": "systems_2" ++ "systems": "systems" + } + }, + "rust-overlay": { +@@ -239,11 +236,11 @@ + "nixpkgs": "nixpkgs_3" + }, + "locked": { +- "lastModified": 1731897198, +- "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", ++ "lastModified": 1748313401, ++ "narHash": "sha256-x5UuDKP2Ui/TresAngUo9U4Ss9xfOmN8dAXU8OrkZmA=", + "owner": "oxalica", + "repo": "rust-overlay", +- "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", ++ "rev": "9c8ea175cf9af29edbcff121512e44092a8f37e4", + "type": "github" + }, + "original": { +@@ -266,21 +263,6 @@ + "repo": "default", + "type": "github" + } +- }, +- "systems_2": { +- "locked": { +- "lastModified": 1681028828, +- "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", +- "owner": "nix-systems", +- "repo": "default", +- "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", +- "type": "github" +- }, +- "original": { +- "owner": "nix-systems", +- "repo": "default", +- "type": "github" +- } + } + }, + "root": "root", +diff --git a/flake.nix b/flake.nix +index 4ff6518aed..76b3c1a4b0 100644 +--- a/flake.nix ++++ b/flake.nix +@@ -82,7 +82,7 @@ + # + # NOTE: We currently need to set the Rust version unnecessarily high + # in order to work around https://github.com/matrix-org/synapse/issues/15939 +- (rust-bin.stable."1.82.0".default.override { ++ (rust-bin.stable."1.87.0".default.override { + # Additionally install the "rust-src" extension to allow diving into the + # Rust source code in an IDE (rust-analyzer will also make use of it). + extensions = [ "rust-src" ]; +@@ -117,6 +117,8 @@ + # For releasing Synapse + debian-devscripts # (`dch` for manipulating the Debian changelog) + libnotify # (the release script uses `notify-send` to tell you when CI jobs are done) ++ ++ postgresql.pg_config + ]; + + # Install Python and manage a virtualenv with Poetry. +@@ -139,6 +141,9 @@ + # force compiling those binaries locally instead. + env.POETRY_INSTALLER_NO_BINARY = "ruff"; + ++ # Required to make git work ++ env.CARGO_NET_GIT_FETCH_WITH_CLI = "true"; ++ + # Install dependencies for the additional programming languages + # involved with Synapse development. + # +@@ -159,6 +164,9 @@ + services.postgres.initialDatabases = [ + { name = "synapse"; } + ]; ++ ++ services.postgres.port = 5433; ++ + # Create a postgres user called 'synapse_user' which has ownership + # over the 'synapse' database. + services.postgres.initialScript = '' +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch deleted file mode 100644 index f2e9695..0000000 --- a/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 57743dad758b46cda09f69ee44c99cbd649f873e Mon Sep 17 00:00:00 2001 -From: Nicolas Werner -Date: Sun, 8 Jun 2025 23:14:31 +0200 -Subject: [PATCH 09/19] Fix pagination with large gaps of rejected events - ---- - synapse/handlers/pagination.py | 13 +++++++++++-- - 1 file changed, 11 insertions(+), 2 deletions(-) - -diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py -index 503c58e32a..43fdbc2e83 100644 ---- a/synapse/handlers/pagination.py -+++ b/synapse/handlers/pagination.py -@@ -512,7 +512,7 @@ class PaginationHandler: - ( - events, - next_key, -- _, -+ limited, - ) = await self.store.paginate_room_events_by_topological_ordering( - room_id=room_id, - from_key=from_token.room_key, -@@ -595,7 +595,7 @@ class PaginationHandler: - ( - events, - next_key, -- _, -+ limited, - ) = await self.store.paginate_room_events_by_topological_ordering( - room_id=room_id, - from_key=from_token.room_key, -@@ -619,6 +619,15 @@ class PaginationHandler: - - next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key) - -+ # We might have hit some internal filtering first, for example rejected -+ # events. Ensure we return a pagination token then. -+ if not events and limited: -+ return { -+ "chunk": [], -+ "start": await from_token.to_string(self.store), -+ "end": await next_token.to_string(self.store), -+ } -+ - # if no events are returned from pagination, that implies - # we have reached the end of the available events. - # In that case we do not return end, to tell the client --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch b/packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch new file mode 100644 index 0000000..6d35398 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0010-Fix-gitignore-to-ignore-.venv.patch @@ -0,0 +1,24 @@ +From 491ac94a9096a00ab250645e624b6caeba592f52 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Mon, 9 Jun 2025 17:46:10 +0200 +Subject: [PATCH 10/18] Fix gitignore to ignore .venv + +--- + .gitignore | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/.gitignore b/.gitignore +index e333f2320b..3aec96e75e 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -30,6 +30,7 @@ __pycache__/ + /*.signing.key + /env/ + /.venv*/ ++/.venv + /homeserver*.yaml + /logs + /media_store/ +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch b/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch deleted file mode 100644 index 2b35d06..0000000 --- a/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch +++ /dev/null @@ -1,189 +0,0 @@ -From 2f4533a3c26c62b2e287c3e5a284ca8ee1042546 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Mon, 9 Jun 2025 17:38:34 +0200 -Subject: [PATCH 10/19] Fix nix flake - ---- - flake.lock | 58 +++++++++++++++++++----------------------------------- - flake.nix | 10 +++++++++- - 2 files changed, 29 insertions(+), 39 deletions(-) - -diff --git a/flake.lock b/flake.lock -index a6a2aea328..4e2f01153b 100644 ---- a/flake.lock -+++ b/flake.lock -@@ -39,15 +39,12 @@ - } - }, - "flake-utils": { -- "inputs": { -- "systems": "systems" -- }, - "locked": { -- "lastModified": 1685518550, -- "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", -+ "lastModified": 1667395993, -+ "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", -- "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", -+ "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { -@@ -152,27 +149,27 @@ - }, - "nixpkgs-stable": { - "locked": { -- "lastModified": 1685801374, -- "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", -+ "lastModified": 1678872516, -+ "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=", - "owner": "NixOS", - "repo": "nixpkgs", -- "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", -+ "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8", - "type": "github" - }, - "original": { - "owner": "NixOS", -- "ref": "nixos-23.05", -+ "ref": "nixos-22.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { -- "lastModified": 1729265718, -- "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=", -+ "lastModified": 1748217807, -+ "narHash": "sha256-P3u2PXxMlo49PutQLnk2PhI/imC69hFl1yY4aT5Nax8=", - "owner": "NixOS", - "repo": "nixpkgs", -- "rev": "ccc0c2126893dd20963580b6478d1a10a4512185", -+ "rev": "3108eaa516ae22c2360928589731a4f1581526ef", - "type": "github" - }, - "original": { -@@ -184,11 +181,11 @@ - }, - "nixpkgs_3": { - "locked": { -- "lastModified": 1728538411, -- "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=", -+ "lastModified": 1744536153, -+ "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", - "owner": "NixOS", - "repo": "nixpkgs", -- "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221", -+ "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", - "type": "github" - }, - "original": { -@@ -213,11 +210,11 @@ - "nixpkgs-stable": "nixpkgs-stable" - }, - "locked": { -- "lastModified": 1688056373, -- "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", -+ "lastModified": 1686050334, -+ "narHash": "sha256-R0mczWjDzBpIvM3XXhO908X5e2CQqjyh/gFbwZk/7/Q=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", -- "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", -+ "rev": "6881eb2ae5d8a3516e34714e7a90d9d95914c4dc", - "type": "github" - }, - "original": { -@@ -231,7 +228,7 @@ - "devenv": "devenv", - "nixpkgs": "nixpkgs_2", - "rust-overlay": "rust-overlay", -- "systems": "systems_2" -+ "systems": "systems" - } - }, - "rust-overlay": { -@@ -239,11 +236,11 @@ - "nixpkgs": "nixpkgs_3" - }, - "locked": { -- "lastModified": 1731897198, -- "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=", -+ "lastModified": 1748313401, -+ "narHash": "sha256-x5UuDKP2Ui/TresAngUo9U4Ss9xfOmN8dAXU8OrkZmA=", - "owner": "oxalica", - "repo": "rust-overlay", -- "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5", -+ "rev": "9c8ea175cf9af29edbcff121512e44092a8f37e4", - "type": "github" - }, - "original": { -@@ -266,21 +263,6 @@ - "repo": "default", - "type": "github" - } -- }, -- "systems_2": { -- "locked": { -- "lastModified": 1681028828, -- "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", -- "owner": "nix-systems", -- "repo": "default", -- "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", -- "type": "github" -- }, -- "original": { -- "owner": "nix-systems", -- "repo": "default", -- "type": "github" -- } - } - }, - "root": "root", -diff --git a/flake.nix b/flake.nix -index 4ff6518aed..76b3c1a4b0 100644 ---- a/flake.nix -+++ b/flake.nix -@@ -82,7 +82,7 @@ - # - # NOTE: We currently need to set the Rust version unnecessarily high - # in order to work around https://github.com/matrix-org/synapse/issues/15939 -- (rust-bin.stable."1.82.0".default.override { -+ (rust-bin.stable."1.87.0".default.override { - # Additionally install the "rust-src" extension to allow diving into the - # Rust source code in an IDE (rust-analyzer will also make use of it). - extensions = [ "rust-src" ]; -@@ -117,6 +117,8 @@ - # For releasing Synapse - debian-devscripts # (`dch` for manipulating the Debian changelog) - libnotify # (the release script uses `notify-send` to tell you when CI jobs are done) -+ -+ postgresql.pg_config - ]; - - # Install Python and manage a virtualenv with Poetry. -@@ -139,6 +141,9 @@ - # force compiling those binaries locally instead. - env.POETRY_INSTALLER_NO_BINARY = "ruff"; - -+ # Required to make git work -+ env.CARGO_NET_GIT_FETCH_WITH_CLI = "true"; -+ - # Install dependencies for the additional programming languages - # involved with Synapse development. - # -@@ -159,6 +164,9 @@ - services.postgres.initialDatabases = [ - { name = "synapse"; } - ]; -+ -+ services.postgres.port = 5433; -+ - # Create a postgres user called 'synapse_user' which has ownership - # over the 'synapse' database. - services.postgres.initialScript = '' --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0011-Devenv-use-postgres-17.patch b/packages/overlays/matrix-synapse/patches/0011-Devenv-use-postgres-17.patch new file mode 100644 index 0000000..825e42d --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0011-Devenv-use-postgres-17.patch @@ -0,0 +1,24 @@ +From 06e9b346f8f23b373b5974ef155c5fee053ba734 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Fri, 25 Jul 2025 08:25:28 +0200 +Subject: [PATCH 11/18] Devenv: use postgres 17 + +--- + flake.nix | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/flake.nix b/flake.nix +index 76b3c1a4b0..cc41490a41 100644 +--- a/flake.nix ++++ b/flake.nix +@@ -157,6 +157,7 @@ + # Postgres is needed to run Synapse with postgres support and + # to run certain unit tests that require postgres. + services.postgres.enable = true; ++ services.postgres.package = pkgs.postgresql_17; + + # On the first invocation of `devenv up`, create a database for + # Synapse to store data in. +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch b/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch deleted file mode 100644 index 8f8b051..0000000 --- a/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 2fb09d660565b39cdd1077f0e29580c72a8bbb47 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Mon, 9 Jun 2025 17:46:10 +0200 -Subject: [PATCH 11/19] Fix gitignore to ignore .venv - ---- - .gitignore | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/.gitignore b/.gitignore -index e333f2320b..3aec96e75e 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -30,6 +30,7 @@ __pycache__/ - /*.signing.key - /env/ - /.venv*/ -+/.venv - /homeserver*.yaml - /logs - /media_store/ --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch b/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch deleted file mode 100644 index 15ee0a3..0000000 --- a/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 80335192cafb2b3b33a8822ba3f720d400853d8c Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Fri, 25 Jul 2025 08:25:28 +0200 -Subject: [PATCH 12/19] Devenv: use postgres 17 - ---- - flake.nix | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/flake.nix b/flake.nix -index 76b3c1a4b0..cc41490a41 100644 ---- a/flake.nix -+++ b/flake.nix -@@ -157,6 +157,7 @@ - # Postgres is needed to run Synapse with postgres support and - # to run certain unit tests that require postgres. - services.postgres.enable = true; -+ services.postgres.package = pkgs.postgresql_17; - - # On the first invocation of `devenv up`, create a database for - # Synapse to store data in. --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0012-RequestRatelimiter-expose-can_do_action.patch b/packages/overlays/matrix-synapse/patches/0012-RequestRatelimiter-expose-can_do_action.patch new file mode 100644 index 0000000..2bfae58 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0012-RequestRatelimiter-expose-can_do_action.patch @@ -0,0 +1,95 @@ +From ebd0ee5a220d700924c66b24f1f1e16fee2588e3 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Fri, 25 Jul 2025 08:26:15 +0200 +Subject: [PATCH 12/18] RequestRatelimiter: expose can_do_action + +--- + synapse/api/ratelimiting.py | 75 +++++++++++++++++++++++++++++++++++++ + 1 file changed, 75 insertions(+) + +diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py +index 9d1c7801bc..2a3fb63a5d 100644 +--- a/synapse/api/ratelimiting.py ++++ b/synapse/api/ratelimiting.py +@@ -435,3 +435,78 @@ class RequestRatelimiter: + update=update, + n_actions=n_actions, + ) ++ ++ async def can_do_action( ++ self, ++ requester: Optional[Requester], ++ burst_count: Optional[int] = None, ++ update: bool = True, ++ is_admin_redaction: bool = False, ++ n_actions: int = 1, ++ ) -> Tuple[bool, float]: ++ """Can the entity (e.g. user or IP address) perform the action? ++ ++ Checks if the user has ratelimiting disabled in the database by looking ++ for null/zero values in the `ratelimit_override` table. (Non-zero ++ values aren't honoured, as they're specific to the event sending ++ ratelimiter, rather than all ratelimiters) ++ ++ Args: ++ requester: The requester that is doing the action, if any. Used to check ++ if the user has ratelimits disabled in the database. ++ key: An arbitrary key used to classify an action. Defaults to the ++ requester's user ID. ++ rate_hz: The long term number of actions that can be performed in a second. ++ Overrides the value set during instantiation if set. ++ burst_count: How many actions that can be performed before being limited. ++ Overrides the value set during instantiation if set. ++ update: Whether to count this check as performing the action. If the action ++ cannot be performed, the user's action count is not incremented at all. ++ n_actions: The number of times the user wants to do this action. If the user ++ cannot do all of the actions, the user's action count is not incremented ++ at all. ++ _time_now_s: The current time. Optional, defaults to the current time according ++ to self.clock. Only used by tests. ++ ++ Returns: ++ A tuple containing: ++ * A bool indicating if they can perform the action now ++ * The reactor timestamp for when the action can be performed next. ++ -1 if rate_hz is less than or equal to zero ++ """ ++ user_id = requester.user.to_string() ++ ++ # The AS user itself is never rate limited. ++ app_service = self.store.get_app_service_by_user_id(user_id) ++ if app_service is not None: ++ return True, 0 # do not ratelimit app service senders ++ ++ messages_per_second = self._rc_message.per_second ++ burst_count = self._rc_message.burst_count ++ ++ # Check if there is a per user override in the DB. ++ override = await self.store.get_ratelimit_for_user(user_id) ++ if override: ++ # If overridden with a null Hz then ratelimiting has been entirely ++ # disabled for the user ++ if not override.messages_per_second: ++ return True, 0 ++ ++ messages_per_second = override.messages_per_second ++ burst_count = override.burst_count ++ ++ if is_admin_redaction and self.admin_redaction_ratelimiter: ++ # If we have separate config for admin redactions, use a separate ++ # ratelimiter as to not have user_ids clash ++ return await self.admin_redaction_ratelimiter.can_do_action( ++ requester, update=update, n_actions=n_actions ++ ) ++ else: ++ # Override rate and burst count per-user ++ return await self.request_ratelimiter.can_do_action( ++ requester, ++ rate_hz=messages_per_second, ++ burst_count=burst_count, ++ update=update, ++ n_actions=n_actions, ++ ) +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0013-Add-bulk-send-events-endpoint.patch b/packages/overlays/matrix-synapse/patches/0013-Add-bulk-send-events-endpoint.patch new file mode 100644 index 0000000..b7472e6 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0013-Add-bulk-send-events-endpoint.patch @@ -0,0 +1,187 @@ +From d03e270736ae957f324a99344bd389eeb9d2c5e6 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sat, 26 Jul 2025 09:50:56 +0200 +Subject: [PATCH 13/18] Add bulk send events endpoint + +--- + synapse/rest/client/capabilities.py | 3 + + synapse/rest/client/room.py | 117 +++++++++++++++++++++++++++- + 2 files changed, 119 insertions(+), 1 deletion(-) + +diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py +index a279db1cc5..5e18750e15 100644 +--- a/synapse/rest/client/capabilities.py ++++ b/synapse/rest/client/capabilities.py +@@ -74,6 +74,9 @@ class CapabilitiesRestServlet(RestServlet): + "m.get_login_token": { + "enabled": self.config.auth.login_via_existing_enabled, + }, ++ "gay.rory.bulk_send_events": { ++ "enabled": True ++ } + } + } + +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index e7f41f0079..d054d87595 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -23,10 +23,12 @@ + + import logging + import re ++import ijson + from enum import Enum + from http import HTTPStatus + from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple + from urllib import parse as urlparse ++from twisted.internet import defer + + from prometheus_client.core import Histogram + +@@ -49,6 +51,7 @@ from synapse.events.utils import ( + format_event_for_client_v2, + serialize_event, + ) ++from synapse.events import EventBase + from synapse.http.server import HttpServer + from synapse.http.servlet import ( + ResolveRoomIdMixin, +@@ -482,7 +485,6 @@ class RoomSendEventRestServlet(TransactionRestServlet): + txn_id, + ) + +- + def _parse_request_delay( + request: SynapseRequest, + max_delay: Optional[int], +@@ -1635,6 +1637,118 @@ class RoomSummaryRestServlet(ResolveRoomIdMixin, RestServlet): + remote_room_hosts, + ) + ++class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): ++ """ ++ Bulk send events to a room. ++ ++ This endpoint allows sending multiple events to a room in a single request, ++ avoiding event linearisation issues. ++ """ ++ ++ PATTERNS = ( ++ re.compile( ++ "^/_matrix/client/unstable/gay.rory.bulk_send_events" ++ "/rooms/(?P[^/]*)/bulk_send_events$" ++ ), ++ ) ++ CATEGORY = "Event sending requests" ++ ++ def __init__(self, hs: "HomeServer"): ++ super().__init__(hs) ++ self._auth = hs.get_auth() ++ self._event_creation_handler = hs.get_event_creation_handler() ++ self._message_handler = hs.get_message_handler() ++ ++ async def on_POST( ++ self, request: SynapseRequest, room_identifier: str ++ ) -> Tuple[int, JsonDict]: ++ logger.warning("bulk_send_events: Got bulk send events request") ++ requester = await self._auth.get_user_by_req(request, allow_guest=False) ++ room_id, remote_room_hosts = await self.resolve_room_id(room_identifier) ++ ++ force_sync_interval = parse_integer(request, "force_sync_interval", default=250) ++ ++ current_state_events = await self._message_handler.get_state_events( ++ room_id=room_id, ++ requester=requester, ++ ) ++ ++ state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} ++ ++ events = ijson.items( ++ request.content, ++ "item" ++ ) ++ ++ i = 0 ++ unpersisted_events = [] ++ ++ for event_data in events: ++ current_index = i ++ i += 1 ++ logger.info("bulk_send_events: Processing event %d: %s", current_index, event_data) ++ ++ event_dict: JsonDict = { ++ "type": event_data.get("type"), ++ "content": event_data.get("content", {}), ++ "room_id": room_id, ++ "sender": requester.user.to_string(), ++ } ++ ++ if "state_key" in event_data: ++ event_dict["state_key"] = event_data["state_key"] ++ ++ # Explicitly handle rate limits in order to avoid compounding effects ++ awaiting_ratelimit = False ++ ratelimit_hit = False ++ while awaiting_ratelimit: ++ can_do_action, ratelimit_expiry = await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=False) ++ if not can_do_action: ++ # can_do_action returns an absolute timestamp, convert it to a relative time ++ time_to_sleep = ratelimit_expiry - self._event_creation_handler.request_ratelimiter.clock.time() ++ logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep) ++ await self._event_creation_handler.request_ratelimiter.clock.sleep(time_to_sleep) ++ ratelimit_hit = True ++ else: ++ awaiting_ratelimit = False ++ await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=True) ++ ++ event, unpersisted_context = await self._event_creation_handler.create_event( ++ requester, ++ event_dict, ++ for_batch=True, ++ state_map=state_map, ++ ) ++ context = await unpersisted_context.persist(event) ++ ++ if event.is_state(): ++ prev_event = await self._event_creation_handler.deduplicate_state_event(event, context) ++ if prev_event is not None: ++ logger.info( ++ "Not bothering to persist state event %s duplicated by %s", ++ event.event_id, ++ prev_event.event_id, ++ ) ++ continue ++ else: ++ state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id ++ logger.warning("bulk_send_events: Updated state_map!") ++ ++ unpersisted_events.append((event, context)) ++ logger.warning("bulk_send_events: Persisted event %d: %s", current_index, event) ++ ++ if ratelimit_hit or len(unpersisted_events) >= force_sync_interval: ++ logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events)) ++ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) ++ unpersisted_events = [] ++ ++ # Finalize any remaining unpersisted events ++ if(len(unpersisted_events) > 0): ++ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) ++ unpersisted_events = [] ++ ++ return 200, {} ++ + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + RoomStateEventRestServlet(hs).register(http_server) +@@ -1644,6 +1758,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + JoinRoomAliasServlet(hs).register(http_server) + RoomMembershipRestServlet(hs).register(http_server) + RoomSendEventRestServlet(hs).register(http_server) ++ RoomBulkSendEventRestServlet(hs).register(http_server) + PublicRoomListRestServlet(hs).register(http_server) + RoomStateRestServlet(hs).register(http_server) + RoomRedactEventRestServlet(hs).register(http_server) +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch b/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch deleted file mode 100644 index ef9dbd6..0000000 --- a/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch +++ /dev/null @@ -1,95 +0,0 @@ -From 49d21ee7150580474879b8c9e149c273dd24ad0e Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Fri, 25 Jul 2025 08:26:15 +0200 -Subject: [PATCH 13/19] RequestRatelimiter: expose can_do_action - ---- - synapse/api/ratelimiting.py | 75 +++++++++++++++++++++++++++++++++++++ - 1 file changed, 75 insertions(+) - -diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py -index 509ef6b2c1..5f22089a6b 100644 ---- a/synapse/api/ratelimiting.py -+++ b/synapse/api/ratelimiting.py -@@ -435,3 +435,78 @@ class RequestRatelimiter: - update=update, - n_actions=n_actions, - ) -+ -+ async def can_do_action( -+ self, -+ requester: Optional[Requester], -+ burst_count: Optional[int] = None, -+ update: bool = True, -+ is_admin_redaction: bool = False, -+ n_actions: int = 1, -+ ) -> Tuple[bool, float]: -+ """Can the entity (e.g. user or IP address) perform the action? -+ -+ Checks if the user has ratelimiting disabled in the database by looking -+ for null/zero values in the `ratelimit_override` table. (Non-zero -+ values aren't honoured, as they're specific to the event sending -+ ratelimiter, rather than all ratelimiters) -+ -+ Args: -+ requester: The requester that is doing the action, if any. Used to check -+ if the user has ratelimits disabled in the database. -+ key: An arbitrary key used to classify an action. Defaults to the -+ requester's user ID. -+ rate_hz: The long term number of actions that can be performed in a second. -+ Overrides the value set during instantiation if set. -+ burst_count: How many actions that can be performed before being limited. -+ Overrides the value set during instantiation if set. -+ update: Whether to count this check as performing the action. If the action -+ cannot be performed, the user's action count is not incremented at all. -+ n_actions: The number of times the user wants to do this action. If the user -+ cannot do all of the actions, the user's action count is not incremented -+ at all. -+ _time_now_s: The current time. Optional, defaults to the current time according -+ to self.clock. Only used by tests. -+ -+ Returns: -+ A tuple containing: -+ * A bool indicating if they can perform the action now -+ * The reactor timestamp for when the action can be performed next. -+ -1 if rate_hz is less than or equal to zero -+ """ -+ user_id = requester.user.to_string() -+ -+ # The AS user itself is never rate limited. -+ app_service = self.store.get_app_service_by_user_id(user_id) -+ if app_service is not None: -+ return True, 0 # do not ratelimit app service senders -+ -+ messages_per_second = self._rc_message.per_second -+ burst_count = self._rc_message.burst_count -+ -+ # Check if there is a per user override in the DB. -+ override = await self.store.get_ratelimit_for_user(user_id) -+ if override: -+ # If overridden with a null Hz then ratelimiting has been entirely -+ # disabled for the user -+ if not override.messages_per_second: -+ return True, 0 -+ -+ messages_per_second = override.messages_per_second -+ burst_count = override.burst_count -+ -+ if is_admin_redaction and self.admin_redaction_ratelimiter: -+ # If we have separate config for admin redactions, use a separate -+ # ratelimiter as to not have user_ids clash -+ return await self.admin_redaction_ratelimiter.can_do_action( -+ requester, update=update, n_actions=n_actions -+ ) -+ else: -+ # Override rate and burst count per-user -+ return await self.request_ratelimiter.can_do_action( -+ requester, -+ rate_hz=messages_per_second, -+ burst_count=burst_count, -+ update=update, -+ n_actions=n_actions, -+ ) --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch b/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch deleted file mode 100644 index daf1e0f..0000000 --- a/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch +++ /dev/null @@ -1,187 +0,0 @@ -From c48c105727590a1a0a0e9fd0ac3422acead00e4b Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sat, 26 Jul 2025 09:50:56 +0200 -Subject: [PATCH 14/19] Add bulk send events endpoint - ---- - synapse/rest/client/capabilities.py | 3 + - synapse/rest/client/room.py | 117 +++++++++++++++++++++++++++- - 2 files changed, 119 insertions(+), 1 deletion(-) - -diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py -index a279db1cc5..5e18750e15 100644 ---- a/synapse/rest/client/capabilities.py -+++ b/synapse/rest/client/capabilities.py -@@ -74,6 +74,9 @@ class CapabilitiesRestServlet(RestServlet): - "m.get_login_token": { - "enabled": self.config.auth.login_via_existing_enabled, - }, -+ "gay.rory.bulk_send_events": { -+ "enabled": True -+ } - } - } - -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index e7f41f0079..d054d87595 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -23,10 +23,12 @@ - - import logging - import re -+import ijson - from enum import Enum - from http import HTTPStatus - from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple - from urllib import parse as urlparse -+from twisted.internet import defer - - from prometheus_client.core import Histogram - -@@ -49,6 +51,7 @@ from synapse.events.utils import ( - format_event_for_client_v2, - serialize_event, - ) -+from synapse.events import EventBase - from synapse.http.server import HttpServer - from synapse.http.servlet import ( - ResolveRoomIdMixin, -@@ -482,7 +485,6 @@ class RoomSendEventRestServlet(TransactionRestServlet): - txn_id, - ) - -- - def _parse_request_delay( - request: SynapseRequest, - max_delay: Optional[int], -@@ -1635,6 +1637,118 @@ class RoomSummaryRestServlet(ResolveRoomIdMixin, RestServlet): - remote_room_hosts, - ) - -+class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): -+ """ -+ Bulk send events to a room. -+ -+ This endpoint allows sending multiple events to a room in a single request, -+ avoiding event linearisation issues. -+ """ -+ -+ PATTERNS = ( -+ re.compile( -+ "^/_matrix/client/unstable/gay.rory.bulk_send_events" -+ "/rooms/(?P[^/]*)/bulk_send_events$" -+ ), -+ ) -+ CATEGORY = "Event sending requests" -+ -+ def __init__(self, hs: "HomeServer"): -+ super().__init__(hs) -+ self._auth = hs.get_auth() -+ self._event_creation_handler = hs.get_event_creation_handler() -+ self._message_handler = hs.get_message_handler() -+ -+ async def on_POST( -+ self, request: SynapseRequest, room_identifier: str -+ ) -> Tuple[int, JsonDict]: -+ logger.warning("bulk_send_events: Got bulk send events request") -+ requester = await self._auth.get_user_by_req(request, allow_guest=False) -+ room_id, remote_room_hosts = await self.resolve_room_id(room_identifier) -+ -+ force_sync_interval = parse_integer(request, "force_sync_interval", default=250) -+ -+ current_state_events = await self._message_handler.get_state_events( -+ room_id=room_id, -+ requester=requester, -+ ) -+ -+ state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} -+ -+ events = ijson.items( -+ request.content, -+ "item" -+ ) -+ -+ i = 0 -+ unpersisted_events = [] -+ -+ for event_data in events: -+ current_index = i -+ i += 1 -+ logger.info("bulk_send_events: Processing event %d: %s", current_index, event_data) -+ -+ event_dict: JsonDict = { -+ "type": event_data.get("type"), -+ "content": event_data.get("content", {}), -+ "room_id": room_id, -+ "sender": requester.user.to_string(), -+ } -+ -+ if "state_key" in event_data: -+ event_dict["state_key"] = event_data["state_key"] -+ -+ # Explicitly handle rate limits in order to avoid compounding effects -+ awaiting_ratelimit = False -+ ratelimit_hit = False -+ while awaiting_ratelimit: -+ can_do_action, ratelimit_expiry = await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=False) -+ if not can_do_action: -+ # can_do_action returns an absolute timestamp, convert it to a relative time -+ time_to_sleep = ratelimit_expiry - self._event_creation_handler.request_ratelimiter.clock.time() -+ logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep) -+ await self._event_creation_handler.request_ratelimiter.clock.sleep(time_to_sleep) -+ ratelimit_hit = True -+ else: -+ awaiting_ratelimit = False -+ await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=True) -+ -+ event, unpersisted_context = await self._event_creation_handler.create_event( -+ requester, -+ event_dict, -+ for_batch=True, -+ state_map=state_map, -+ ) -+ context = await unpersisted_context.persist(event) -+ -+ if event.is_state(): -+ prev_event = await self._event_creation_handler.deduplicate_state_event(event, context) -+ if prev_event is not None: -+ logger.info( -+ "Not bothering to persist state event %s duplicated by %s", -+ event.event_id, -+ prev_event.event_id, -+ ) -+ continue -+ else: -+ state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id -+ logger.warning("bulk_send_events: Updated state_map!") -+ -+ unpersisted_events.append((event, context)) -+ logger.warning("bulk_send_events: Persisted event %d: %s", current_index, event) -+ -+ if ratelimit_hit or len(unpersisted_events) >= force_sync_interval: -+ logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events)) -+ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) -+ unpersisted_events = [] -+ -+ # Finalize any remaining unpersisted events -+ if(len(unpersisted_events) > 0): -+ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) -+ unpersisted_events = [] -+ -+ return 200, {} -+ - - def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - RoomStateEventRestServlet(hs).register(http_server) -@@ -1644,6 +1758,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - JoinRoomAliasServlet(hs).register(http_server) - RoomMembershipRestServlet(hs).register(http_server) - RoomSendEventRestServlet(hs).register(http_server) -+ RoomBulkSendEventRestServlet(hs).register(http_server) - PublicRoomListRestServlet(hs).register(http_server) - RoomStateRestServlet(hs).register(http_server) - RoomRedactEventRestServlet(hs).register(http_server) --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0014-Temporarily-disable-go-in-flake.patch b/packages/overlays/matrix-synapse/patches/0014-Temporarily-disable-go-in-flake.patch new file mode 100644 index 0000000..63dd979 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0014-Temporarily-disable-go-in-flake.patch @@ -0,0 +1,25 @@ +From dc13671aaea35a90c2f21f74e60bc20c09364858 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sat, 16 Aug 2025 20:18:45 +0200 +Subject: [PATCH 14/18] Temporarily disable go in flake + +--- + flake.nix | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/flake.nix b/flake.nix +index cc41490a41..291e81814d 100644 +--- a/flake.nix ++++ b/flake.nix +@@ -151,7 +151,7 @@ + # * Perl is needed to run the SyTest test suite. + # * Rust is used for developing and running Synapse. + # It is installed manually with `packages` above. +- languages.go.enable = true; ++ #languages.go.enable = true; + languages.perl.enable = true; + + # Postgres is needed to run Synapse with postgres support and +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0015-Clarify-asserts.patch b/packages/overlays/matrix-synapse/patches/0015-Clarify-asserts.patch new file mode 100644 index 0000000..1e69a5a --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0015-Clarify-asserts.patch @@ -0,0 +1,25 @@ +From b2b70c9edd11f718c098400b55f3f68660af3123 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sat, 16 Aug 2025 20:19:08 +0200 +Subject: [PATCH 15/18] Clarify asserts + +--- + synapse/handlers/message.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py +index c8c86d8749..edd5477371 100644 +--- a/synapse/handlers/message.py ++++ b/synapse/handlers/message.py +@@ -1269,7 +1269,7 @@ class EventCreationHandler: + if state_event_ids is not None: + # Do a quick check to make sure that prev_event_ids is present to + # make the type-checking around `builder.build` happy. +- assert prev_event_ids is not None ++ assert prev_event_ids is not None, "create_new_client_event called with state_event_ids but no prev_event_ids" + + temp_event = await builder.build( + prev_event_ids=prev_event_ids, +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch b/packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch deleted file mode 100644 index fcd14aa..0000000 --- a/packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 8c1651a65f6d45bfb242762b6839de0c3a1728fa Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sat, 16 Aug 2025 20:18:45 +0200 -Subject: [PATCH 15/19] Temporarily disable go in flake - ---- - flake.nix | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/flake.nix b/flake.nix -index cc41490a41..291e81814d 100644 ---- a/flake.nix -+++ b/flake.nix -@@ -151,7 +151,7 @@ - # * Perl is needed to run the SyTest test suite. - # * Rust is used for developing and running Synapse. - # It is installed manually with `packages` above. -- languages.go.enable = true; -+ #languages.go.enable = true; - languages.perl.enable = true; - - # Postgres is needed to run Synapse with postgres support and --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch b/packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch deleted file mode 100644 index 5dc2321..0000000 --- a/packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 79d7af8028bfa3c512f0cb57600ca2b94e0666c1 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sat, 16 Aug 2025 20:19:08 +0200 -Subject: [PATCH 16/19] Clarify asserts - ---- - synapse/handlers/message.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py -index fff46b640b..2a5595c7b5 100644 ---- a/synapse/handlers/message.py -+++ b/synapse/handlers/message.py -@@ -1244,7 +1244,7 @@ class EventCreationHandler: - if state_event_ids is not None: - # Do a quick check to make sure that prev_event_ids is present to - # make the type-checking around `builder.build` happy. -- assert prev_event_ids is not None -+ assert prev_event_ids is not None, "create_new_client_event called with state_event_ids but no prev_event_ids" - - temp_event = await builder.build( - prev_event_ids=prev_event_ids, --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0016-Try-to-optimise-bulk-sending-some-more.patch b/packages/overlays/matrix-synapse/patches/0016-Try-to-optimise-bulk-sending-some-more.patch new file mode 100644 index 0000000..7bcfeb0 --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0016-Try-to-optimise-bulk-sending-some-more.patch @@ -0,0 +1,117 @@ +From 093b1514ff5945f609a5ca7af67e8bb7aea9b778 Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sat, 16 Aug 2025 20:19:26 +0200 +Subject: [PATCH 16/18] Try to optimise bulk sending some more + +--- + synapse/rest/client/room.py | 42 +++++++++++++++++++++++++++++++++---- + 1 file changed, 38 insertions(+), 4 deletions(-) + +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index d054d87595..304999aa5d 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -1658,6 +1658,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + self._auth = hs.get_auth() + self._event_creation_handler = hs.get_event_creation_handler() + self._message_handler = hs.get_message_handler() ++ self._storage_controllers = hs.get_storage_controllers() + + async def on_POST( + self, request: SynapseRequest, room_identifier: str +@@ -1668,12 +1669,24 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + + force_sync_interval = parse_integer(request, "force_sync_interval", default=250) + ++ latest_events = await self._storage_controllers.main.get_latest_event_ids_in_room( ++ room_id ++ ) ++ state_groups = await self._storage_controllers.state.get_state_group_for_events( ++ #[ latest_events[len(latest_events) - 1] ] ++ latest_events ++ ) + current_state_events = await self._message_handler.get_state_events( + room_id=room_id, + requester=requester, + ) + +- state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} ++ #state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} ++ state_map = await self._storage_controllers.persistence._calculate_current_state(room_id) ++ #state_map_evts = await self._storage_controllers.persistence.state_store.state( ++ # room_id, ++ # state_map, ++ #) + + events = ijson.items( + request.content, +@@ -1682,10 +1695,14 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + + i = 0 + unpersisted_events = [] ++ persistance_coroutines = [] ++ depth = None ++ prev_event_id = None + + for event_data in events: + current_index = i + i += 1 ++ current_time_ms = self._event_creation_handler.request_ratelimiter.clock.time_msec() + logger.info("bulk_send_events: Processing event %d: %s", current_index, event_data) + + event_dict: JsonDict = { +@@ -1717,12 +1734,27 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + requester, + event_dict, + for_batch=True, ++ require_consent=False, + state_map=state_map, ++ depth=depth, ++ prev_event_ids = [prev_event_id] if prev_event_id else None, ++ state_event_ids = list(state_map.values()) if prev_event_id else None, ++ current_state_group = state_groups.get(room_id, None) + ) ++ logger.warning("bulk_send_events: created event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) + context = await unpersisted_context.persist(event) ++ logger.warning("bulk_send_events: persisted event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) ++ if event.depth is not None: ++ depth = event.depth + 1 ++ if event.event_id is not None: ++ prev_event_id = event.event_id + + if event.is_state(): +- prev_event = await self._event_creation_handler.deduplicate_state_event(event, context) ++ logger.warning("bulk_send_events: scanning state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) ++ state_map_contains_event = state_map.get((event_dict["type"], event_dict["state_key"])) ++ logger.warning("bulk_send_events: scanned state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) ++ prev_event = None if state_map_contains_event is None else await self._event_creation_handler.deduplicate_state_event(event, context) ++ logger.warning("bulk_send_events: deduplicated state event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) + if prev_event is not None: + logger.info( + "Not bothering to persist state event %s duplicated by %s", +@@ -1732,18 +1764,20 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + continue + else: + state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id +- logger.warning("bulk_send_events: Updated state_map!") ++ logger.warning("bulk_send_events: Updated state_map! (State map contained entry: %s)", state_map_contains_event) + + unpersisted_events.append((event, context)) +- logger.warning("bulk_send_events: Persisted event %d: %s", current_index, event) ++ logger.warning("bulk_send_events: Persisted event %d after %d ms: %s", current_index, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms), event) + + if ratelimit_hit or len(unpersisted_events) >= force_sync_interval: + logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events)) ++ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... + await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) + unpersisted_events = [] + + # Finalize any remaining unpersisted events + if(len(unpersisted_events) > 0): ++ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... + await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) + unpersisted_events = [] + +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0017-Further-optimisation-attempts.patch b/packages/overlays/matrix-synapse/patches/0017-Further-optimisation-attempts.patch new file mode 100644 index 0000000..1d50b3d --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0017-Further-optimisation-attempts.patch @@ -0,0 +1,90 @@ +From 86792ae8aadb7463109b6305890d74547bfab9be Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sat, 16 Aug 2025 21:02:57 +0200 +Subject: [PATCH 17/18] Further optimisation attempts + +--- + synapse/rest/client/room.py | 39 ++++++++++++++++++++++++------------- + 1 file changed, 25 insertions(+), 14 deletions(-) + +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index 304999aa5d..d23c0d08ab 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -46,6 +46,7 @@ from synapse.api.errors import ( + UnredactedContentDeletedError, + ) + from synapse.api.filtering import Filter ++from synapse.api.ratelimiting import RequestRatelimiter + from synapse.events.utils import ( + SerializeEventConfig, + format_event_for_client_v2, +@@ -1660,6 +1661,25 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + self._message_handler = hs.get_message_handler() + self._storage_controllers = hs.get_storage_controllers() + ++ async def wait_ratelimit(ratelimiter: RequestRatelimiter, requester: Requester) -> bool: ++ """ ++ Wait until the ratelimiter allows the requester to proceed. ++ """ ++ ratelimit_hit = False ++ awaiting_ratelimit = False ++ while awaiting_ratelimit: ++ can_do_action, ratelimit_expiry = await ratelimiter.can_do_action(requester, update=False) ++ if not can_do_action: ++ # can_do_action returns an absolute timestamp, convert it to a relative time ++ time_to_sleep = ratelimit_expiry - ratelimiter.clock.time() ++ logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep) ++ await ratelimiter.clock.sleep(time_to_sleep) ++ ratelimit_hit = True ++ else: ++ awaiting_ratelimit = False ++ await ratelimiter.can_do_action(requester, update=True) ++ return ratelimit_hit ++ + async def on_POST( + self, request: SynapseRequest, room_identifier: str + ) -> Tuple[int, JsonDict]: +@@ -1695,7 +1715,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + + i = 0 + unpersisted_events = [] +- persistance_coroutines = [] ++ unpersisted_events_with_tasks = [] + depth = None + prev_event_id = None + +@@ -1716,19 +1736,8 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + event_dict["state_key"] = event_data["state_key"] + + # Explicitly handle rate limits in order to avoid compounding effects +- awaiting_ratelimit = False +- ratelimit_hit = False +- while awaiting_ratelimit: +- can_do_action, ratelimit_expiry = await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=False) +- if not can_do_action: +- # can_do_action returns an absolute timestamp, convert it to a relative time +- time_to_sleep = ratelimit_expiry - self._event_creation_handler.request_ratelimiter.clock.time() +- logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep) +- await self._event_creation_handler.request_ratelimiter.clock.sleep(time_to_sleep) +- ratelimit_hit = True +- else: +- awaiting_ratelimit = False +- await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=True) ++ ratelimit_hit = False # await self.wait_ratelimit(self._event_creation_handler.request_ratelimiter, requester) ++ + + event, unpersisted_context = await self._event_creation_handler.create_event( + requester, +@@ -1775,6 +1784,8 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) + unpersisted_events = [] + ++ logger.warning("bulk_send_events: Finished processing event %d in %d ms", i, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) ++ logger.warning("bulk_send_events: Finished processing %d events", i) + # Finalize any remaining unpersisted events + if(len(unpersisted_events) > 0): + await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch b/packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch deleted file mode 100644 index e43f1a4..0000000 --- a/packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch +++ /dev/null @@ -1,117 +0,0 @@ -From a5d49077038cf0e7235711cea3e756d5e96627d4 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sat, 16 Aug 2025 20:19:26 +0200 -Subject: [PATCH 17/19] Try to optimise bulk sending some more - ---- - synapse/rest/client/room.py | 42 +++++++++++++++++++++++++++++++++---- - 1 file changed, 38 insertions(+), 4 deletions(-) - -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index d054d87595..304999aa5d 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -1658,6 +1658,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - self._auth = hs.get_auth() - self._event_creation_handler = hs.get_event_creation_handler() - self._message_handler = hs.get_message_handler() -+ self._storage_controllers = hs.get_storage_controllers() - - async def on_POST( - self, request: SynapseRequest, room_identifier: str -@@ -1668,12 +1669,24 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - - force_sync_interval = parse_integer(request, "force_sync_interval", default=250) - -+ latest_events = await self._storage_controllers.main.get_latest_event_ids_in_room( -+ room_id -+ ) -+ state_groups = await self._storage_controllers.state.get_state_group_for_events( -+ #[ latest_events[len(latest_events) - 1] ] -+ latest_events -+ ) - current_state_events = await self._message_handler.get_state_events( - room_id=room_id, - requester=requester, - ) - -- state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} -+ #state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} -+ state_map = await self._storage_controllers.persistence._calculate_current_state(room_id) -+ #state_map_evts = await self._storage_controllers.persistence.state_store.state( -+ # room_id, -+ # state_map, -+ #) - - events = ijson.items( - request.content, -@@ -1682,10 +1695,14 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - - i = 0 - unpersisted_events = [] -+ persistance_coroutines = [] -+ depth = None -+ prev_event_id = None - - for event_data in events: - current_index = i - i += 1 -+ current_time_ms = self._event_creation_handler.request_ratelimiter.clock.time_msec() - logger.info("bulk_send_events: Processing event %d: %s", current_index, event_data) - - event_dict: JsonDict = { -@@ -1717,12 +1734,27 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - requester, - event_dict, - for_batch=True, -+ require_consent=False, - state_map=state_map, -+ depth=depth, -+ prev_event_ids = [prev_event_id] if prev_event_id else None, -+ state_event_ids = list(state_map.values()) if prev_event_id else None, -+ current_state_group = state_groups.get(room_id, None) - ) -+ logger.warning("bulk_send_events: created event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) - context = await unpersisted_context.persist(event) -+ logger.warning("bulk_send_events: persisted event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -+ if event.depth is not None: -+ depth = event.depth + 1 -+ if event.event_id is not None: -+ prev_event_id = event.event_id - - if event.is_state(): -- prev_event = await self._event_creation_handler.deduplicate_state_event(event, context) -+ logger.warning("bulk_send_events: scanning state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -+ state_map_contains_event = state_map.get((event_dict["type"], event_dict["state_key"])) -+ logger.warning("bulk_send_events: scanned state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -+ prev_event = None if state_map_contains_event is None else await self._event_creation_handler.deduplicate_state_event(event, context) -+ logger.warning("bulk_send_events: deduplicated state event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) - if prev_event is not None: - logger.info( - "Not bothering to persist state event %s duplicated by %s", -@@ -1732,18 +1764,20 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - continue - else: - state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id -- logger.warning("bulk_send_events: Updated state_map!") -+ logger.warning("bulk_send_events: Updated state_map! (State map contained entry: %s)", state_map_contains_event) - - unpersisted_events.append((event, context)) -- logger.warning("bulk_send_events: Persisted event %d: %s", current_index, event) -+ logger.warning("bulk_send_events: Persisted event %d after %d ms: %s", current_index, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms), event) - - if ratelimit_hit or len(unpersisted_events) >= force_sync_interval: - logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events)) -+ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... - await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) - unpersisted_events = [] - - # Finalize any remaining unpersisted events - if(len(unpersisted_events) > 0): -+ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... - await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) - unpersisted_events = [] - --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch b/packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch deleted file mode 100644 index 6ada1d8..0000000 --- a/packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch +++ /dev/null @@ -1,90 +0,0 @@ -From d9df11ecb0942fd7f3ba317ba743a9e61ea64a94 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sat, 16 Aug 2025 21:02:57 +0200 -Subject: [PATCH 18/19] Further optimisation attempts - ---- - synapse/rest/client/room.py | 39 ++++++++++++++++++++++++------------- - 1 file changed, 25 insertions(+), 14 deletions(-) - -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index 304999aa5d..d23c0d08ab 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -46,6 +46,7 @@ from synapse.api.errors import ( - UnredactedContentDeletedError, - ) - from synapse.api.filtering import Filter -+from synapse.api.ratelimiting import RequestRatelimiter - from synapse.events.utils import ( - SerializeEventConfig, - format_event_for_client_v2, -@@ -1660,6 +1661,25 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - self._message_handler = hs.get_message_handler() - self._storage_controllers = hs.get_storage_controllers() - -+ async def wait_ratelimit(ratelimiter: RequestRatelimiter, requester: Requester) -> bool: -+ """ -+ Wait until the ratelimiter allows the requester to proceed. -+ """ -+ ratelimit_hit = False -+ awaiting_ratelimit = False -+ while awaiting_ratelimit: -+ can_do_action, ratelimit_expiry = await ratelimiter.can_do_action(requester, update=False) -+ if not can_do_action: -+ # can_do_action returns an absolute timestamp, convert it to a relative time -+ time_to_sleep = ratelimit_expiry - ratelimiter.clock.time() -+ logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep) -+ await ratelimiter.clock.sleep(time_to_sleep) -+ ratelimit_hit = True -+ else: -+ awaiting_ratelimit = False -+ await ratelimiter.can_do_action(requester, update=True) -+ return ratelimit_hit -+ - async def on_POST( - self, request: SynapseRequest, room_identifier: str - ) -> Tuple[int, JsonDict]: -@@ -1695,7 +1715,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - - i = 0 - unpersisted_events = [] -- persistance_coroutines = [] -+ unpersisted_events_with_tasks = [] - depth = None - prev_event_id = None - -@@ -1716,19 +1736,8 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - event_dict["state_key"] = event_data["state_key"] - - # Explicitly handle rate limits in order to avoid compounding effects -- awaiting_ratelimit = False -- ratelimit_hit = False -- while awaiting_ratelimit: -- can_do_action, ratelimit_expiry = await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=False) -- if not can_do_action: -- # can_do_action returns an absolute timestamp, convert it to a relative time -- time_to_sleep = ratelimit_expiry - self._event_creation_handler.request_ratelimiter.clock.time() -- logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep) -- await self._event_creation_handler.request_ratelimiter.clock.sleep(time_to_sleep) -- ratelimit_hit = True -- else: -- awaiting_ratelimit = False -- await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=True) -+ ratelimit_hit = False # await self.wait_ratelimit(self._event_creation_handler.request_ratelimiter, requester) -+ - - event, unpersisted_context = await self._event_creation_handler.create_event( - requester, -@@ -1775,6 +1784,8 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) - unpersisted_events = [] - -+ logger.warning("bulk_send_events: Finished processing event %d in %d ms", i, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -+ logger.warning("bulk_send_events: Finished processing %d events", i) - # Finalize any remaining unpersisted events - if(len(unpersisted_events) > 0): - await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... --- -2.50.1 - diff --git a/packages/overlays/matrix-synapse/patches/0018-Use-create_and_send_new_client_events-for-bulk-sendi.patch b/packages/overlays/matrix-synapse/patches/0018-Use-create_and_send_new_client_events-for-bulk-sendi.patch new file mode 100644 index 0000000..2d42a9f --- /dev/null +++ b/packages/overlays/matrix-synapse/patches/0018-Use-create_and_send_new_client_events-for-bulk-sendi.patch @@ -0,0 +1,144 @@ +From 80672bddc8ed74d82d27a899d13e9de360f1df6e Mon Sep 17 00:00:00 2001 +From: Rory& +Date: Sat, 16 Aug 2025 21:37:56 +0200 +Subject: [PATCH 18/18] Use create_and_send_new_client_events for bulk sending + +--- + synapse/rest/client/room.py | 104 ++++++++++-------------------------- + 1 file changed, 27 insertions(+), 77 deletions(-) + +diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py +index d23c0d08ab..7699069e37 100644 +--- a/synapse/rest/client/room.py ++++ b/synapse/rest/client/room.py +@@ -1687,26 +1687,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + requester = await self._auth.get_user_by_req(request, allow_guest=False) + room_id, remote_room_hosts = await self.resolve_room_id(room_identifier) + +- force_sync_interval = parse_integer(request, "force_sync_interval", default=250) +- +- latest_events = await self._storage_controllers.main.get_latest_event_ids_in_room( +- room_id +- ) +- state_groups = await self._storage_controllers.state.get_state_group_for_events( +- #[ latest_events[len(latest_events) - 1] ] +- latest_events +- ) +- current_state_events = await self._message_handler.get_state_events( +- room_id=room_id, +- requester=requester, +- ) +- +- #state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} +- state_map = await self._storage_controllers.persistence._calculate_current_state(room_id) +- #state_map_evts = await self._storage_controllers.persistence.state_store.state( +- # room_id, +- # state_map, +- #) ++ #force_sync_interval = parse_integer(request, "force_sync_interval", default=250) + + events = ijson.items( + request.content, +@@ -1714,10 +1695,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + ) + + i = 0 +- unpersisted_events = [] +- unpersisted_events_with_tasks = [] +- depth = None +- prev_event_id = None ++ queued_event_dicts = [] + + for event_data in events: + current_index = i +@@ -1735,62 +1713,34 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): + if "state_key" in event_data: + event_dict["state_key"] = event_data["state_key"] + ++ queued_event_dicts.append(event_dict) + # Explicitly handle rate limits in order to avoid compounding effects + ratelimit_hit = False # await self.wait_ratelimit(self._event_creation_handler.request_ratelimiter, requester) + +- +- event, unpersisted_context = await self._event_creation_handler.create_event( +- requester, +- event_dict, +- for_batch=True, +- require_consent=False, +- state_map=state_map, +- depth=depth, +- prev_event_ids = [prev_event_id] if prev_event_id else None, +- state_event_ids = list(state_map.values()) if prev_event_id else None, +- current_state_group = state_groups.get(room_id, None) +- ) +- logger.warning("bulk_send_events: created event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) +- context = await unpersisted_context.persist(event) +- logger.warning("bulk_send_events: persisted event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) +- if event.depth is not None: +- depth = event.depth + 1 +- if event.event_id is not None: +- prev_event_id = event.event_id +- +- if event.is_state(): +- logger.warning("bulk_send_events: scanning state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) +- state_map_contains_event = state_map.get((event_dict["type"], event_dict["state_key"])) +- logger.warning("bulk_send_events: scanned state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) +- prev_event = None if state_map_contains_event is None else await self._event_creation_handler.deduplicate_state_event(event, context) +- logger.warning("bulk_send_events: deduplicated state event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) +- if prev_event is not None: +- logger.info( +- "Not bothering to persist state event %s duplicated by %s", +- event.event_id, +- prev_event.event_id, +- ) +- continue +- else: +- state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id +- logger.warning("bulk_send_events: Updated state_map! (State map contained entry: %s)", state_map_contains_event) +- +- unpersisted_events.append((event, context)) +- logger.warning("bulk_send_events: Persisted event %d after %d ms: %s", current_index, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms), event) +- +- if ratelimit_hit or len(unpersisted_events) >= force_sync_interval: +- logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events)) +- await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... +- await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) +- unpersisted_events = [] +- +- logger.warning("bulk_send_events: Finished processing event %d in %d ms", i, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) +- logger.warning("bulk_send_events: Finished processing %d events", i) +- # Finalize any remaining unpersisted events +- if(len(unpersisted_events) > 0): +- await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... +- await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) +- unpersisted_events = [] ++ #if ratelimit_hit: ++ # logger.warning("bulk_send_events: Rate limit hit after %d events", i) ++ # await self._event_creation_handler.create_and_send_new_client_events( ++ # requester, ++ # room_id, ++ # prev_event_id, ++ # event_dicts=queued_event_dicts, ++ # ratelimit=False, ++ # ignore_shadow_ban=True #TODO: remove ++ # ) ++ # #logger.warning("bulk_send_events: ", i) ++ # queued_event_dicts = [] ++ ++ latest_events: list = await self._storage_controllers.main.get_latest_event_ids_in_room(room_id) ++ prev_event_id = list(latest_events).pop() ++ await self._event_creation_handler.create_and_send_new_client_events( ++ requester, ++ room_id, ++ prev_event_id, ++ event_dicts=queued_event_dicts, ++ ratelimit=False, ++ ignore_shadow_ban=True #TODO: remove ++ ) ++ logger.warning("bulk_send_events: Finished processing %d events in %d ms", i, self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms) + + return 200, {} + +-- +2.51.0 + diff --git a/packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch b/packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch deleted file mode 100644 index 9df201b..0000000 --- a/packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch +++ /dev/null @@ -1,144 +0,0 @@ -From 6f7e3824c7af012ba8eb7bedce89704d8446b7b0 Mon Sep 17 00:00:00 2001 -From: Rory& -Date: Sat, 16 Aug 2025 21:37:56 +0200 -Subject: [PATCH 19/19] Use create_and_send_new_client_events for bulk sending - ---- - synapse/rest/client/room.py | 104 ++++++++++-------------------------- - 1 file changed, 27 insertions(+), 77 deletions(-) - -diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py -index d23c0d08ab..7699069e37 100644 ---- a/synapse/rest/client/room.py -+++ b/synapse/rest/client/room.py -@@ -1687,26 +1687,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - requester = await self._auth.get_user_by_req(request, allow_guest=False) - room_id, remote_room_hosts = await self.resolve_room_id(room_identifier) - -- force_sync_interval = parse_integer(request, "force_sync_interval", default=250) -- -- latest_events = await self._storage_controllers.main.get_latest_event_ids_in_room( -- room_id -- ) -- state_groups = await self._storage_controllers.state.get_state_group_for_events( -- #[ latest_events[len(latest_events) - 1] ] -- latest_events -- ) -- current_state_events = await self._message_handler.get_state_events( -- room_id=room_id, -- requester=requester, -- ) -- -- #state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events} -- state_map = await self._storage_controllers.persistence._calculate_current_state(room_id) -- #state_map_evts = await self._storage_controllers.persistence.state_store.state( -- # room_id, -- # state_map, -- #) -+ #force_sync_interval = parse_integer(request, "force_sync_interval", default=250) - - events = ijson.items( - request.content, -@@ -1714,10 +1695,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - ) - - i = 0 -- unpersisted_events = [] -- unpersisted_events_with_tasks = [] -- depth = None -- prev_event_id = None -+ queued_event_dicts = [] - - for event_data in events: - current_index = i -@@ -1735,62 +1713,34 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet): - if "state_key" in event_data: - event_dict["state_key"] = event_data["state_key"] - -+ queued_event_dicts.append(event_dict) - # Explicitly handle rate limits in order to avoid compounding effects - ratelimit_hit = False # await self.wait_ratelimit(self._event_creation_handler.request_ratelimiter, requester) - -- -- event, unpersisted_context = await self._event_creation_handler.create_event( -- requester, -- event_dict, -- for_batch=True, -- require_consent=False, -- state_map=state_map, -- depth=depth, -- prev_event_ids = [prev_event_id] if prev_event_id else None, -- state_event_ids = list(state_map.values()) if prev_event_id else None, -- current_state_group = state_groups.get(room_id, None) -- ) -- logger.warning("bulk_send_events: created event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -- context = await unpersisted_context.persist(event) -- logger.warning("bulk_send_events: persisted event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -- if event.depth is not None: -- depth = event.depth + 1 -- if event.event_id is not None: -- prev_event_id = event.event_id -- -- if event.is_state(): -- logger.warning("bulk_send_events: scanning state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -- state_map_contains_event = state_map.get((event_dict["type"], event_dict["state_key"])) -- logger.warning("bulk_send_events: scanned state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -- prev_event = None if state_map_contains_event is None else await self._event_creation_handler.deduplicate_state_event(event, context) -- logger.warning("bulk_send_events: deduplicated state event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -- if prev_event is not None: -- logger.info( -- "Not bothering to persist state event %s duplicated by %s", -- event.event_id, -- prev_event.event_id, -- ) -- continue -- else: -- state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id -- logger.warning("bulk_send_events: Updated state_map! (State map contained entry: %s)", state_map_contains_event) -- -- unpersisted_events.append((event, context)) -- logger.warning("bulk_send_events: Persisted event %d after %d ms: %s", current_index, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms), event) -- -- if ratelimit_hit or len(unpersisted_events) >= force_sync_interval: -- logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events)) -- await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... -- await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) -- unpersisted_events = [] -- -- logger.warning("bulk_send_events: Finished processing event %d in %d ms", i, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)) -- logger.warning("bulk_send_events: Finished processing %d events", i) -- # Finalize any remaining unpersisted events -- if(len(unpersisted_events) > 0): -- await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning... -- await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False) -- unpersisted_events = [] -+ #if ratelimit_hit: -+ # logger.warning("bulk_send_events: Rate limit hit after %d events", i) -+ # await self._event_creation_handler.create_and_send_new_client_events( -+ # requester, -+ # room_id, -+ # prev_event_id, -+ # event_dicts=queued_event_dicts, -+ # ratelimit=False, -+ # ignore_shadow_ban=True #TODO: remove -+ # ) -+ # #logger.warning("bulk_send_events: ", i) -+ # queued_event_dicts = [] -+ -+ latest_events: list = await self._storage_controllers.main.get_latest_event_ids_in_room(room_id) -+ prev_event_id = list(latest_events).pop() -+ await self._event_creation_handler.create_and_send_new_client_events( -+ requester, -+ room_id, -+ prev_event_id, -+ event_dicts=queued_event_dicts, -+ ratelimit=False, -+ ignore_shadow_ban=True #TODO: remove -+ ) -+ logger.warning("bulk_send_events: Finished processing %d events in %d ms", i, self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms) - - return 200, {} - --- -2.50.1 - -- cgit 1.5.1