diff --git a/flake.nix b/flake.nix
index 8dfa93f..5968feb 100755
--- a/flake.nix
+++ b/flake.nix
@@ -411,6 +411,7 @@
packages.redpanda-connect = (pkgs.callPackage ./packages/redpanda-connect/default.nix { });
packages.matrix-synapse-unwrapped-patched = (self.inputs.nixpkgs-override-synapse or self.inputs.nixpkgs-master).legacyPackages.${system}.matrix-synapse-unwrapped.overrideAttrs (oldAttrs: rec {
+ doCheck = false;
patches =
(if oldAttrs ? patches then oldAttrs.patches else [ ])
++ pkgs.lib.map (path: ./packages/overlays/matrix-synapse/patches/${path}) (builtins.attrNames (builtins.readDir ./packages/overlays/matrix-synapse/patches));
diff --git a/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch b/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch
index 7c5cf47..2b971cf 100644
--- a/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch
+++ b/packages/overlays/matrix-synapse/patches/0001-Fast-auth-links.patch
@@ -1,7 +1,7 @@
From a4c542d11e60728426b85b8a59a7c02062930ede Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Tue, 22 Jul 2025 05:07:01 +0200
-Subject: [PATCH 01/14] Fast auth links
+Subject: [PATCH 01/19] Fast auth links
---
synapse/storage/database.py | 43 +++++++++++++++++++
diff --git a/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch b/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch
index 568c3ec..b44a6ea 100644
--- a/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch
+++ b/packages/overlays/matrix-synapse/patches/0002-Hotfix-ignore-rejected-events-in-delayed_events.patch
@@ -1,7 +1,7 @@
From 46e8ebbb8253470489be0c5a9a481eea86507f79 Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Sun, 20 Apr 2025 00:30:29 +0200
-Subject: [PATCH 02/14] Hotfix: ignore rejected events in delayed_events
+Subject: [PATCH 02/19] Hotfix: ignore rejected events in delayed_events
---
synapse/handlers/delayed_events.py | 7 ++++++-
diff --git a/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch b/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch
index a5b5f43..88b0db4 100644
--- a/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch
+++ b/packages/overlays/matrix-synapse/patches/0003-Add-too-much-logging-to-room-summary-over-federation.patch
@@ -1,7 +1,7 @@
From 370ea67df812fc3174bf480ebe12068b14922d90 Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Wed, 23 Apr 2025 17:53:52 +0200
-Subject: [PATCH 03/14] Add too much logging to room summary over federation
+Subject: [PATCH 03/19] Add too much logging to room summary over federation
Signed-off-by: Rory& <root@rory.gay>
---
diff --git a/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch b/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch
index f5db38e..4741c42 100644
--- a/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch
+++ b/packages/overlays/matrix-synapse/patches/0004-Log-entire-room-if-accessibility-check-fails.patch
@@ -1,7 +1,7 @@
From 5c0e74cb20848a62e9921e697249d0cb1d6df035 Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Wed, 23 Apr 2025 18:24:57 +0200
-Subject: [PATCH 04/14] Log entire room if accessibility check fails
+Subject: [PATCH 04/19] Log entire room if accessibility check fails
Signed-off-by: Rory& <root@rory.gay>
---
diff --git a/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch
index 401ff4d..9158bff 100644
--- a/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch
+++ b/packages/overlays/matrix-synapse/patches/0005-Log-policy-server-rejected-events.patch
@@ -1,7 +1,7 @@
From 34afe28e5c91d72f449cd453944a7763b6cc5c6b Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Tue, 27 May 2025 05:21:46 +0200
-Subject: [PATCH 05/14] Log policy server rejected events
+Subject: [PATCH 05/19] Log policy server rejected events
---
synapse/handlers/room_policy.py | 7 +++++++
diff --git a/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch b/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch
index e0ee048..a2f52f7 100644
--- a/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch
+++ b/packages/overlays/matrix-synapse/patches/0006-Use-parse_boolean-for-unredacted-content.patch
@@ -1,7 +1,7 @@
From b8ac66b50a38d38a34e4d5f003d39674eaffe77f Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Tue, 27 May 2025 06:14:26 +0200
-Subject: [PATCH 06/14] Use parse_boolean for unredacted content
+Subject: [PATCH 06/19] Use parse_boolean for unredacted content
---
synapse/rest/client/room.py | 5 ++---
diff --git a/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch b/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch
index 5b20b06..74195eb 100644
--- a/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch
+++ b/packages/overlays/matrix-synapse/patches/0007-Expose-tombstone-in-room-admin-api.patch
@@ -1,7 +1,7 @@
From 09eebe99f37c523fe1de000fbc7fb3dc692faee7 Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Tue, 27 May 2025 06:37:52 +0200
-Subject: [PATCH 07/14] Expose tombstone in room admin api
+Subject: [PATCH 07/19] Expose tombstone in room admin api
---
synapse/rest/admin/rooms.py | 5 ++++
diff --git a/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch b/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch
index f0051e6..d607b7c 100644
--- a/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch
+++ b/packages/overlays/matrix-synapse/patches/0008-fix-Always-recheck-messages-pagination-data-if-a-bac.patch
@@ -1,7 +1,7 @@
From 0ed11113c980e5edfd79c8c0f78b245adf823ac2 Mon Sep 17 00:00:00 2001
From: Jason Little <j.little@famedly.com>
Date: Wed, 30 Apr 2025 09:29:42 -0500
-Subject: [PATCH 08/14] fix: Always recheck `/messages` pagination data if a
+Subject: [PATCH 08/19] fix: Always recheck `/messages` pagination data if a
backfill might have been needed (#28)
---
diff --git a/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch b/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch
index f0ee50e..f2e9695 100644
--- a/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch
+++ b/packages/overlays/matrix-synapse/patches/0009-Fix-pagination-with-large-gaps-of-rejected-events.patch
@@ -1,7 +1,7 @@
From 57743dad758b46cda09f69ee44c99cbd649f873e Mon Sep 17 00:00:00 2001
From: Nicolas Werner <nicolas.werner@hotmail.de>
Date: Sun, 8 Jun 2025 23:14:31 +0200
-Subject: [PATCH 09/14] Fix pagination with large gaps of rejected events
+Subject: [PATCH 09/19] Fix pagination with large gaps of rejected events
---
synapse/handlers/pagination.py | 13 +++++++++++--
diff --git a/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch b/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch
index 3db7c75..2b35d06 100644
--- a/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch
+++ b/packages/overlays/matrix-synapse/patches/0010-Fix-nix-flake.patch
@@ -1,7 +1,7 @@
From 2f4533a3c26c62b2e287c3e5a284ca8ee1042546 Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Mon, 9 Jun 2025 17:38:34 +0200
-Subject: [PATCH 10/14] Fix nix flake
+Subject: [PATCH 10/19] Fix nix flake
---
flake.lock | 58 +++++++++++++++++++-----------------------------------
diff --git a/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch b/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch
index 0bd53e3..8f8b051 100644
--- a/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch
+++ b/packages/overlays/matrix-synapse/patches/0011-Fix-gitignore-to-ignore-.venv.patch
@@ -1,7 +1,7 @@
From 2fb09d660565b39cdd1077f0e29580c72a8bbb47 Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Mon, 9 Jun 2025 17:46:10 +0200
-Subject: [PATCH 11/14] Fix gitignore to ignore .venv
+Subject: [PATCH 11/19] Fix gitignore to ignore .venv
---
.gitignore | 1 +
diff --git a/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch b/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch
index 545933b..15ee0a3 100644
--- a/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch
+++ b/packages/overlays/matrix-synapse/patches/0012-Devenv-use-postgres-17.patch
@@ -1,7 +1,7 @@
From 80335192cafb2b3b33a8822ba3f720d400853d8c Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Fri, 25 Jul 2025 08:25:28 +0200
-Subject: [PATCH 12/14] Devenv: use postgres 17
+Subject: [PATCH 12/19] Devenv: use postgres 17
---
flake.nix | 1 +
diff --git a/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch b/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch
index be4e1be..ef9dbd6 100644
--- a/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch
+++ b/packages/overlays/matrix-synapse/patches/0013-RequestRatelimiter-expose-can_do_action.patch
@@ -1,7 +1,7 @@
From 49d21ee7150580474879b8c9e149c273dd24ad0e Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Fri, 25 Jul 2025 08:26:15 +0200
-Subject: [PATCH 13/14] RequestRatelimiter: expose can_do_action
+Subject: [PATCH 13/19] RequestRatelimiter: expose can_do_action
---
synapse/api/ratelimiting.py | 75 +++++++++++++++++++++++++++++++++++++
diff --git a/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch b/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch
index b051bf7..daf1e0f 100644
--- a/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch
+++ b/packages/overlays/matrix-synapse/patches/0014-Add-bulk-send-events-endpoint.patch
@@ -1,7 +1,7 @@
From c48c105727590a1a0a0e9fd0ac3422acead00e4b Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Sat, 26 Jul 2025 09:50:56 +0200
-Subject: [PATCH 14/14] Add bulk send events endpoint
+Subject: [PATCH 14/19] Add bulk send events endpoint
---
synapse/rest/client/capabilities.py | 3 +
diff --git a/packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch b/packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch
new file mode 100644
index 0000000..fcd14aa
--- /dev/null
+++ b/packages/overlays/matrix-synapse/patches/0015-Temporarily-disable-go-in-flake.patch
@@ -0,0 +1,25 @@
+From 8c1651a65f6d45bfb242762b6839de0c3a1728fa Mon Sep 17 00:00:00 2001
+From: Rory& <root@rory.gay>
+Date: Sat, 16 Aug 2025 20:18:45 +0200
+Subject: [PATCH 15/19] Temporarily disable go in flake
+
+---
+ flake.nix | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/flake.nix b/flake.nix
+index cc41490a41..291e81814d 100644
+--- a/flake.nix
++++ b/flake.nix
+@@ -151,7 +151,7 @@
+ # * Perl is needed to run the SyTest test suite.
+ # * Rust is used for developing and running Synapse.
+ # It is installed manually with `packages` above.
+- languages.go.enable = true;
++ #languages.go.enable = true;
+ languages.perl.enable = true;
+
+ # Postgres is needed to run Synapse with postgres support and
+--
+2.50.1
+
diff --git a/packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch b/packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch
new file mode 100644
index 0000000..5dc2321
--- /dev/null
+++ b/packages/overlays/matrix-synapse/patches/0016-Clarify-asserts.patch
@@ -0,0 +1,25 @@
+From 79d7af8028bfa3c512f0cb57600ca2b94e0666c1 Mon Sep 17 00:00:00 2001
+From: Rory& <root@rory.gay>
+Date: Sat, 16 Aug 2025 20:19:08 +0200
+Subject: [PATCH 16/19] Clarify asserts
+
+---
+ synapse/handlers/message.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
+index fff46b640b..2a5595c7b5 100644
+--- a/synapse/handlers/message.py
++++ b/synapse/handlers/message.py
+@@ -1244,7 +1244,7 @@ class EventCreationHandler:
+ if state_event_ids is not None:
+ # Do a quick check to make sure that prev_event_ids is present to
+ # make the type-checking around `builder.build` happy.
+- assert prev_event_ids is not None
++ assert prev_event_ids is not None, "create_new_client_event called with state_event_ids but no prev_event_ids"
+
+ temp_event = await builder.build(
+ prev_event_ids=prev_event_ids,
+--
+2.50.1
+
diff --git a/packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch b/packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch
new file mode 100644
index 0000000..e43f1a4
--- /dev/null
+++ b/packages/overlays/matrix-synapse/patches/0017-Try-to-optimise-bulk-sending-some-more.patch
@@ -0,0 +1,117 @@
+From a5d49077038cf0e7235711cea3e756d5e96627d4 Mon Sep 17 00:00:00 2001
+From: Rory& <root@rory.gay>
+Date: Sat, 16 Aug 2025 20:19:26 +0200
+Subject: [PATCH 17/19] Try to optimise bulk sending some more
+
+---
+ synapse/rest/client/room.py | 42 +++++++++++++++++++++++++++++++++----
+ 1 file changed, 38 insertions(+), 4 deletions(-)
+
+diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
+index d054d87595..304999aa5d 100644
+--- a/synapse/rest/client/room.py
++++ b/synapse/rest/client/room.py
+@@ -1658,6 +1658,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ self._auth = hs.get_auth()
+ self._event_creation_handler = hs.get_event_creation_handler()
+ self._message_handler = hs.get_message_handler()
++ self._storage_controllers = hs.get_storage_controllers()
+
+ async def on_POST(
+ self, request: SynapseRequest, room_identifier: str
+@@ -1668,12 +1669,24 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+
+ force_sync_interval = parse_integer(request, "force_sync_interval", default=250)
+
++ latest_events = await self._storage_controllers.main.get_latest_event_ids_in_room(
++ room_id
++ )
++ state_groups = await self._storage_controllers.state.get_state_group_for_events(
++ #[ latest_events[len(latest_events) - 1] ]
++ latest_events
++ )
+ current_state_events = await self._message_handler.get_state_events(
+ room_id=room_id,
+ requester=requester,
+ )
+
+- state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events}
++ #state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events}
++ state_map = await self._storage_controllers.persistence._calculate_current_state(room_id)
++ #state_map_evts = await self._storage_controllers.persistence.state_store.state(
++ # room_id,
++ # state_map,
++ #)
+
+ events = ijson.items(
+ request.content,
+@@ -1682,10 +1695,14 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+
+ i = 0
+ unpersisted_events = []
++ persistance_coroutines = []
++ depth = None
++ prev_event_id = None
+
+ for event_data in events:
+ current_index = i
+ i += 1
++ current_time_ms = self._event_creation_handler.request_ratelimiter.clock.time_msec()
+ logger.info("bulk_send_events: Processing event %d: %s", current_index, event_data)
+
+ event_dict: JsonDict = {
+@@ -1717,12 +1734,27 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ requester,
+ event_dict,
+ for_batch=True,
++ require_consent=False,
+ state_map=state_map,
++ depth=depth,
++ prev_event_ids = [prev_event_id] if prev_event_id else None,
++ state_event_ids = list(state_map.values()) if prev_event_id else None,
++ current_state_group = state_groups.get(room_id, None)
+ )
++ logger.warning("bulk_send_events: created event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+ context = await unpersisted_context.persist(event)
++ logger.warning("bulk_send_events: persisted event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
++ if event.depth is not None:
++ depth = event.depth + 1
++ if event.event_id is not None:
++ prev_event_id = event.event_id
+
+ if event.is_state():
+- prev_event = await self._event_creation_handler.deduplicate_state_event(event, context)
++ logger.warning("bulk_send_events: scanning state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
++ state_map_contains_event = state_map.get((event_dict["type"], event_dict["state_key"]))
++ logger.warning("bulk_send_events: scanned state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
++ prev_event = None if state_map_contains_event is None else await self._event_creation_handler.deduplicate_state_event(event, context)
++ logger.warning("bulk_send_events: deduplicated state event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+ if prev_event is not None:
+ logger.info(
+ "Not bothering to persist state event %s duplicated by %s",
+@@ -1732,18 +1764,20 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ continue
+ else:
+ state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id
+- logger.warning("bulk_send_events: Updated state_map!")
++ logger.warning("bulk_send_events: Updated state_map! (State map contained entry: %s)", state_map_contains_event)
+
+ unpersisted_events.append((event, context))
+- logger.warning("bulk_send_events: Persisted event %d: %s", current_index, event)
++ logger.warning("bulk_send_events: Persisted event %d after %d ms: %s", current_index, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms), event)
+
+ if ratelimit_hit or len(unpersisted_events) >= force_sync_interval:
+ logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events))
++ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning...
+ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False)
+ unpersisted_events = []
+
+ # Finalize any remaining unpersisted events
+ if(len(unpersisted_events) > 0):
++ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning...
+ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False)
+ unpersisted_events = []
+
+--
+2.50.1
+
diff --git a/packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch b/packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch
new file mode 100644
index 0000000..6ada1d8
--- /dev/null
+++ b/packages/overlays/matrix-synapse/patches/0018-Further-optimisation-attempts.patch
@@ -0,0 +1,90 @@
+From d9df11ecb0942fd7f3ba317ba743a9e61ea64a94 Mon Sep 17 00:00:00 2001
+From: Rory& <root@rory.gay>
+Date: Sat, 16 Aug 2025 21:02:57 +0200
+Subject: [PATCH 18/19] Further optimisation attempts
+
+---
+ synapse/rest/client/room.py | 39 ++++++++++++++++++++++++-------------
+ 1 file changed, 25 insertions(+), 14 deletions(-)
+
+diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
+index 304999aa5d..d23c0d08ab 100644
+--- a/synapse/rest/client/room.py
++++ b/synapse/rest/client/room.py
+@@ -46,6 +46,7 @@ from synapse.api.errors import (
+ UnredactedContentDeletedError,
+ )
+ from synapse.api.filtering import Filter
++from synapse.api.ratelimiting import RequestRatelimiter
+ from synapse.events.utils import (
+ SerializeEventConfig,
+ format_event_for_client_v2,
+@@ -1660,6 +1661,25 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ self._message_handler = hs.get_message_handler()
+ self._storage_controllers = hs.get_storage_controllers()
+
++ async def wait_ratelimit(ratelimiter: RequestRatelimiter, requester: Requester) -> bool:
++ """
++ Wait until the ratelimiter allows the requester to proceed.
++ """
++ ratelimit_hit = False
++ awaiting_ratelimit = False
++ while awaiting_ratelimit:
++ can_do_action, ratelimit_expiry = await ratelimiter.can_do_action(requester, update=False)
++ if not can_do_action:
++ # can_do_action returns an absolute timestamp, convert it to a relative time
++ time_to_sleep = ratelimit_expiry - ratelimiter.clock.time()
++ logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep)
++ await ratelimiter.clock.sleep(time_to_sleep)
++ ratelimit_hit = True
++ else:
++ awaiting_ratelimit = False
++ await ratelimiter.can_do_action(requester, update=True)
++ return ratelimit_hit
++
+ async def on_POST(
+ self, request: SynapseRequest, room_identifier: str
+ ) -> Tuple[int, JsonDict]:
+@@ -1695,7 +1715,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+
+ i = 0
+ unpersisted_events = []
+- persistance_coroutines = []
++ unpersisted_events_with_tasks = []
+ depth = None
+ prev_event_id = None
+
+@@ -1716,19 +1736,8 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ event_dict["state_key"] = event_data["state_key"]
+
+ # Explicitly handle rate limits in order to avoid compounding effects
+- awaiting_ratelimit = False
+- ratelimit_hit = False
+- while awaiting_ratelimit:
+- can_do_action, ratelimit_expiry = await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=False)
+- if not can_do_action:
+- # can_do_action returns an absolute timestamp, convert it to a relative time
+- time_to_sleep = ratelimit_expiry - self._event_creation_handler.request_ratelimiter.clock.time()
+- logger.warning("bulk_send_events: Got rate limited in bulk sending events, waiting %ds", time_to_sleep)
+- await self._event_creation_handler.request_ratelimiter.clock.sleep(time_to_sleep)
+- ratelimit_hit = True
+- else:
+- awaiting_ratelimit = False
+- await self._event_creation_handler.request_ratelimiter.can_do_action(requester, update=True)
++ ratelimit_hit = False # await self.wait_ratelimit(self._event_creation_handler.request_ratelimiter, requester)
++
+
+ event, unpersisted_context = await self._event_creation_handler.create_event(
+ requester,
+@@ -1775,6 +1784,8 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False)
+ unpersisted_events = []
+
++ logger.warning("bulk_send_events: Finished processing event %d in %d ms", i, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
++ logger.warning("bulk_send_events: Finished processing %d events", i)
+ # Finalize any remaining unpersisted events
+ if(len(unpersisted_events) > 0):
+ await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning...
+--
+2.50.1
+
diff --git a/packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch b/packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch
new file mode 100644
index 0000000..9df201b
--- /dev/null
+++ b/packages/overlays/matrix-synapse/patches/0019-Use-create_and_send_new_client_events-for-bulk-sendi.patch
@@ -0,0 +1,144 @@
+From 6f7e3824c7af012ba8eb7bedce89704d8446b7b0 Mon Sep 17 00:00:00 2001
+From: Rory& <root@rory.gay>
+Date: Sat, 16 Aug 2025 21:37:56 +0200
+Subject: [PATCH 19/19] Use create_and_send_new_client_events for bulk sending
+
+---
+ synapse/rest/client/room.py | 104 ++++++++++--------------------------
+ 1 file changed, 27 insertions(+), 77 deletions(-)
+
+diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
+index d23c0d08ab..7699069e37 100644
+--- a/synapse/rest/client/room.py
++++ b/synapse/rest/client/room.py
+@@ -1687,26 +1687,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ requester = await self._auth.get_user_by_req(request, allow_guest=False)
+ room_id, remote_room_hosts = await self.resolve_room_id(room_identifier)
+
+- force_sync_interval = parse_integer(request, "force_sync_interval", default=250)
+-
+- latest_events = await self._storage_controllers.main.get_latest_event_ids_in_room(
+- room_id
+- )
+- state_groups = await self._storage_controllers.state.get_state_group_for_events(
+- #[ latest_events[len(latest_events) - 1] ]
+- latest_events
+- )
+- current_state_events = await self._message_handler.get_state_events(
+- room_id=room_id,
+- requester=requester,
+- )
+-
+- #state_map = {(event["type"], event.get("state_key", "")): event.get("event_id") for event in current_state_events}
+- state_map = await self._storage_controllers.persistence._calculate_current_state(room_id)
+- #state_map_evts = await self._storage_controllers.persistence.state_store.state(
+- # room_id,
+- # state_map,
+- #)
++ #force_sync_interval = parse_integer(request, "force_sync_interval", default=250)
+
+ events = ijson.items(
+ request.content,
+@@ -1714,10 +1695,7 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ )
+
+ i = 0
+- unpersisted_events = []
+- unpersisted_events_with_tasks = []
+- depth = None
+- prev_event_id = None
++ queued_event_dicts = []
+
+ for event_data in events:
+ current_index = i
+@@ -1735,62 +1713,34 @@ class RoomBulkSendEventRestServlet(ResolveRoomIdMixin, RestServlet):
+ if "state_key" in event_data:
+ event_dict["state_key"] = event_data["state_key"]
+
++ queued_event_dicts.append(event_dict)
+ # Explicitly handle rate limits in order to avoid compounding effects
+ ratelimit_hit = False # await self.wait_ratelimit(self._event_creation_handler.request_ratelimiter, requester)
+
+-
+- event, unpersisted_context = await self._event_creation_handler.create_event(
+- requester,
+- event_dict,
+- for_batch=True,
+- require_consent=False,
+- state_map=state_map,
+- depth=depth,
+- prev_event_ids = [prev_event_id] if prev_event_id else None,
+- state_event_ids = list(state_map.values()) if prev_event_id else None,
+- current_state_group = state_groups.get(room_id, None)
+- )
+- logger.warning("bulk_send_events: created event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+- context = await unpersisted_context.persist(event)
+- logger.warning("bulk_send_events: persisted event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+- if event.depth is not None:
+- depth = event.depth + 1
+- if event.event_id is not None:
+- prev_event_id = event.event_id
+-
+- if event.is_state():
+- logger.warning("bulk_send_events: scanning state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+- state_map_contains_event = state_map.get((event_dict["type"], event_dict["state_key"]))
+- logger.warning("bulk_send_events: scanned state map after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+- prev_event = None if state_map_contains_event is None else await self._event_creation_handler.deduplicate_state_event(event, context)
+- logger.warning("bulk_send_events: deduplicated state event after %s ms", (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+- if prev_event is not None:
+- logger.info(
+- "Not bothering to persist state event %s duplicated by %s",
+- event.event_id,
+- prev_event.event_id,
+- )
+- continue
+- else:
+- state_map[(event_dict["type"], event_dict["state_key"])] = event.event_id
+- logger.warning("bulk_send_events: Updated state_map! (State map contained entry: %s)", state_map_contains_event)
+-
+- unpersisted_events.append((event, context))
+- logger.warning("bulk_send_events: Persisted event %d after %d ms: %s", current_index, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms), event)
+-
+- if ratelimit_hit or len(unpersisted_events) >= force_sync_interval:
+- logger.warning("bulk_send_events: Hit rate limit or max batch size, sending %d events", len(unpersisted_events))
+- await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning...
+- await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False)
+- unpersisted_events = []
+-
+- logger.warning("bulk_send_events: Finished processing event %d in %d ms", i, (self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms))
+- logger.warning("bulk_send_events: Finished processing %d events", i)
+- # Finalize any remaining unpersisted events
+- if(len(unpersisted_events) > 0):
+- await self._event_creation_handler.cache_joined_hosts_for_events(unpersisted_events) # Gets rid of a warning...
+- await self._event_creation_handler.handle_new_client_event(requester, unpersisted_events, ratelimit=False)
+- unpersisted_events = []
++ #if ratelimit_hit:
++ # logger.warning("bulk_send_events: Rate limit hit after %d events", i)
++ # await self._event_creation_handler.create_and_send_new_client_events(
++ # requester,
++ # room_id,
++ # prev_event_id,
++ # event_dicts=queued_event_dicts,
++ # ratelimit=False,
++ # ignore_shadow_ban=True #TODO: remove
++ # )
++ # #logger.warning("bulk_send_events: ", i)
++ # queued_event_dicts = []
++
++ latest_events: list = await self._storage_controllers.main.get_latest_event_ids_in_room(room_id)
++ prev_event_id = list(latest_events).pop()
++ await self._event_creation_handler.create_and_send_new_client_events(
++ requester,
++ room_id,
++ prev_event_id,
++ event_dicts=queued_event_dicts,
++ ratelimit=False,
++ ignore_shadow_ban=True #TODO: remove
++ )
++ logger.warning("bulk_send_events: Finished processing %d events in %d ms", i, self._event_creation_handler.request_ratelimiter.clock.time_msec() - current_time_ms)
+
+ return 200, {}
+
+--
+2.50.1
+
|
