From fc2286e9d402e8f94ed2892918e8c13e21a70293 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 12 Oct 2022 18:11:00 +0100 Subject: Avoid checking the event cache when backfilling events --- synapse/handlers/federation_event.py | 46 ++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 13 deletions(-) (limited to 'synapse') diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index da319943cc..a84c65fd31 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -792,9 +792,41 @@ class FederationEventHandler: ], ) + # Check if we already any of these have these events. + # Note: we currently make a lookup in the database directly here rather than + # checking the event cache, due to: + # https://github.com/matrix-org/synapse/issues/13476 + existing_events_map = await self._store._get_events_from_db( + [event.event_id for event in events] + ) + + new_events = [] + for event in events: + event_id = event.event_id + + # If we've already seen this event ID... + if event_id in existing_events_map: + existing_event = existing_events_map[event_id] + + # ...and the event itself was not previously stored as an outlier... + if not existing_event.event.internal_metadata.is_outlier(): + # ...then there's no need to persist it. We have it already. + logger.info( + "_process_pulled_event: Ignoring received event %s which we " + "have already seen", + event.event_id, + ) + + # While we have seen this event before, it was stored as an outlier. + # We'll now persist it as a non-outlier. + logger.info("De-outliering event %s", event_id) + + # Continue on with the events that are new to us. + new_events.append(event) + # We want to sort these by depth so we process them and # tell clients about them in order. - sorted_events = sorted(events, key=lambda x: x.depth) + sorted_events = sorted(new_events, key=lambda x: x.depth) for ev in sorted_events: with nested_logging_context(ev.event_id): await self._process_pulled_event(origin, ev, backfilled=backfilled) @@ -846,18 +878,6 @@ class FederationEventHandler: event_id = event.event_id - existing = await self._store.get_event( - event_id, allow_none=True, allow_rejected=True - ) - if existing: - if not existing.internal_metadata.is_outlier(): - logger.info( - "_process_pulled_event: Ignoring received event %s which we have already seen", - event_id, - ) - return - logger.info("De-outliering event %s", event_id) - try: self._sanity_check_event(event) except SynapseError as err: -- cgit 1.5.1 From 74812f9e18421baa7d53694ec00cd64cc64c7a0a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 18 Oct 2022 09:23:37 +0100 Subject: Add missing 'continue' --- synapse/handlers/federation_event.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index a84c65fd31..b94641a87d 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -816,6 +816,7 @@ class FederationEventHandler: "have already seen", event.event_id, ) + continue # While we have seen this event before, it was stored as an outlier. # We'll now persist it as a non-outlier. -- cgit 1.5.1 From a9709f07825af59fb379f3d8bffa47f81cc909f4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 18 Oct 2022 09:23:50 +0100 Subject: Fix up docstring --- synapse/storage/databases/main/events_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 7cdc9fe98f..221864d9cc 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -374,7 +374,7 @@ class EventsWorkerStore(SQLBaseStore): If there is a mismatch, behave as per allow_none. Returns: - The event, or None if the event was not found. + The event, or None if the event was not found and allow_none is `True`. """ if not isinstance(event_id, str): raise TypeError("Invalid event event_id %r" % (event_id,)) -- cgit 1.5.1