From 11e15d79b8a0af593fd9467e0cc7f8a9dfcb6c4f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 31 May 2023 13:59:56 +0000 Subject: Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. (#15693) * Add indices required to efficiently validate new foreign key constraints on stream_ordering * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --------- Signed-off-by: Olivier Wilkinson (reivilibre) --- synapse/storage/databases/state/bg_updates.py | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'synapse/storage/databases') diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 86eb1a8a08..5b8ba436d4 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -261,6 +261,16 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" + CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "current_state_events_stream_ordering_idx" + ) + ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "room_memberships_stream_ordering_idx" + ) + LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "local_current_membership_stream_ordering_idx" + ) + def __init__( self, database: DatabasePool, @@ -297,6 +307,27 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): replaces_index="state_group_edges_idx", ) + # These indices are needed to validate the foreign key constraint + # when events are deleted. + self.db_pool.updates.register_background_index_update( + self.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="current_state_events_stream_ordering_idx", + table="current_state_events", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + self.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="room_memberships_stream_ordering_idx", + table="room_memberships", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + self.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="local_current_membership_stream_ordering_idx", + table="local_current_membership", + columns=["event_stream_ordering"], + ) + async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: -- cgit 1.5.1 From 6d9e2fd8782a6610d6daf499d141e67f476b2f8c Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 31 May 2023 15:13:48 -0700 Subject: Speed up background jobs populate_full_user_id_user_filters and populate_full_user_id_profiles (#15700) --- changelog.d/15700.misc | 1 + synapse/storage/databases/main/filtering.py | 2 +- synapse/storage/databases/main/profile.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/15700.misc (limited to 'synapse/storage/databases') diff --git a/changelog.d/15700.misc b/changelog.d/15700.misc new file mode 100644 index 0000000000..e96bc681aa --- /dev/null +++ b/changelog.d/15700.misc @@ -0,0 +1 @@ +Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. \ No newline at end of file diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index da31eb44dc..f777777cbf 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -71,7 +71,7 @@ class FilteringWorkerStore(SQLBaseStore): SELECT user_id FROM user_filters WHERE user_id > ? ORDER BY user_id - LIMIT 1 OFFSET 50 + LIMIT 1 OFFSET 1000 """ txn.execute(sql, (lower_bound_id,)) res = txn.fetchone() diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 65c92bef51..21d54c7a7a 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -65,7 +65,7 @@ class ProfileWorkerStore(SQLBaseStore): SELECT user_id FROM profiles WHERE user_id > ? ORDER BY user_id - LIMIT 1 OFFSET 50 + LIMIT 1 OFFSET 1000 """ txn.execute(sql, (lower_bound_id,)) res = txn.fetchone() -- cgit 1.5.1