From 11e15d79b8a0af593fd9467e0cc7f8a9dfcb6c4f Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 31 May 2023 13:59:56 +0000 Subject: Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. (#15693) * Add indices required to efficiently validate new foreign key constraints on stream_ordering * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --------- Signed-off-by: Olivier Wilkinson (reivilibre) --- synapse/storage/databases/state/bg_updates.py | 31 ++++++++++++++++++++++ .../77/14bg_indices_event_stream_ordering.sql | 20 ++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql (limited to 'synapse/storage') diff --git a/synapse/storage/databases/state/bg_updates.py b/synapse/storage/databases/state/bg_updates.py index 86eb1a8a08..5b8ba436d4 100644 --- a/synapse/storage/databases/state/bg_updates.py +++ b/synapse/storage/databases/state/bg_updates.py @@ -261,6 +261,16 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): STATE_GROUPS_ROOM_INDEX_UPDATE_NAME = "state_groups_room_id_idx" STATE_GROUP_EDGES_UNIQUE_INDEX_UPDATE_NAME = "state_group_edges_unique_idx" + CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "current_state_events_stream_ordering_idx" + ) + ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "room_memberships_stream_ordering_idx" + ) + LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME = ( + "local_current_membership_stream_ordering_idx" + ) + def __init__( self, database: DatabasePool, @@ -297,6 +307,27 @@ class StateBackgroundUpdateStore(StateGroupBackgroundUpdateStore): replaces_index="state_group_edges_idx", ) + # These indices are needed to validate the foreign key constraint + # when events are deleted. + self.db_pool.updates.register_background_index_update( + self.CURRENT_STATE_EVENTS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="current_state_events_stream_ordering_idx", + table="current_state_events", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + self.ROOM_MEMBERSHIPS_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="room_memberships_stream_ordering_idx", + table="room_memberships", + columns=["event_stream_ordering"], + ) + self.db_pool.updates.register_background_index_update( + self.LOCAL_CURRENT_MEMBERSHIP_STREAM_ORDERING_INDEX_UPDATE_NAME, + index_name="local_current_membership_stream_ordering_idx", + table="local_current_membership", + columns=["event_stream_ordering"], + ) + async def _background_deduplicate_state( self, progress: dict, batch_size: int ) -> int: diff --git a/synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql b/synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql new file mode 100644 index 0000000000..ec8cd522ec --- /dev/null +++ b/synapse/storage/schema/main/delta/77/14bg_indices_event_stream_ordering.sql @@ -0,0 +1,20 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (ordering, update_name, progress_json) + VALUES + (7714, 'current_state_events_stream_ordering_idx', '{}'), + (7714, 'local_current_membership_stream_ordering_idx', '{}'), + (7714, 'room_memberships_stream_ordering_idx', '{}'); -- cgit 1.4.1 From 6d9e2fd8782a6610d6daf499d141e67f476b2f8c Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 31 May 2023 15:13:48 -0700 Subject: Speed up background jobs populate_full_user_id_user_filters and populate_full_user_id_profiles (#15700) --- changelog.d/15700.misc | 1 + synapse/storage/databases/main/filtering.py | 2 +- synapse/storage/databases/main/profile.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/15700.misc (limited to 'synapse/storage') diff --git a/changelog.d/15700.misc b/changelog.d/15700.misc new file mode 100644 index 0000000000..e96bc681aa --- /dev/null +++ b/changelog.d/15700.misc @@ -0,0 +1 @@ +Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. \ No newline at end of file diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index da31eb44dc..f777777cbf 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -71,7 +71,7 @@ class FilteringWorkerStore(SQLBaseStore): SELECT user_id FROM user_filters WHERE user_id > ? ORDER BY user_id - LIMIT 1 OFFSET 50 + LIMIT 1 OFFSET 1000 """ txn.execute(sql, (lower_bound_id,)) res = txn.fetchone() diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 65c92bef51..21d54c7a7a 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -65,7 +65,7 @@ class ProfileWorkerStore(SQLBaseStore): SELECT user_id FROM profiles WHERE user_id > ? ORDER BY user_id - LIMIT 1 OFFSET 50 + LIMIT 1 OFFSET 1000 """ txn.execute(sql, (lower_bound_id,)) res = txn.fetchone() -- cgit 1.4.1