From dc75fb7f0552e6a9903a7c173672c96610219ec0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 1 Sep 2021 10:27:58 -0500 Subject: Populate `rooms.creator` field for easy lookup (#10697) Part of https://github.com/matrix-org/synapse/pull/10566 - Fill in creator whenever we insert into the rooms table - Add background update to backfill any missing creator values --- tests/storage/databases/main/test_room.py | 98 +++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tests/storage/databases/main/test_room.py (limited to 'tests/storage') diff --git a/tests/storage/databases/main/test_room.py b/tests/storage/databases/main/test_room.py new file mode 100644 index 0000000000..ffee707153 --- /dev/null +++ b/tests/storage/databases/main/test_room.py @@ -0,0 +1,98 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.storage.databases.main.room import _BackgroundUpdates + +from tests.unittest import HomeserverTestCase + + +class RoomBackgroundUpdateStoreTestCase(HomeserverTestCase): + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + self.user_id = self.register_user("foo", "pass") + self.token = self.login("foo", "pass") + + def _generate_room(self) -> str: + room_id = self.helper.create_room_as(self.user_id, tok=self.token) + + return room_id + + def test_background_populate_rooms_creator_column(self): + """Test that the background update to populate the rooms creator column + works properly. + """ + + # Insert a room without the creator + room_id = self._generate_room() + self.get_success( + self.store.db_pool.simple_update( + table="rooms", + keyvalues={"room_id": room_id}, + updatevalues={"creator": None}, + desc="test", + ) + ) + + # Make sure the test is starting out with a room without a creator + room_creator_before = self.get_success( + self.store.db_pool.simple_select_one_onecol( + table="rooms", + keyvalues={"room_id": room_id}, + retcol="creator", + allow_none=True, + ) + ) + self.assertEqual(room_creator_before, None) + + # Insert and run the background update. + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": _BackgroundUpdates.POPULATE_ROOMS_CREATOR_COLUMN, + "progress_json": "{}", + }, + ) + ) + + # ... and tell the DataStore that it hasn't finished all updates yet + self.store.db_pool.updates._all_done = False + + # Now let's actually drive the updates to completion + while not self.get_success( + self.store.db_pool.updates.has_completed_background_updates() + ): + self.get_success( + self.store.db_pool.updates.do_next_background_update(100), by=0.1 + ) + + # Make sure the background update filled in the room creator + room_creator_after = self.get_success( + self.store.db_pool.simple_select_one_onecol( + table="rooms", + keyvalues={"room_id": room_id}, + retcol="creator", + allow_none=True, + ) + ) + self.assertEqual(room_creator_after, self.user_id) -- cgit 1.5.1 From 74f01e11c9e762101b834493a52da12538477e75 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Sep 2021 15:18:35 +0100 Subject: Skip handling of push actions for outlier events (#10780) Outlier events don't ever have push actions associated with them, so we can skip some expensive queries during event persistence. --- changelog.d/10780.misc | 1 + synapse/storage/databases/main/events.py | 21 +++++++++++++++++---- tests/storage/test_event_push_actions.py | 1 + 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 changelog.d/10780.misc (limited to 'tests/storage') diff --git a/changelog.d/10780.misc b/changelog.d/10780.misc new file mode 100644 index 0000000000..3b7acff03f --- /dev/null +++ b/changelog.d/10780.misc @@ -0,0 +1 @@ +Minor speed ups when joining large rooms over federation. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index f07e288056..14ada8a8b3 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1990,6 +1990,15 @@ class PersistEventsStore: events_and_context. """ + # Only non outlier events will have push actions associated with them, + # so let's filter them out. (This makes joining large rooms faster, as + # these queries took seconds to process all the state events). + non_outlier_events = [ + event + for event, _ in events_and_contexts + if not event.internal_metadata.is_outlier() + ] + sql = """ INSERT INTO event_push_actions ( room_id, event_id, user_id, actions, stream_ordering, @@ -2000,7 +2009,7 @@ class PersistEventsStore: WHERE event_id = ? """ - if events_and_contexts: + if non_outlier_events: txn.execute_batch( sql, ( @@ -2010,12 +2019,12 @@ class PersistEventsStore: event.depth, event.event_id, ) - for event, _ in events_and_contexts + for event in non_outlier_events ), ) room_to_event_ids: Dict[str, List[str]] = {} - for e, _ in events_and_contexts: + for e in non_outlier_events: room_to_event_ids.setdefault(e.room_id, []).append(e.event_id) for room_id, event_ids in room_to_event_ids.items(): @@ -2040,7 +2049,11 @@ class PersistEventsStore: # persisted. txn.execute_batch( "DELETE FROM event_push_actions_staging WHERE event_id = ?", - ((event.event_id,) for event, _ in all_events_and_contexts), + ( + (event.event_id,) + for event, _ in all_events_and_contexts + if not event.internal_metadata.is_outlier() + ), ) def _remove_push_actions_for_event_id_txn(self, txn, room_id, event_id): diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index 1930b37eda..bb5939ba4a 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -69,6 +69,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): event.room_id = room_id event.event_id = "$test:example.com" event.internal_metadata.stream_ordering = stream + event.internal_metadata.is_outlier.return_value = False event.depth = stream self.get_success( -- cgit 1.5.1