diff options
| author | Amber Brown <hawkowl@atleastfornow.net> | 2019-03-21 03:06:36 +1100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-03-21 03:06:36 +1100 |
| commit | 4d53017432e05da621a13b3fe4d9e67108f856fd (patch) | |
| tree | f3b4d292f3bfb02d3a0adc55ba67552d02a44ab8 | |
| parent | Add a config option for torture-testing worker replication. (#4902) (diff) | |
| download | synapse-4d53017432e05da621a13b3fe4d9e67108f856fd.tar.xz | |
Batching in the user directory import (#4900)
| -rw-r--r-- | changelog.d/4900.feature | 1 | ||||
| -rw-r--r-- | synapse/storage/user_directory.py | 13 |
2 files changed, 14 insertions, 0 deletions
diff --git a/changelog.d/4900.feature b/changelog.d/4900.feature new file mode 100644 index 0000000000..8f792b8890 --- /dev/null +++ b/changelog.d/4900.feature @@ -0,0 +1 @@ +The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index 4ee653210f..d360e857d1 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -32,6 +32,11 @@ TEMP_TABLE = "_temp_populate_user_directory" class UserDirectoryStore(BackgroundUpdateStore): + + # How many records do we calculate before sending it to + # add_users_who_share_private_rooms? + SHARE_PRIVATE_WORKING_SET = 500 + def __init__(self, db_conn, hs): super(UserDirectoryStore, self).__init__(db_conn, hs) @@ -218,6 +223,14 @@ class UserDirectoryStore(BackgroundUpdateStore): user_set = (user_id, other_user_id) to_insert.add(user_set) + # If it gets too big, stop and write to the database + # to prevent storing too much in RAM. + if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET: + yield self.add_users_who_share_private_room( + room_id, to_insert + ) + to_insert.clear() + if to_insert: yield self.add_users_who_share_private_room(room_id, to_insert) to_insert.clear() |