summary refs log tree commit diff
path: root/synapse/storage
diff options
context:
space:
mode:
authorAmber Brown <hawkowl@atleastfornow.net>2019-03-21 03:06:36 +1100
committerGitHub <noreply@github.com>2019-03-21 03:06:36 +1100
commit4d53017432e05da621a13b3fe4d9e67108f856fd (patch)
treef3b4d292f3bfb02d3a0adc55ba67552d02a44ab8 /synapse/storage
parentAdd a config option for torture-testing worker replication. (#4902) (diff)
downloadsynapse-4d53017432e05da621a13b3fe4d9e67108f856fd.tar.xz
Batching in the user directory import (#4900)
Diffstat (limited to 'synapse/storage')
-rw-r--r--synapse/storage/user_directory.py13
1 files changed, 13 insertions, 0 deletions
diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py
index 4ee653210f..d360e857d1 100644
--- a/synapse/storage/user_directory.py
+++ b/synapse/storage/user_directory.py
@@ -32,6 +32,11 @@ TEMP_TABLE = "_temp_populate_user_directory"
 
 
 class UserDirectoryStore(BackgroundUpdateStore):
+
+    # How many records do we calculate before sending it to
+    # add_users_who_share_private_rooms?
+    SHARE_PRIVATE_WORKING_SET = 500
+
     def __init__(self, db_conn, hs):
         super(UserDirectoryStore, self).__init__(db_conn, hs)
 
@@ -218,6 +223,14 @@ class UserDirectoryStore(BackgroundUpdateStore):
                             user_set = (user_id, other_user_id)
                             to_insert.add(user_set)
 
+                            # If it gets too big, stop and write to the database
+                            # to prevent storing too much in RAM.
+                            if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET:
+                                yield self.add_users_who_share_private_room(
+                                    room_id, to_insert
+                                )
+                                to_insert.clear()
+
                     if to_insert:
                         yield self.add_users_who_share_private_room(room_id, to_insert)
                         to_insert.clear()