summary refs log tree commit diff
path: root/synapse/storage
diff options
context:
space:
mode:
authorAndrew Morgan <1342360+anoadragon453@users.noreply.github.com>2020-06-19 16:14:37 +0100
committerGitHub <noreply@github.com>2020-06-19 16:14:37 +0100
commit6708163271b0e274c3c0106b31e8fe27a50b48df (patch)
tree2eb5503b1d436513187fa40b1fb565302fb89abd /synapse/storage
parentMerge pull request #45 from matrix-org/dinsic-release-v1.14.x (diff)
downloadsynapse-6708163271b0e274c3c0106b31e8fe27a50b48df.tar.xz
Performance improvements to marking expired users as inactive (#47)
This is a performance-related improvement to #13, which queried and hid active *and* already inactive users, one-by-one. This PR updates the code to query only **active**, expired users, all at once, and then mark them as inactive, all at once.
Diffstat (limited to 'synapse/storage')
-rw-r--r--synapse/storage/data_stores/main/profile.py47
-rw-r--r--synapse/storage/data_stores/main/registration.py21
2 files changed, 52 insertions, 16 deletions
diff --git a/synapse/storage/data_stores/main/profile.py b/synapse/storage/data_stores/main/profile.py

index 7c69996041..cd2472feb0 100644 --- a/synapse/storage/data_stores/main/profile.py +++ b/synapse/storage/data_stores/main/profile.py
@@ -14,11 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List, Tuple + from twisted.internet import defer from synapse.api.errors import StoreError from synapse.storage._base import SQLBaseStore from synapse.storage.data_stores.main.roommember import ProfileInfo +from synapse.types import UserID from synapse.util.caches.descriptors import cached BATCH_SIZE = 100 @@ -149,19 +152,43 @@ class ProfileWorkerStore(SQLBaseStore): lock=False, # we can do this because user_id has a unique index ) - def set_profile_active(self, user_localpart, active, hide, batchnum): - values = {"active": int(active), "batch": batchnum} + def set_profiles_active( + self, users: List[UserID], active: bool, hide: bool, batchnum: int, + ): + """Given a set of users, set active and hidden flags on them. + + Args: + users: A list of UserIDs + active: Whether to set the users to active or inactive + hide: Whether to hide the users (withold from replication). If + False and active is False, users will have their profiles + erased + batchnum: The batch number, used for profile replication + + Returns: + Deferred + """ + # Convert list of localparts to list of tuples containing localparts + user_localparts = [(user.localpart,) for user in users] + + # Generate list of value tuples for each user + value_names = ("active", "batch") + values = [(int(active), batchnum) for _ in user_localparts] # type: List[Tuple] + if not active and not hide: # we are deactivating for real (not in hide mode) - # so clear the profile. - values["avatar_url"] = None - values["displayname"] = None - return self.db.simple_upsert( + # so clear the profile information + value_names += ("avatar_url", "displayname") + values = [v + (None, None) for v in values] + + return self.db.runInteraction( + "set_profiles_active", + self.db.simple_upsert_many_txn, table="profiles", - keyvalues={"user_id": user_localpart}, - values=values, - desc="set_profile_active", - lock=False, # we can do this because user_id has a unique index + key_names=("user_id",), + key_values=user_localparts, + value_names=value_names, + value_values=values, ) diff --git a/synapse/storage/data_stores/main/registration.py b/synapse/storage/data_stores/main/registration.py
index e91634b322..b07c44d87a 100644 --- a/synapse/storage/data_stores/main/registration.py +++ b/synapse/storage/data_stores/main/registration.py
@@ -159,25 +159,34 @@ class RegistrationWorkerStore(SQLBaseStore): @defer.inlineCallbacks def get_expired_users(self): - """Get IDs of all expired users + """Get UserIDs of all expired users. + + Users who are not active, or do not have profile information, are + excluded from the results. Returns: - Deferred[list[str]]: List of expired user IDs + Deferred[List[UserID]]: List of expired user IDs """ def get_expired_users_txn(txn, now_ms): + # We need to use pattern matching as profiles.user_id is confusingly just the + # user's localpart, whereas account_validity.user_id is a full user ID sql = """ - SELECT user_id from account_validity - WHERE expiration_ts_ms <= ? + SELECT av.user_id from account_validity AS av + LEFT JOIN profiles as p + ON av.user_id LIKE '%%' || p.user_id || ':%%' + WHERE expiration_ts_ms <= ? + AND p.active = 1 """ txn.execute(sql, (now_ms,)) rows = txn.fetchall() - return [row[0] for row in rows] + + return [UserID.from_string(row[0]) for row in rows] res = yield self.db.runInteraction( "get_expired_users", get_expired_users_txn, self.clock.time_msec() ) - defer.returnValue(res) + return res @defer.inlineCallbacks def set_renewal_token_for_user(self, user_id, renewal_token):