diff options
| -rw-r--r-- | changelog.d/15181.bugfix | 1 | ||||
| -rw-r--r-- | changelog.d/15393.misc | 1 | ||||
| -rwxr-xr-x | synapse/_scripts/synapse_port_db.py | 2 | ||||
| -rw-r--r-- | synapse/api/room_versions.py | 37 | ||||
| -rw-r--r-- | synapse/events/utils.py | 5 | ||||
| -rw-r--r-- | synapse/handlers/deactivate_account.py | 3 | ||||
| -rw-r--r-- | synapse/storage/databases/main/e2e_room_keys.py | 114 | ||||
| -rw-r--r-- | synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql | 17 | ||||
| -rw-r--r-- | tests/events/test_utils.py | 7 | ||||
| -rw-r--r-- | tests/rest/client/test_account.py | 157 |
10 files changed, 340 insertions, 4 deletions
diff --git a/changelog.d/15181.bugfix b/changelog.d/15181.bugfix new file mode 100644 index 0000000000..191bb6f611 --- /dev/null +++ b/changelog.d/15181.bugfix @@ -0,0 +1 @@ +Delete server-side backup keys when deactivating an account. \ No newline at end of file diff --git a/changelog.d/15393.misc b/changelog.d/15393.misc new file mode 100644 index 0000000000..24483c8d78 --- /dev/null +++ b/changelog.d/15393.misc @@ -0,0 +1 @@ +Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 1dcb397ba4..a58ae2a308 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -59,6 +59,7 @@ from synapse.storage.databases.main.account_data import AccountDataWorkerStore from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore +from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore from synapse.storage.databases.main.event_push_actions import EventPushActionsStore from synapse.storage.databases.main.events_bg_updates import ( @@ -225,6 +226,7 @@ class Store( MainStateBackgroundUpdateStore, UserDirectoryBackgroundUpdateStore, EndToEndKeyBackgroundStore, + EndToEndRoomKeyBackgroundStore, StatsStore, AccountDataWorkerStore, PushRuleStore, diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index c397920fe5..bc15f2d063 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -104,6 +104,8 @@ class RoomVersion: # support the flag. Unknown flags are ignored by the evaluator, making conditions # fail if used. msc3931_push_features: Tuple[str, ...] # values from PushRuleRoomFlag + # MSC3989: Redact the origin field. + msc3989_redaction_rules: bool class RoomVersions: @@ -125,6 +127,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V2 = RoomVersion( "2", @@ -144,6 +147,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V3 = RoomVersion( "3", @@ -163,6 +167,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V4 = RoomVersion( "4", @@ -182,6 +187,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V5 = RoomVersion( "5", @@ -201,6 +207,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V6 = RoomVersion( "6", @@ -220,6 +227,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -239,6 +247,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V7 = RoomVersion( "7", @@ -258,6 +267,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V8 = RoomVersion( "8", @@ -277,6 +287,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V9 = RoomVersion( "9", @@ -296,6 +307,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC3787 = RoomVersion( "org.matrix.msc3787", @@ -315,6 +327,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=True, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V10 = RoomVersion( "10", @@ -334,6 +347,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=True, msc3667_int_only_power_levels=True, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC2716v4 = RoomVersion( "org.matrix.msc2716v4", @@ -353,6 +367,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC1767v10 = RoomVersion( # MSC1767 (Extensible Events) based on room version "10" @@ -373,6 +388,27 @@ class RoomVersions: msc3787_knock_restricted_join_rule=True, msc3667_int_only_power_levels=True, msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,), + msc3989_redaction_rules=False, + ) + MSC3989 = RoomVersion( + "org.matrix.msc3989", + RoomDisposition.UNSTABLE, + EventFormatVersions.ROOM_V4_PLUS, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + msc3083_join_rules=True, + msc3375_redaction_rules=True, + msc2403_knocking=True, + msc2716_historical=False, + msc2716_redactions=False, + msc3787_knock_restricted_join_rule=True, + msc3667_int_only_power_levels=True, + msc3931_push_features=(), + msc3989_redaction_rules=True, ) @@ -392,6 +428,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = { RoomVersions.MSC3787, RoomVersions.V10, RoomVersions.MSC2716v4, + RoomVersions.MSC3989, ) } diff --git a/synapse/events/utils.py b/synapse/events/utils.py index c14c7791db..1d5d7491cd 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -106,7 +106,6 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic "depth", "prev_events", "auth_events", - "origin", "origin_server_ts", ] @@ -114,6 +113,10 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic if not room_version.msc2176_redaction_rules: allowed_keys.extend(["prev_state", "membership"]) + # Room versions before MSC3989 kept the origin field. + if not room_version.msc3989_redaction_rules: + allowed_keys.append("origin") + event_type = event_dict["type"] new_content = {} diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index d31263c717..bd5867491b 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -176,6 +176,9 @@ class DeactivateAccountHandler: # Remove account data (including ignored users and push rules). await self.store.purge_account_data_for_user(user_id) + # Delete any server-side backup keys + await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id) + # Let modules know the user has been deactivated. await self._third_party_rules.on_user_deactivation_status_changed( user_id, diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index 9f8d2e4bea..d01f28cc80 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -13,17 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Iterable, Mapping, Optional, Tuple, cast +from typing import TYPE_CHECKING, Dict, Iterable, Mapping, Optional, Tuple, cast from typing_extensions import Literal, TypedDict from synapse.api.errors import StoreError from synapse.logging.opentracing import log_kv, trace from synapse.storage._base import SQLBaseStore, db_to_json -from synapse.storage.database import LoggingTransaction +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.types import JsonDict, JsonSerializable, StreamKeyType from synapse.util import json_encoder +if TYPE_CHECKING: + from synapse.server import HomeServer + class RoomKey(TypedDict): """`KeyBackupData` in the Matrix spec. @@ -37,7 +44,82 @@ class RoomKey(TypedDict): session_data: JsonSerializable -class EndToEndRoomKeyStore(SQLBaseStore): +class EndToEndRoomKeyBackgroundStore(SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.db_pool.updates.register_background_update_handler( + "delete_e2e_backup_keys_for_deactivated_users", + self._delete_e2e_backup_keys_for_deactivated_users, + ) + + def _delete_keys_txn(self, txn: LoggingTransaction, user_id: str) -> None: + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys", + keyvalues={"user_id": user_id}, + ) + + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={"user_id": user_id}, + ) + + async def _delete_e2e_backup_keys_for_deactivated_users( + self, progress: JsonDict, batch_size: int + ) -> int: + """ + Retroactively purges account data for users that have already been deactivated. + Gets run as a background update caused by a schema delta. + """ + + last_user: str = progress.get("last_user", "") + + def _delete_backup_keys_for_deactivated_users_txn( + txn: LoggingTransaction, + ) -> int: + sql = """ + SELECT name FROM users + WHERE deactivated = ? and name > ? + ORDER BY name ASC + LIMIT ? + """ + + txn.execute(sql, (1, last_user, batch_size)) + users = [row[0] for row in txn] + + for user in users: + self._delete_keys_txn(txn, user) + + if users: + self.db_pool.updates._background_update_progress_txn( + txn, + "delete_e2e_backup_keys_for_deactivated_users", + {"last_user": users[-1]}, + ) + + return len(users) + + number_deleted = await self.db_pool.runInteraction( + "_delete_backup_keys_for_deactivated_users", + _delete_backup_keys_for_deactivated_users_txn, + ) + + if number_deleted < batch_size: + await self.db_pool.updates._end_background_update( + "delete_e2e_backup_keys_for_deactivated_users" + ) + + return number_deleted + + +class EndToEndRoomKeyStore(EndToEndRoomKeyBackgroundStore): """The store for end to end room key backups. See https://spec.matrix.org/v1.1/client-server-api/#server-side-key-backups @@ -550,3 +632,29 @@ class EndToEndRoomKeyStore(SQLBaseStore): await self.db_pool.runInteraction( "delete_e2e_room_keys_version", _delete_e2e_room_keys_version_txn ) + + async def bulk_delete_backup_keys_and_versions_for_user(self, user_id: str) -> None: + """ + Bulk deletes all backup room keys and versions for a given user. + + Args: + user_id: the user whose backup keys and versions we're deleting + """ + + def _delete_all_e2e_room_keys_and_versions_txn(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys", + keyvalues={"user_id": user_id}, + ) + + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={"user_id": user_id}, + ) + + await self.db_pool.runInteraction( + "delete_all_e2e_room_keys_and_versions", + _delete_all_e2e_room_keys_and_versions_txn, + ) diff --git a/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql new file mode 100644 index 0000000000..a194f4cece --- /dev/null +++ b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql @@ -0,0 +1,17 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (7404, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 4174a237ec..c35f58f462 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -143,6 +143,13 @@ class PruneEventTestCase(stdlib_unittest.TestCase): room_version=RoomVersions.MSC2176, ) + # As of MSC3989 we now redact the origin key. + self.run_test( + {"type": "A", "origin": "example.com"}, + {"type": "A", "content": {}, "signatures": {}, "unsigned": {}}, + room_version=RoomVersions.MSC3989, + ) + def test_unsigned(self) -> None: """Ensure that unsigned properties get stripped (except age_ts and replaces_state).""" self.run_test( diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 7f675c44a2..ac19f3c6da 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -474,6 +474,163 @@ class DeactivateTestCase(unittest.HomeserverTestCase): self.assertEqual(len(memberships), 1, memberships) self.assertEqual(memberships[0].room_id, room_id, memberships) + def test_deactivate_account_deletes_server_side_backup_keys(self) -> None: + key_handler = self.hs.get_e2e_room_keys_handler() + room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK", + } + } + } + } + } + + user_id = self.register_user("missPiggy", "test") + tok = self.login("missPiggy", "test") + + # add some backup keys/versions + version = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version, room_keys)) + + version2 = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys)) + + self.deactivate(user_id, tok) + store = self.hs.get_datastores().main + + # Check that the user has been marked as deactivated. + self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id))) + + # Check that there are no entries in 'e2e_room_keys` and `e2e_room_keys_versions` + res = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys", {"user_id": user_id}, "*", "simple_select" + ) + ) + self.assertEqual(len(res), 0) + + res2 = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys_versions", {"user_id": user_id}, "*", "simple_select" + ) + ) + self.assertEqual(len(res2), 0) + + def test_background_update_deletes_deactivated_users_server_side_backup_keys( + self, + ) -> None: + key_handler = self.hs.get_e2e_room_keys_handler() + room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK", + } + } + } + } + } + self.store = self.hs.get_datastores().main + + # create a bunch of users and add keys for them + users = [] + for i in range(0, 20): + user_id = self.register_user("missPiggy" + str(i), "test") + users.append((user_id,)) + + # add some backup keys/versions + version = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": str(i) + "_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version, room_keys)) + + version2 = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": str(i) + "_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys)) + + # deactivate most of the users by editing DB + self.get_success( + self.store.db_pool.simple_update_many( + table="users", + key_names=("name",), + key_values=users[0:18], + value_names=("deactivated",), + value_values=[(1,) for i in range(1, 19)], + desc="", + ) + ) + + # run background update + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": "delete_e2e_backup_keys_for_deactivated_users", + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # check that keys are deleted for the deactivated users but not the others + res = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys", None, ("user_id",), "simple_select" + ) + ) + self.assertEqual(len(res), 4) + + res2 = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys_versions", None, ("user_id",), "simple_select" + ) + ) + self.assertEqual(len(res2), 4) + def deactivate(self, user_id: str, tok: str) -> None: request_data = { "auth": { |