diff --git a/changelog.d/15181.bugfix b/changelog.d/15181.bugfix
new file mode 100644
index 0000000000..191bb6f611
--- /dev/null
+++ b/changelog.d/15181.bugfix
@@ -0,0 +1 @@
+Delete server-side backup keys when deactivating an account.
\ No newline at end of file
diff --git a/changelog.d/15393.misc b/changelog.d/15393.misc
new file mode 100644
index 0000000000..24483c8d78
--- /dev/null
+++ b/changelog.d/15393.misc
@@ -0,0 +1 @@
+Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm.
diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py
index 1dcb397ba4..a58ae2a308 100755
--- a/synapse/_scripts/synapse_port_db.py
+++ b/synapse/_scripts/synapse_port_db.py
@@ -59,6 +59,7 @@ from synapse.storage.databases.main.account_data import AccountDataWorkerStore
from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore
from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore
from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
+from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
from synapse.storage.databases.main.events_bg_updates import (
@@ -225,6 +226,7 @@ class Store(
MainStateBackgroundUpdateStore,
UserDirectoryBackgroundUpdateStore,
EndToEndKeyBackgroundStore,
+ EndToEndRoomKeyBackgroundStore,
StatsStore,
AccountDataWorkerStore,
PushRuleStore,
diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py
index c397920fe5..bc15f2d063 100644
--- a/synapse/api/room_versions.py
+++ b/synapse/api/room_versions.py
@@ -104,6 +104,8 @@ class RoomVersion:
# support the flag. Unknown flags are ignored by the evaluator, making conditions
# fail if used.
msc3931_push_features: Tuple[str, ...] # values from PushRuleRoomFlag
+ # MSC3989: Redact the origin field.
+ msc3989_redaction_rules: bool
class RoomVersions:
@@ -125,6 +127,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V2 = RoomVersion(
"2",
@@ -144,6 +147,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V3 = RoomVersion(
"3",
@@ -163,6 +167,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V4 = RoomVersion(
"4",
@@ -182,6 +187,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V5 = RoomVersion(
"5",
@@ -201,6 +207,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V6 = RoomVersion(
"6",
@@ -220,6 +227,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
MSC2176 = RoomVersion(
"org.matrix.msc2176",
@@ -239,6 +247,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V7 = RoomVersion(
"7",
@@ -258,6 +267,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V8 = RoomVersion(
"8",
@@ -277,6 +287,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V9 = RoomVersion(
"9",
@@ -296,6 +307,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
MSC3787 = RoomVersion(
"org.matrix.msc3787",
@@ -315,6 +327,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=True,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
V10 = RoomVersion(
"10",
@@ -334,6 +347,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=True,
msc3667_int_only_power_levels=True,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
MSC2716v4 = RoomVersion(
"org.matrix.msc2716v4",
@@ -353,6 +367,7 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=False,
msc3667_int_only_power_levels=False,
msc3931_push_features=(),
+ msc3989_redaction_rules=False,
)
MSC1767v10 = RoomVersion(
# MSC1767 (Extensible Events) based on room version "10"
@@ -373,6 +388,27 @@ class RoomVersions:
msc3787_knock_restricted_join_rule=True,
msc3667_int_only_power_levels=True,
msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,),
+ msc3989_redaction_rules=False,
+ )
+ MSC3989 = RoomVersion(
+ "org.matrix.msc3989",
+ RoomDisposition.UNSTABLE,
+ EventFormatVersions.ROOM_V4_PLUS,
+ StateResolutionVersions.V2,
+ enforce_key_validity=True,
+ special_case_aliases_auth=False,
+ strict_canonicaljson=True,
+ limit_notifications_power_levels=True,
+ msc2176_redaction_rules=False,
+ msc3083_join_rules=True,
+ msc3375_redaction_rules=True,
+ msc2403_knocking=True,
+ msc2716_historical=False,
+ msc2716_redactions=False,
+ msc3787_knock_restricted_join_rule=True,
+ msc3667_int_only_power_levels=True,
+ msc3931_push_features=(),
+ msc3989_redaction_rules=True,
)
@@ -392,6 +428,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
RoomVersions.MSC3787,
RoomVersions.V10,
RoomVersions.MSC2716v4,
+ RoomVersions.MSC3989,
)
}
diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index c14c7791db..1d5d7491cd 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -106,7 +106,6 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
"depth",
"prev_events",
"auth_events",
- "origin",
"origin_server_ts",
]
@@ -114,6 +113,10 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
if not room_version.msc2176_redaction_rules:
allowed_keys.extend(["prev_state", "membership"])
+ # Room versions before MSC3989 kept the origin field.
+ if not room_version.msc3989_redaction_rules:
+ allowed_keys.append("origin")
+
event_type = event_dict["type"]
new_content = {}
diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index d31263c717..bd5867491b 100644
--- a/synapse/handlers/deactivate_account.py
+++ b/synapse/handlers/deactivate_account.py
@@ -176,6 +176,9 @@ class DeactivateAccountHandler:
# Remove account data (including ignored users and push rules).
await self.store.purge_account_data_for_user(user_id)
+ # Delete any server-side backup keys
+ await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id)
+
# Let modules know the user has been deactivated.
await self._third_party_rules.on_user_deactivation_status_changed(
user_id,
diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py
index 9f8d2e4bea..d01f28cc80 100644
--- a/synapse/storage/databases/main/e2e_room_keys.py
+++ b/synapse/storage/databases/main/e2e_room_keys.py
@@ -13,17 +13,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Dict, Iterable, Mapping, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Dict, Iterable, Mapping, Optional, Tuple, cast
from typing_extensions import Literal, TypedDict
from synapse.api.errors import StoreError
from synapse.logging.opentracing import log_kv, trace
from synapse.storage._base import SQLBaseStore, db_to_json
-from synapse.storage.database import LoggingTransaction
+from synapse.storage.database import (
+ DatabasePool,
+ LoggingDatabaseConnection,
+ LoggingTransaction,
+)
from synapse.types import JsonDict, JsonSerializable, StreamKeyType
from synapse.util import json_encoder
+if TYPE_CHECKING:
+ from synapse.server import HomeServer
+
class RoomKey(TypedDict):
"""`KeyBackupData` in the Matrix spec.
@@ -37,7 +44,82 @@ class RoomKey(TypedDict):
session_data: JsonSerializable
-class EndToEndRoomKeyStore(SQLBaseStore):
+class EndToEndRoomKeyBackgroundStore(SQLBaseStore):
+ def __init__(
+ self,
+ database: DatabasePool,
+ db_conn: LoggingDatabaseConnection,
+ hs: "HomeServer",
+ ):
+ super().__init__(database, db_conn, hs)
+
+ self.db_pool.updates.register_background_update_handler(
+ "delete_e2e_backup_keys_for_deactivated_users",
+ self._delete_e2e_backup_keys_for_deactivated_users,
+ )
+
+ def _delete_keys_txn(self, txn: LoggingTransaction, user_id: str) -> None:
+ self.db_pool.simple_delete_txn(
+ txn,
+ table="e2e_room_keys",
+ keyvalues={"user_id": user_id},
+ )
+
+ self.db_pool.simple_delete_txn(
+ txn,
+ table="e2e_room_keys_versions",
+ keyvalues={"user_id": user_id},
+ )
+
+ async def _delete_e2e_backup_keys_for_deactivated_users(
+ self, progress: JsonDict, batch_size: int
+ ) -> int:
+ """
+ Retroactively purges account data for users that have already been deactivated.
+ Gets run as a background update caused by a schema delta.
+ """
+
+ last_user: str = progress.get("last_user", "")
+
+ def _delete_backup_keys_for_deactivated_users_txn(
+ txn: LoggingTransaction,
+ ) -> int:
+ sql = """
+ SELECT name FROM users
+ WHERE deactivated = ? and name > ?
+ ORDER BY name ASC
+ LIMIT ?
+ """
+
+ txn.execute(sql, (1, last_user, batch_size))
+ users = [row[0] for row in txn]
+
+ for user in users:
+ self._delete_keys_txn(txn, user)
+
+ if users:
+ self.db_pool.updates._background_update_progress_txn(
+ txn,
+ "delete_e2e_backup_keys_for_deactivated_users",
+ {"last_user": users[-1]},
+ )
+
+ return len(users)
+
+ number_deleted = await self.db_pool.runInteraction(
+ "_delete_backup_keys_for_deactivated_users",
+ _delete_backup_keys_for_deactivated_users_txn,
+ )
+
+ if number_deleted < batch_size:
+ await self.db_pool.updates._end_background_update(
+ "delete_e2e_backup_keys_for_deactivated_users"
+ )
+
+ return number_deleted
+
+
+class EndToEndRoomKeyStore(EndToEndRoomKeyBackgroundStore):
"""The store for end to end room key backups.
See https://spec.matrix.org/v1.1/client-server-api/#server-side-key-backups
@@ -550,3 +632,29 @@ class EndToEndRoomKeyStore(SQLBaseStore):
await self.db_pool.runInteraction(
"delete_e2e_room_keys_version", _delete_e2e_room_keys_version_txn
)
+
+ async def bulk_delete_backup_keys_and_versions_for_user(self, user_id: str) -> None:
+ """
+ Bulk deletes all backup room keys and versions for a given user.
+
+ Args:
+ user_id: the user whose backup keys and versions we're deleting
+ """
+
+ def _delete_all_e2e_room_keys_and_versions_txn(txn: LoggingTransaction) -> None:
+ self.db_pool.simple_delete_txn(
+ txn,
+ table="e2e_room_keys",
+ keyvalues={"user_id": user_id},
+ )
+
+ self.db_pool.simple_delete_txn(
+ txn,
+ table="e2e_room_keys_versions",
+ keyvalues={"user_id": user_id},
+ )
+
+ await self.db_pool.runInteraction(
+ "delete_all_e2e_room_keys_and_versions",
+ _delete_all_e2e_room_keys_and_versions_txn,
+ )
diff --git a/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql
new file mode 100644
index 0000000000..a194f4cece
--- /dev/null
+++ b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql
@@ -0,0 +1,17 @@
+/* Copyright 2023 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+ (7404, 'delete_e2e_backup_keys_for_deactivated_users', '{}');
\ No newline at end of file
diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py
index 4174a237ec..c35f58f462 100644
--- a/tests/events/test_utils.py
+++ b/tests/events/test_utils.py
@@ -143,6 +143,13 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
room_version=RoomVersions.MSC2176,
)
+ # As of MSC3989 we now redact the origin key.
+ self.run_test(
+ {"type": "A", "origin": "example.com"},
+ {"type": "A", "content": {}, "signatures": {}, "unsigned": {}},
+ room_version=RoomVersions.MSC3989,
+ )
+
def test_unsigned(self) -> None:
"""Ensure that unsigned properties get stripped (except age_ts and replaces_state)."""
self.run_test(
diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py
index 7f675c44a2..ac19f3c6da 100644
--- a/tests/rest/client/test_account.py
+++ b/tests/rest/client/test_account.py
@@ -474,6 +474,163 @@ class DeactivateTestCase(unittest.HomeserverTestCase):
self.assertEqual(len(memberships), 1, memberships)
self.assertEqual(memberships[0].room_id, room_id, memberships)
+ def test_deactivate_account_deletes_server_side_backup_keys(self) -> None:
+ key_handler = self.hs.get_e2e_room_keys_handler()
+ room_keys = {
+ "rooms": {
+ "!abc:matrix.org": {
+ "sessions": {
+ "c0ff33": {
+ "first_message_index": 1,
+ "forwarded_count": 1,
+ "is_verified": False,
+ "session_data": "SSBBTSBBIEZJU0gK",
+ }
+ }
+ }
+ }
+ }
+
+ user_id = self.register_user("missPiggy", "test")
+ tok = self.login("missPiggy", "test")
+
+ # add some backup keys/versions
+ version = self.get_success(
+ key_handler.create_version(
+ user_id,
+ {
+ "algorithm": "m.megolm_backup.v1",
+ "auth_data": "first_version_auth_data",
+ },
+ )
+ )
+
+ self.get_success(key_handler.upload_room_keys(user_id, version, room_keys))
+
+ version2 = self.get_success(
+ key_handler.create_version(
+ user_id,
+ {
+ "algorithm": "m.megolm_backup.v1",
+ "auth_data": "second_version_auth_data",
+ },
+ )
+ )
+
+ self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys))
+
+ self.deactivate(user_id, tok)
+ store = self.hs.get_datastores().main
+
+ # Check that the user has been marked as deactivated.
+ self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id)))
+
+ # Check that there are no entries in 'e2e_room_keys` and `e2e_room_keys_versions`
+ res = self.get_success(
+ self.hs.get_datastores().main.db_pool.simple_select_list(
+ "e2e_room_keys", {"user_id": user_id}, "*", "simple_select"
+ )
+ )
+ self.assertEqual(len(res), 0)
+
+ res2 = self.get_success(
+ self.hs.get_datastores().main.db_pool.simple_select_list(
+ "e2e_room_keys_versions", {"user_id": user_id}, "*", "simple_select"
+ )
+ )
+ self.assertEqual(len(res2), 0)
+
+ def test_background_update_deletes_deactivated_users_server_side_backup_keys(
+ self,
+ ) -> None:
+ key_handler = self.hs.get_e2e_room_keys_handler()
+ room_keys = {
+ "rooms": {
+ "!abc:matrix.org": {
+ "sessions": {
+ "c0ff33": {
+ "first_message_index": 1,
+ "forwarded_count": 1,
+ "is_verified": False,
+ "session_data": "SSBBTSBBIEZJU0gK",
+ }
+ }
+ }
+ }
+ }
+ self.store = self.hs.get_datastores().main
+
+ # create a bunch of users and add keys for them
+ users = []
+ for i in range(0, 20):
+ user_id = self.register_user("missPiggy" + str(i), "test")
+ users.append((user_id,))
+
+ # add some backup keys/versions
+ version = self.get_success(
+ key_handler.create_version(
+ user_id,
+ {
+ "algorithm": "m.megolm_backup.v1",
+ "auth_data": str(i) + "_version_auth_data",
+ },
+ )
+ )
+
+ self.get_success(key_handler.upload_room_keys(user_id, version, room_keys))
+
+ version2 = self.get_success(
+ key_handler.create_version(
+ user_id,
+ {
+ "algorithm": "m.megolm_backup.v1",
+ "auth_data": str(i) + "_version_auth_data",
+ },
+ )
+ )
+
+ self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys))
+
+ # deactivate most of the users by editing DB
+ self.get_success(
+ self.store.db_pool.simple_update_many(
+ table="users",
+ key_names=("name",),
+ key_values=users[0:18],
+ value_names=("deactivated",),
+ value_values=[(1,) for i in range(1, 19)],
+ desc="",
+ )
+ )
+
+ # run background update
+ self.get_success(
+ self.store.db_pool.simple_insert(
+ "background_updates",
+ {
+ "update_name": "delete_e2e_backup_keys_for_deactivated_users",
+ "progress_json": "{}",
+ },
+ )
+ )
+ self.store.db_pool.updates._all_done = False
+ self.wait_for_background_updates()
+
+ # check that keys are deleted for the deactivated users but not the others
+ res = self.get_success(
+ self.hs.get_datastores().main.db_pool.simple_select_list(
+ "e2e_room_keys", None, ("user_id",), "simple_select"
+ )
+ )
+ self.assertEqual(len(res), 4)
+
+ res2 = self.get_success(
+ self.hs.get_datastores().main.db_pool.simple_select_list(
+ "e2e_room_keys_versions", None, ("user_id",), "simple_select"
+ )
+ )
+ self.assertEqual(len(res2), 4)
+
def deactivate(self, user_id: str, tok: str) -> None:
request_data = {
"auth": {
|
