diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py
index caf04b54cb..2c970fc827 100644
--- a/tests/federation/test_federation_sender.py
+++ b/tests/federation/test_federation_sender.py
@@ -478,7 +478,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase):
# expect two edus, in one or two transactions. We don't know what order the
# devices will be updated.
self.assertEqual(len(self.edus), 2)
- stream_id = None # FIXME: there is a discontinuity in the stream IDs: see #7142
+ stream_id = None # FIXME: there is a discontinuity in the stream IDs: see https://github.com/matrix-org/synapse/issues/7142
for edu in self.edus:
self.assertEqual(edu["edu_type"], EduTypes.DEVICE_LIST_UPDATE)
c = edu["content"]
diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py
index 90b4da9ad5..07eb63f95e 100644
--- a/tests/handlers/test_e2e_keys.py
+++ b/tests/handlers/test_e2e_keys.py
@@ -1602,3 +1602,50 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
}
},
)
+
+ def test_check_cross_signing_setup(self) -> None:
+ # First check what happens with no master key.
+ alice = "@alice:test"
+ exists, replaceable_without_uia = self.get_success(
+ self.handler.check_cross_signing_setup(alice)
+ )
+ self.assertIs(exists, False)
+ self.assertIs(replaceable_without_uia, False)
+
+ # Upload a master key but don't specify a replacement timestamp.
+ dummy_key = {"keys": {"a": "b"}}
+ self.get_success(
+ self.store.set_e2e_cross_signing_key("@alice:test", "master", dummy_key)
+ )
+
+ # Should now find the key exists.
+ exists, replaceable_without_uia = self.get_success(
+ self.handler.check_cross_signing_setup(alice)
+ )
+ self.assertIs(exists, True)
+ self.assertIs(replaceable_without_uia, False)
+
+ # Set an expiry timestamp in the future.
+ self.get_success(
+ self.store.allow_master_cross_signing_key_replacement_without_uia(
+ alice,
+ 1000,
+ )
+ )
+
+ # Should now be allowed to replace the key without UIA.
+ exists, replaceable_without_uia = self.get_success(
+ self.handler.check_cross_signing_setup(alice)
+ )
+ self.assertIs(exists, True)
+ self.assertIs(replaceable_without_uia, True)
+
+ # Wait 2 seconds, so that the timestamp is in the past.
+ self.reactor.advance(2.0)
+
+ # Should no longer be allowed to replace the key without UIA.
+ exists, replaceable_without_uia = self.get_success(
+ self.handler.check_cross_signing_setup(alice)
+ )
+ self.assertIs(exists, True)
+ self.assertIs(replaceable_without_uia, False)
diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py
index 4fc0742413..a035232905 100644
--- a/tests/handlers/test_federation.py
+++ b/tests/handlers/test_federation.py
@@ -112,7 +112,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase):
"""
Check that we store the state group correctly for rejected non-state events.
- Regression test for #6289.
+ Regression test for https://github.com/matrix-org/synapse/issues/6289.
"""
OTHER_SERVER = "otherserver"
OTHER_USER = "@otheruser:" + OTHER_SERVER
@@ -165,7 +165,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase):
"""
Check that we store the state group correctly for rejected state events.
- Regression test for #6289.
+ Regression test for https://github.com/matrix-org/synapse/issues/6289.
"""
OTHER_SERVER = "otherserver"
OTHER_USER = "@otheruser:" + OTHER_SERVER
@@ -222,7 +222,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase):
of backwards extremities(the magic number is more than 5),
no errors are thrown.
- Regression test, see #11027
+ Regression test, see https://github.com/matrix-org/synapse/pull/11027
"""
# create the room
user_id = self.register_user("kermit", "test")
diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py
index bf1d287699..b7337d3926 100644
--- a/tests/http/test_matrixfederationclient.py
+++ b/tests/http/test_matrixfederationclient.py
@@ -368,7 +368,8 @@ class FederationClientTests(HomeserverTestCase):
"""
If a connection is made to a client but the client rejects it due to
requiring a trailing slash. We need to retry the request with a
- trailing slash. Workaround for Synapse <= v0.99.3, explained in #3622.
+ trailing slash. Workaround for Synapse <= v0.99.3, explained in
+ https://github.com/matrix-org/synapse/issues/3622.
"""
d = defer.ensureDeferred(
self.cl.get_json("testserv:8008", "foo/bar", try_trailing_slash_on_400=True)
diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py
index a8e7a76b29..f262304c3d 100644
--- a/tests/media/test_media_storage.py
+++ b/tests/media/test_media_storage.py
@@ -318,7 +318,9 @@ class MediaRepoTests(unittest.HomeserverTestCase):
self.assertEqual(
self.fetches[0][2], "/_matrix/media/r0/download/" + self.media_id
)
- self.assertEqual(self.fetches[0][3], {"allow_remote": "false"})
+ self.assertEqual(
+ self.fetches[0][3], {"allow_remote": "false", "timeout_ms": "20000"}
+ )
headers = {
b"Content-Length": [b"%d" % (len(self.test_image.data))],
diff --git a/tests/push/test_bulk_push_rule_evaluator.py b/tests/push/test_bulk_push_rule_evaluator.py
index 7c23b77e0a..907ee1488c 100644
--- a/tests/push/test_bulk_push_rule_evaluator.py
+++ b/tests/push/test_bulk_push_rule_evaluator.py
@@ -92,7 +92,7 @@ class TestBulkPushRuleEvaluator(HomeserverTestCase):
- the bad power level value for "room", before JSON serisalistion
- whether Bob should expect the message to be highlighted
- Reproduces #14060.
+ Reproduces https://github.com/matrix-org/synapse/issues/14060.
A lack of validation: the gift that keeps on giving.
"""
diff --git a/tests/replication/tcp/streams/test_to_device.py b/tests/replication/tcp/streams/test_to_device.py
index ab379e8cf1..85adf84ece 100644
--- a/tests/replication/tcp/streams/test_to_device.py
+++ b/tests/replication/tcp/streams/test_to_device.py
@@ -62,7 +62,7 @@ class ToDeviceStreamTestCase(BaseStreamTestCase):
)
# add one more message, for user2 this time
- # this message would be dropped before fixing #15335
+ # this message would be dropped before fixing https://github.com/matrix-org/synapse/issues/15335
msg["content"] = {"device": {}}
messages = {user2: {"device": msg}}
diff --git a/tests/replication/tcp/streams/test_typing.py b/tests/replication/tcp/streams/test_typing.py
index 5a38ac831f..20b3d431ba 100644
--- a/tests/replication/tcp/streams/test_typing.py
+++ b/tests/replication/tcp/streams/test_typing.py
@@ -35,6 +35,10 @@ class TypingStreamTestCase(BaseStreamTestCase):
typing = self.hs.get_typing_handler()
assert isinstance(typing, TypingWriterHandler)
+ # Create a typing update before we reconnect so that there is a missing
+ # update to fetch.
+ typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True)
+
self.reconnect()
typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True)
@@ -91,6 +95,10 @@ class TypingStreamTestCase(BaseStreamTestCase):
typing = self.hs.get_typing_handler()
assert isinstance(typing, TypingWriterHandler)
+ # Create a typing update before we reconnect so that there is a missing
+ # update to fetch.
+ typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True)
+
self.reconnect()
typing._push_update(member=RoomMember(ROOM_ID, USER_ID), typing=True)
diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py
index 42b065d883..cf71bbb461 100644
--- a/tests/rest/admin/test_user.py
+++ b/tests/rest/admin/test_user.py
@@ -1478,7 +1478,7 @@ class DeactivateAccountTestCase(unittest.HomeserverTestCase):
def test_deactivate_user_erase_true_avatar_nonnull_but_empty(self) -> None:
"""Check we can erase a user whose avatar is the empty string.
- Reproduces #12257.
+ Reproduces https://github.com/matrix-org/synapse/issues/12257.
"""
# Patch `self.other_user` to have an empty string as their avatar.
self.get_success(
@@ -4854,3 +4854,59 @@ class UsersByThreePidTestCase(unittest.HomeserverTestCase):
{"user_id": self.other_user},
channel.json_body,
)
+
+
+class AllowCrossSigningReplacementTestCase(unittest.HomeserverTestCase):
+ servlets = [
+ synapse.rest.admin.register_servlets,
+ login.register_servlets,
+ ]
+
+ @staticmethod
+ def url(user: str) -> str:
+ template = (
+ "/_synapse/admin/v1/users/{}/_allow_cross_signing_replacement_without_uia"
+ )
+ return template.format(urllib.parse.quote(user))
+
+ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
+ self.store = hs.get_datastores().main
+
+ self.admin_user = self.register_user("admin", "pass", admin=True)
+ self.admin_user_tok = self.login("admin", "pass")
+
+ self.other_user = self.register_user("user", "pass")
+
+ def test_error_cases(self) -> None:
+ fake_user = "@bums:other"
+ channel = self.make_request(
+ "POST", self.url(fake_user), access_token=self.admin_user_tok
+ )
+ # Fail: user doesn't exist
+ self.assertEqual(404, channel.code, msg=channel.json_body)
+
+ channel = self.make_request(
+ "POST", self.url(self.other_user), access_token=self.admin_user_tok
+ )
+ # Fail: user exists, but has no master cross-signing key
+ self.assertEqual(404, channel.code, msg=channel.json_body)
+
+ def test_success(self) -> None:
+ # Upload a master key.
+ dummy_key = {"keys": {"a": "b"}}
+ self.get_success(
+ self.store.set_e2e_cross_signing_key(self.other_user, "master", dummy_key)
+ )
+
+ channel = self.make_request(
+ "POST", self.url(self.other_user), access_token=self.admin_user_tok
+ )
+ # Success!
+ self.assertEqual(200, channel.code, msg=channel.json_body)
+
+ # Should now find that the key exists.
+ _, timestamp = self.get_success(
+ self.store.get_master_cross_signing_key_updatable_before(self.other_user)
+ )
+ assert timestamp is not None
+ self.assertGreater(timestamp, self.clock.time_msec())
diff --git a/tests/rest/client/test_events.py b/tests/rest/client/test_events.py
index 141e0f57a3..8bea860beb 100644
--- a/tests/rest/client/test_events.py
+++ b/tests/rest/client/test_events.py
@@ -64,7 +64,7 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
# 403. However, since the v1 spec no longer exists and the v1
# implementation is now part of the r0 implementation, the newer
# behaviour is used instead to be consistent with the r0 spec.
- # see issue #2602
+ # see issue https://github.com/matrix-org/synapse/issues/2602
channel = self.make_request(
"GET", "/events?access_token=%s" % ("invalid" + self.token,)
)
diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py
index 8ee5489057..9f81a695fa 100644
--- a/tests/rest/client/test_keys.py
+++ b/tests/rest/client/test_keys.py
@@ -11,8 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
-
+import urllib.parse
from http import HTTPStatus
+from unittest.mock import patch
from signedjson.key import (
encode_verify_key_base64,
@@ -24,12 +25,19 @@ from signedjson.sign import sign_json
from synapse.api.errors import Codes
from synapse.rest import admin
from synapse.rest.client import keys, login
-from synapse.types import JsonDict
+from synapse.types import JsonDict, Requester, create_requester
from tests import unittest
from tests.http.server._base import make_request_with_cancellation_test
from tests.unittest import override_config
+try:
+ import authlib # noqa: F401
+
+ HAS_AUTHLIB = True
+except ImportError:
+ HAS_AUTHLIB = False
+
class KeyQueryTestCase(unittest.HomeserverTestCase):
servlets = [
@@ -259,3 +267,179 @@ class KeyQueryTestCase(unittest.HomeserverTestCase):
alice_token,
)
self.assertEqual(channel.code, HTTPStatus.OK, channel.result)
+
+
+class SigningKeyUploadServletTestCase(unittest.HomeserverTestCase):
+ servlets = [
+ admin.register_servlets,
+ keys.register_servlets,
+ ]
+
+ OIDC_ADMIN_TOKEN = "_oidc_admin_token"
+
+ @unittest.skip_unless(HAS_AUTHLIB, "requires authlib")
+ @override_config(
+ {
+ "enable_registration": False,
+ "experimental_features": {
+ "msc3861": {
+ "enabled": True,
+ "issuer": "https://issuer",
+ "account_management_url": "https://my-account.issuer",
+ "client_id": "id",
+ "client_auth_method": "client_secret_post",
+ "client_secret": "secret",
+ "admin_token": OIDC_ADMIN_TOKEN,
+ },
+ },
+ }
+ )
+ def test_master_cross_signing_key_replacement_msc3861(self) -> None:
+ # Provision a user like MAS would, cribbing from
+ # https://github.com/matrix-org/matrix-authentication-service/blob/08d46a79a4adb22819ac9d55e15f8375dfe2c5c7/crates/matrix-synapse/src/lib.rs#L224-L229
+ alice = "@alice:test"
+ channel = self.make_request(
+ "PUT",
+ f"/_synapse/admin/v2/users/{urllib.parse.quote(alice)}",
+ access_token=self.OIDC_ADMIN_TOKEN,
+ content={},
+ )
+ self.assertEqual(channel.code, HTTPStatus.CREATED, channel.json_body)
+
+ # Provision a device like MAS would, cribbing from
+ # https://github.com/matrix-org/matrix-authentication-service/blob/08d46a79a4adb22819ac9d55e15f8375dfe2c5c7/crates/matrix-synapse/src/lib.rs#L260-L262
+ alice_device = "alice_device"
+ channel = self.make_request(
+ "POST",
+ f"/_synapse/admin/v2/users/{urllib.parse.quote(alice)}/devices",
+ access_token=self.OIDC_ADMIN_TOKEN,
+ content={"device_id": alice_device},
+ )
+ self.assertEqual(channel.code, HTTPStatus.CREATED, channel.json_body)
+
+ # Prepare a mock MAS access token.
+ alice_token = "alice_token_1234_oidcwhatyoudidthere"
+
+ async def mocked_get_user_by_access_token(
+ token: str, allow_expired: bool = False
+ ) -> Requester:
+ self.assertEqual(token, alice_token)
+ return create_requester(
+ user_id=alice,
+ device_id=alice_device,
+ scope=[],
+ is_guest=False,
+ )
+
+ patch_get_user_by_access_token = patch.object(
+ self.hs.get_auth(),
+ "get_user_by_access_token",
+ wraps=mocked_get_user_by_access_token,
+ )
+
+ # Copied from E2eKeysHandlerTestCase
+ master_pubkey = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
+ master_pubkey2 = "fHZ3NPiKxoLQm5OoZbKa99SYxprOjNs4TwJUKP+twCM"
+ master_pubkey3 = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY"
+
+ master_key: JsonDict = {
+ "user_id": alice,
+ "usage": ["master"],
+ "keys": {"ed25519:" + master_pubkey: master_pubkey},
+ }
+ master_key2: JsonDict = {
+ "user_id": alice,
+ "usage": ["master"],
+ "keys": {"ed25519:" + master_pubkey2: master_pubkey2},
+ }
+ master_key3: JsonDict = {
+ "user_id": alice,
+ "usage": ["master"],
+ "keys": {"ed25519:" + master_pubkey3: master_pubkey3},
+ }
+
+ with patch_get_user_by_access_token:
+ # Upload an initial cross-signing key.
+ channel = self.make_request(
+ "POST",
+ "/_matrix/client/v3/keys/device_signing/upload",
+ access_token=alice_token,
+ content={
+ "master_key": master_key,
+ },
+ )
+ self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
+
+ # Should not be able to upload another master key.
+ channel = self.make_request(
+ "POST",
+ "/_matrix/client/v3/keys/device_signing/upload",
+ access_token=alice_token,
+ content={
+ "master_key": master_key2,
+ },
+ )
+ self.assertEqual(
+ channel.code, HTTPStatus.NOT_IMPLEMENTED, channel.json_body
+ )
+
+ # Pretend that MAS did UIA and allowed us to replace the master key.
+ channel = self.make_request(
+ "POST",
+ f"/_synapse/admin/v1/users/{urllib.parse.quote(alice)}/_allow_cross_signing_replacement_without_uia",
+ access_token=self.OIDC_ADMIN_TOKEN,
+ )
+ self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
+
+ with patch_get_user_by_access_token:
+ # Should now be able to upload master key2.
+ channel = self.make_request(
+ "POST",
+ "/_matrix/client/v3/keys/device_signing/upload",
+ access_token=alice_token,
+ content={
+ "master_key": master_key2,
+ },
+ )
+ self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
+
+ # Even though we're still in the grace period, we shouldn't be able to
+ # upload master key 3 immediately after uploading key 2.
+ channel = self.make_request(
+ "POST",
+ "/_matrix/client/v3/keys/device_signing/upload",
+ access_token=alice_token,
+ content={
+ "master_key": master_key3,
+ },
+ )
+ self.assertEqual(
+ channel.code, HTTPStatus.NOT_IMPLEMENTED, channel.json_body
+ )
+
+ # Pretend that MAS did UIA and allowed us to replace the master key.
+ channel = self.make_request(
+ "POST",
+ f"/_synapse/admin/v1/users/{urllib.parse.quote(alice)}/_allow_cross_signing_replacement_without_uia",
+ access_token=self.OIDC_ADMIN_TOKEN,
+ )
+ self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
+ timestamp_ms = channel.json_body["updatable_without_uia_before_ms"]
+
+ # Advance to 1 second after the replacement period ends.
+ self.reactor.advance(timestamp_ms - self.clock.time_msec() + 1000)
+
+ with patch_get_user_by_access_token:
+ # We should not be able to upload master key3 because the replacement has
+ # expired.
+ channel = self.make_request(
+ "POST",
+ "/_matrix/client/v3/keys/device_signing/upload",
+ access_token=alice_token,
+ content={
+ "master_key": master_key3,
+ },
+ )
+ self.assertEqual(
+ channel.code, HTTPStatus.NOT_IMPLEMENTED, channel.json_body
+ )
diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py
index ecae092b47..8f923fd40f 100644
--- a/tests/rest/client/test_profile.py
+++ b/tests/rest/client/test_profile.py
@@ -170,7 +170,8 @@ class ProfileTestCase(unittest.HomeserverTestCase):
)
self.assertEqual(channel.code, 200, channel.result)
# FIXME: If a user has no displayname set, Synapse returns 200 and omits a
- # displayname from the response. This contradicts the spec, see #13137.
+ # displayname from the response. This contradicts the spec, see
+ # https://github.com/matrix-org/synapse/issues/13137.
return channel.json_body.get("displayname")
def _get_avatar_url(self, name: Optional[str] = None) -> Optional[str]:
@@ -179,7 +180,8 @@ class ProfileTestCase(unittest.HomeserverTestCase):
)
self.assertEqual(channel.code, 200, channel.result)
# FIXME: If a user has no avatar set, Synapse returns 200 and omits an
- # avatar_url from the response. This contradicts the spec, see #13137.
+ # avatar_url from the response. This contradicts the spec, see
+ # https://github.com/matrix-org/synapse/issues/13137.
return channel.json_body.get("avatar_url")
@unittest.override_config({"max_avatar_size": 50})
diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py
index aaa4f3bba0..bb24ed6aa7 100644
--- a/tests/rest/client/test_rooms.py
+++ b/tests/rest/client/test_rooms.py
@@ -888,7 +888,8 @@ class RoomsCreateTestCase(RoomBase):
)
def test_room_creation_ratelimiting(self) -> None:
"""
- Regression test for #14312, where ratelimiting was made too strict.
+ Regression test for https://github.com/matrix-org/synapse/issues/14312,
+ where ratelimiting was made too strict.
Clients should be able to create 10 rooms in a row
without hitting rate limits, using default rate limit config.
(We override rate limiting config back to its default value.)
diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py
index d60665254e..07c81d7f76 100644
--- a/tests/rest/client/test_sync.py
+++ b/tests/rest/client/test_sync.py
@@ -642,7 +642,7 @@ class SyncCacheTestCase(unittest.HomeserverTestCase):
def test_noop_sync_does_not_tightloop(self) -> None:
"""If the sync times out, we shouldn't cache the result
- Essentially a regression test for #8518.
+ Essentially a regression test for https://github.com/matrix-org/synapse/issues/8518.
"""
self.user_id = self.register_user("kermit", "monkey")
self.tok = self.login("kermit", "monkey")
diff --git a/tests/server.py b/tests/server.py
index c8342db399..2b63ed3dd8 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -88,7 +88,7 @@ from synapse.module_api.callbacks.third_party_event_rules_callbacks import (
from synapse.server import HomeServer
from synapse.storage import DataStore
from synapse.storage.database import LoggingDatabaseConnection
-from synapse.storage.engines import PostgresEngine, create_engine
+from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import prepare_database
from synapse.types import ISynapseReactor, JsonDict
from synapse.util import Clock
@@ -974,7 +974,7 @@ def setup_test_homeserver(
database_config = {
"name": "psycopg2",
"args": {
- "database": test_db,
+ "dbname": test_db,
"host": POSTGRES_HOST,
"password": POSTGRES_PASSWORD,
"user": POSTGRES_USER,
@@ -1029,18 +1029,15 @@ def setup_test_homeserver(
# Create the database before we actually try and connect to it, based off
# the template database we generate in setupdb()
- if isinstance(db_engine, PostgresEngine):
- import psycopg2.extensions
-
+ if USE_POSTGRES_FOR_TESTS:
db_conn = db_engine.module.connect(
- database=POSTGRES_BASE_DB,
+ dbname=POSTGRES_BASE_DB,
user=POSTGRES_USER,
host=POSTGRES_HOST,
port=POSTGRES_PORT,
password=POSTGRES_PASSWORD,
)
- assert isinstance(db_conn, psycopg2.extensions.connection)
- db_conn.autocommit = True
+ db_engine.attempt_to_set_autocommit(db_conn, True)
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
cur.execute(
@@ -1065,13 +1062,12 @@ def setup_test_homeserver(
hs.setup()
- if isinstance(db_engine, PostgresEngine):
+ if USE_POSTGRES_FOR_TESTS:
database_pool = hs.get_datastores().databases[0]
# We need to do cleanup on PostgreSQL
def cleanup() -> None:
import psycopg2
- import psycopg2.extensions
# Close all the db pools
database_pool._db_pool.close()
@@ -1080,14 +1076,13 @@ def setup_test_homeserver(
# Drop the test database
db_conn = db_engine.module.connect(
- database=POSTGRES_BASE_DB,
+ dbname=POSTGRES_BASE_DB,
user=POSTGRES_USER,
host=POSTGRES_HOST,
port=POSTGRES_PORT,
password=POSTGRES_PASSWORD,
)
- assert isinstance(db_conn, psycopg2.extensions.connection)
- db_conn.autocommit = True
+ db_engine.attempt_to_set_autocommit(db_conn, True)
cur = db_conn.cursor()
# Try a few times to drop the DB. Some things may hold on to the
diff --git a/tests/storage/databases/main/test_end_to_end_keys.py b/tests/storage/databases/main/test_end_to_end_keys.py
new file mode 100644
index 0000000000..23e6f82c75
--- /dev/null
+++ b/tests/storage/databases/main/test_end_to_end_keys.py
@@ -0,0 +1,121 @@
+# Copyright 2023 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List, Optional, Tuple
+
+from twisted.test.proto_helpers import MemoryReactor
+
+from synapse.server import HomeServer
+from synapse.storage._base import db_to_json
+from synapse.storage.database import LoggingTransaction
+from synapse.types import JsonDict
+from synapse.util import Clock
+
+from tests.unittest import HomeserverTestCase
+
+
+class EndToEndKeyWorkerStoreTestCase(HomeserverTestCase):
+ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
+ self.store = hs.get_datastores().main
+
+ def test_get_master_cross_signing_key_updatable_before(self) -> None:
+ # Should return False, None when there is no master key.
+ alice = "@alice:test"
+ exists, timestamp = self.get_success(
+ self.store.get_master_cross_signing_key_updatable_before(alice)
+ )
+ self.assertIs(exists, False)
+ self.assertIsNone(timestamp)
+
+ # Upload a master key.
+ dummy_key = {"keys": {"a": "b"}}
+ self.get_success(
+ self.store.set_e2e_cross_signing_key(alice, "master", dummy_key)
+ )
+
+ # Should now find that the key exists.
+ exists, timestamp = self.get_success(
+ self.store.get_master_cross_signing_key_updatable_before(alice)
+ )
+ self.assertIs(exists, True)
+ self.assertIsNone(timestamp)
+
+ # Write an updateable_before timestamp.
+ written_timestamp = self.get_success(
+ self.store.allow_master_cross_signing_key_replacement_without_uia(
+ alice, 1000
+ )
+ )
+
+ # Should now find that the key exists.
+ exists, timestamp = self.get_success(
+ self.store.get_master_cross_signing_key_updatable_before(alice)
+ )
+ self.assertIs(exists, True)
+ self.assertEqual(timestamp, written_timestamp)
+
+ def test_master_replacement_only_applies_to_latest_master_key(
+ self,
+ ) -> None:
+ """We shouldn't allow updates w/o UIA to old master keys or other key types."""
+ alice = "@alice:test"
+ # Upload two master keys.
+ key1 = {"keys": {"a": "b"}}
+ key2 = {"keys": {"c": "d"}}
+ key3 = {"keys": {"e": "f"}}
+ self.get_success(self.store.set_e2e_cross_signing_key(alice, "master", key1))
+ self.get_success(self.store.set_e2e_cross_signing_key(alice, "other", key2))
+ self.get_success(self.store.set_e2e_cross_signing_key(alice, "master", key3))
+
+ # Third key should be the current one.
+ key = self.get_success(
+ self.store.get_e2e_cross_signing_key(alice, "master", alice)
+ )
+ self.assertEqual(key, key3)
+
+ timestamp = self.get_success(
+ self.store.allow_master_cross_signing_key_replacement_without_uia(
+ alice, 1000
+ )
+ )
+ assert timestamp is not None
+
+ def check_timestamp_column(
+ txn: LoggingTransaction,
+ ) -> List[Tuple[JsonDict, Optional[int]]]:
+ """Fetch all rows for Alice's keys."""
+ txn.execute(
+ """
+ SELECT keydata, updatable_without_uia_before_ms
+ FROM e2e_cross_signing_keys
+ WHERE user_id = ?
+ ORDER BY stream_id ASC;
+ """,
+ (alice,),
+ )
+ return [(db_to_json(keydata), ts) for keydata, ts in txn.fetchall()]
+
+ values = self.get_success(
+ self.store.db_pool.runInteraction(
+ "check_timestamp_column",
+ check_timestamp_column,
+ )
+ )
+ self.assertEqual(
+ values,
+ [
+ (key1, None),
+ (key2, None),
+ (key3, timestamp),
+ ],
+ )
diff --git a/tests/storage/databases/main/test_lock.py b/tests/storage/databases/main/test_lock.py
index 35f77052a7..6c4d44c05c 100644
--- a/tests/storage/databases/main/test_lock.py
+++ b/tests/storage/databases/main/test_lock.py
@@ -66,9 +66,9 @@ class LockTestCase(unittest.HomeserverTestCase):
# Run the tasks to completion.
# To work around `Linearizer`s using a different reactor to sleep when
- # contended (#12841), we call `runUntilCurrent` on
- # `twisted.internet.reactor`, which is a different reactor to that used
- # by the homeserver.
+ # contended (https://github.com/matrix-org/synapse/issues/12841), we call
+ # `runUntilCurrent` on `twisted.internet.reactor`, which is a different
+ # reactor to that used by the homeserver.
assert isinstance(reactor, ReactorBase)
self.get_success(task1)
reactor.runUntilCurrent()
@@ -217,9 +217,9 @@ class ReadWriteLockTestCase(unittest.HomeserverTestCase):
# Run the tasks to completion.
# To work around `Linearizer`s using a different reactor to sleep when
- # contended (#12841), we call `runUntilCurrent` on
- # `twisted.internet.reactor`, which is a different reactor to that used
- # by the homeserver.
+ # contended (https://github.com/matrix-org/synapse/issues/12841), we call
+ # `runUntilCurrent` on `twisted.internet.reactor`, which is a different
+ # reactor to that used by the homeserver.
assert isinstance(reactor, ReactorBase)
self.get_success(task1)
reactor.runUntilCurrent()
@@ -269,9 +269,9 @@ class ReadWriteLockTestCase(unittest.HomeserverTestCase):
# Run the tasks to completion.
# To work around `Linearizer`s using a different reactor to sleep when
- # contended (#12841), we call `runUntilCurrent` on
- # `twisted.internet.reactor`, which is a different reactor to that used
- # by the homeserver.
+ # contended (https://github.com/matrix-org/synapse/issues/12841), we call
+ # `runUntilCurrent` on `twisted.internet.reactor`, which is a different
+ # reactor to that used by the homeserver.
assert isinstance(reactor, ReactorBase)
self.get_success(task1)
reactor.runUntilCurrent()
diff --git a/tests/storage/test_database.py b/tests/storage/test_database.py
index 8cd7c89ca2..aa8c76f187 100644
--- a/tests/storage/test_database.py
+++ b/tests/storage/test_database.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Callable, Tuple
+from typing import Callable, List, Tuple
from unittest.mock import Mock, call
from twisted.internet import defer
@@ -29,6 +29,7 @@ from synapse.storage.database import (
from synapse.util import Clock
from tests import unittest
+from tests.utils import USE_POSTGRES_FOR_TESTS
class TupleComparisonClauseTestCase(unittest.TestCase):
@@ -213,7 +214,8 @@ class CallbacksTestCase(unittest.HomeserverTestCase):
after_callback, exception_callback = self._run_interaction(_test_txn)
# Calling both `after_callback`s when the first attempt failed is rather
- # surprising (#12184). Let's document the behaviour in a test.
+ # surprising (https://github.com/matrix-org/synapse/issues/12184).
+ # Let's document the behaviour in a test.
after_callback.assert_has_calls(
[
call(123, 456, extra=789),
@@ -278,3 +280,85 @@ class CancellationTestCase(unittest.HomeserverTestCase):
]
)
self.assertEqual(exception_callback.call_count, 6) # no additional calls
+
+
+class PostgresReplicaIdentityTestCase(unittest.HomeserverTestCase):
+ if not USE_POSTGRES_FOR_TESTS:
+ skip = "Requires Postgres"
+
+ def prepare(
+ self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer
+ ) -> None:
+ self.db_pools = homeserver.get_datastores().databases
+
+ def test_all_tables_have_postgres_replica_identity(self) -> None:
+ """
+ Tests that all tables have a Postgres REPLICA IDENTITY.
+ (See https://github.com/matrix-org/synapse/issues/16224).
+
+ Tables with a PRIMARY KEY have an implied REPLICA IDENTITY and are fine.
+ Other tables need them to be set with `ALTER TABLE`.
+
+ A REPLICA IDENTITY is required for Postgres logical replication to work
+ properly without blocking updates and deletes.
+ """
+
+ sql = """
+ -- Select tables that have no primary key and use the default replica identity rule
+ -- (the default is to use the primary key)
+ WITH tables_no_pkey AS (
+ SELECT tbl.table_schema, tbl.table_name
+ FROM information_schema.tables tbl
+ WHERE table_type = 'BASE TABLE'
+ AND table_schema not in ('pg_catalog', 'information_schema')
+ AND NOT EXISTS (
+ SELECT 1
+ FROM information_schema.table_constraints tc
+ WHERE tc.constraint_type = 'PRIMARY KEY'
+ AND tc.table_schema = tbl.table_schema
+ AND tc.table_name = tbl.table_name
+ )
+ )
+ SELECT pg_class.oid::regclass FROM tables_no_pkey INNER JOIN pg_class ON pg_class.oid::regclass = table_name::regclass
+ WHERE relreplident = 'd'
+
+ UNION
+
+ -- Also select tables that use an index as a replica identity
+ -- but where the index doesn't exist
+ -- (e.g. it could have been deleted)
+ SELECT pg_class.oid::regclass
+ FROM information_schema.tables tbl
+ INNER JOIN pg_class ON pg_class.oid::regclass = table_name::regclass
+ WHERE table_type = 'BASE TABLE'
+ AND table_schema not in ('pg_catalog', 'information_schema')
+
+ -- 'i' means an index is used as the replica identity
+ AND relreplident = 'i'
+
+ -- look for indices that are marked as the replica identity
+ AND NOT EXISTS (
+ SELECT indexrelid::regclass
+ FROM pg_index
+ WHERE indrelid = pg_class.oid::regclass AND indisreplident
+ )
+ """
+
+ def _list_tables_with_missing_replica_identities_txn(
+ txn: LoggingTransaction,
+ ) -> List[str]:
+ txn.execute(sql)
+ return [table_name for table_name, in txn]
+
+ for pool in self.db_pools:
+ missing = self.get_success(
+ pool.runInteraction(
+ "test_list_missing_replica_identities",
+ _list_tables_with_missing_replica_identities_txn,
+ )
+ )
+ self.assertEqual(
+ len(missing),
+ 0,
+ f"The following tables in the {pool.name()!r} database are missing REPLICA IDENTITIES: {missing!r}.",
+ )
diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py
index d3e20f44b2..66a027887d 100644
--- a/tests/storage/test_event_federation.py
+++ b/tests/storage/test_event_federation.py
@@ -1060,7 +1060,7 @@ class EventFederationWorkerStoreTestCase(tests.unittest.HomeserverTestCase):
self,
) -> None:
"""
- A test that reproduces #13929 (Postgres only).
+ A test that reproduces https://github.com/matrix-org/synapse/issues/13929 (Postgres only).
Test to make sure we can still get backfill points after many failed pull
attempts that cause us to backoff to the limit. Even if the backoff formula
diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py
index 52ffa91c81..e3dc3623cb 100644
--- a/tests/storage/test_room_search.py
+++ b/tests/storage/test_room_search.py
@@ -93,7 +93,7 @@ class EventSearchInsertionTest(HomeserverTestCase):
both strings and integers. When using Postgres, integers are automatically
converted to strings.
- Regression test for #11918.
+ Regression test for https://github.com/matrix-org/synapse/issues/11918.
"""
store = self.hs.get_datastores().main
diff --git a/tests/util/test_check_dependencies.py b/tests/util/test_check_dependencies.py
index aa20fe6780..c1392d8bfc 100644
--- a/tests/util/test_check_dependencies.py
+++ b/tests/util/test_check_dependencies.py
@@ -89,7 +89,8 @@ class TestDependencyChecker(TestCase):
def test_version_reported_as_none(self) -> None:
"""Complain if importlib.metadata.version() returns None.
- This shouldn't normally happen, but it was seen in the wild (#12223).
+ This shouldn't normally happen, but it was seen in the wild
+ (https://github.com/matrix-org/synapse/issues/12223).
"""
with patch(
"synapse.util.check_dependencies.metadata.requires",
@@ -148,7 +149,7 @@ class TestDependencyChecker(TestCase):
"""
Tests that release candidates count as far as satisfying a dependency
is concerned.
- (Regression test, see #12176.)
+ (Regression test, see https://github.com/matrix-org/synapse/issues/12176.)
"""
with patch(
"synapse.util.check_dependencies.metadata.requires",
@@ -162,7 +163,10 @@ class TestDependencyChecker(TestCase):
check_requirements()
def test_setuptools_rust_ignored(self) -> None:
- """Test a workaround for a `poetry build` problem. Reproduces #13926."""
+ """
+ Test a workaround for a `poetry build` problem. Reproduces
+ https://github.com/matrix-org/synapse/issues/13926.
+ """
with patch(
"synapse.util.check_dependencies.metadata.requires",
return_value=["setuptools_rust >= 1.3"],
diff --git a/tests/util/test_itertools.py b/tests/util/test_itertools.py
index 406c16cdcf..fabb05c7e4 100644
--- a/tests/util/test_itertools.py
+++ b/tests/util/test_itertools.py
@@ -13,7 +13,11 @@
# limitations under the License.
from typing import Dict, Iterable, List, Sequence
-from synapse.util.iterutils import chunk_seq, sorted_topologically
+from synapse.util.iterutils import (
+ chunk_seq,
+ sorted_topologically,
+ sorted_topologically_batched,
+)
from tests.unittest import TestCase
@@ -107,3 +111,73 @@ class SortTopologically(TestCase):
graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3, 2, 1]}
self.assertEqual(list(sorted_topologically([4, 3, 2, 1], graph)), [1, 2, 3, 4])
+
+
+class SortTopologicallyBatched(TestCase):
+ "Test cases for `sorted_topologically_batched`"
+
+ def test_empty(self) -> None:
+ "Test that an empty graph works correctly"
+
+ graph: Dict[int, List[int]] = {}
+ self.assertEqual(list(sorted_topologically_batched([], graph)), [])
+
+ def test_handle_empty_graph(self) -> None:
+ "Test that a graph where a node doesn't have an entry is treated as empty"
+
+ graph: Dict[int, List[int]] = {}
+
+ # For disconnected nodes the output is simply sorted.
+ self.assertEqual(list(sorted_topologically_batched([1, 2], graph)), [[1, 2]])
+
+ def test_disconnected(self) -> None:
+ "Test that a graph with no edges work"
+
+ graph: Dict[int, List[int]] = {1: [], 2: []}
+
+ # For disconnected nodes the output is simply sorted.
+ self.assertEqual(list(sorted_topologically_batched([1, 2], graph)), [[1, 2]])
+
+ def test_linear(self) -> None:
+ "Test that a simple `4 -> 3 -> 2 -> 1` graph works"
+
+ graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3]}
+
+ self.assertEqual(
+ list(sorted_topologically_batched([4, 3, 2, 1], graph)),
+ [[1], [2], [3], [4]],
+ )
+
+ def test_subset(self) -> None:
+ "Test that only sorting a subset of the graph works"
+ graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3]}
+
+ self.assertEqual(list(sorted_topologically_batched([4, 3], graph)), [[3], [4]])
+
+ def test_fork(self) -> None:
+ "Test that a forked graph works"
+ graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [1], 4: [2, 3]}
+
+ # Valid orderings are `[1, 3, 2, 4]` or `[1, 2, 3, 4]`, but we should
+ # always get the same one.
+ self.assertEqual(
+ list(sorted_topologically_batched([4, 3, 2, 1], graph)), [[1], [2, 3], [4]]
+ )
+
+ def test_duplicates(self) -> None:
+ "Test that a graph with duplicate edges work"
+ graph: Dict[int, List[int]] = {1: [], 2: [1, 1], 3: [2, 2], 4: [3]}
+
+ self.assertEqual(
+ list(sorted_topologically_batched([4, 3, 2, 1], graph)),
+ [[1], [2], [3], [4]],
+ )
+
+ def test_multiple_paths(self) -> None:
+ "Test that a graph with multiple paths between two nodes work"
+ graph: Dict[int, List[int]] = {1: [], 2: [1], 3: [2], 4: [3, 2, 1]}
+
+ self.assertEqual(
+ list(sorted_topologically_batched([4, 3, 2, 1], graph)),
+ [[1], [2], [3], [4]],
+ )
diff --git a/tests/utils.py b/tests/utils.py
index e73b46944b..a0c87ad628 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -80,7 +80,7 @@ def setupdb() -> None:
# Set up in the db
db_conn = db_engine.module.connect(
- database=POSTGRES_BASE_DB,
+ dbname=POSTGRES_BASE_DB,
user=POSTGRES_USER,
host=POSTGRES_HOST,
port=POSTGRES_PORT,
|
