diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py
index 5a965f233b..21c5309740 100644
--- a/tests/app/test_openid_listener.py
+++ b/tests/app/test_openid_listener.py
@@ -31,9 +31,7 @@ from tests.unittest import HomeserverTestCase
class FederationReaderOpenIDListenerTests(HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver(
- federation_http_client=None, homeserver_to_use=GenericWorkerServer
- )
+ hs = self.setup_test_homeserver(homeserver_to_use=GenericWorkerServer)
return hs
def default_config(self) -> JsonDict:
@@ -91,9 +89,7 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase):
@patch("synapse.app.homeserver.KeyResource", new=Mock())
class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver(
- federation_http_client=None, homeserver_to_use=SynapseHomeServer
- )
+ hs = self.setup_test_homeserver(homeserver_to_use=SynapseHomeServer)
return hs
@parameterized.expand(
diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py
index c9a610db9a..6a52af4d82 100644
--- a/tests/events/test_utils.py
+++ b/tests/events/test_utils.py
@@ -140,18 +140,16 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
},
)
- # As of MSC2176 we now redact the membership and prev_states keys.
+ # As of room versions we now redact the membership, prev_states, and origin keys.
self.run_test(
- {"type": "A", "prev_state": "prev_state", "membership": "join"},
- {"type": "A", "content": {}, "signatures": {}, "unsigned": {}},
- room_version=RoomVersions.MSC2176,
- )
-
- # As of MSC3989 we now redact the origin key.
- self.run_test(
- {"type": "A", "origin": "example.com"},
+ {
+ "type": "A",
+ "prev_state": "prev_state",
+ "membership": "join",
+ "origin": "example.com",
+ },
{"type": "A", "content": {}, "signatures": {}, "unsigned": {}},
- room_version=RoomVersions.MSC3989,
+ room_version=RoomVersions.V11,
)
def test_unsigned(self) -> None:
@@ -236,7 +234,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
"signatures": {},
"unsigned": {},
},
- room_version=RoomVersions.MSC2176,
+ room_version=RoomVersions.V11,
)
def test_power_levels(self) -> None:
@@ -286,7 +284,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
"signatures": {},
"unsigned": {},
},
- room_version=RoomVersions.MSC2176,
+ room_version=RoomVersions.V11,
)
def test_alias_event(self) -> None:
@@ -349,7 +347,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
"signatures": {},
"unsigned": {},
},
- room_version=RoomVersions.MSC2176,
+ room_version=RoomVersions.V11,
)
def test_join_rules(self) -> None:
@@ -472,7 +470,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
"signatures": {},
"unsigned": {},
},
- room_version=RoomVersions.MSC3821,
+ room_version=RoomVersions.V11,
)
# Ensure this doesn't break if an invalid field is sent.
@@ -491,7 +489,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
"signatures": {},
"unsigned": {},
},
- room_version=RoomVersions.MSC3821,
+ room_version=RoomVersions.V11,
)
self.run_test(
@@ -509,7 +507,7 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
"signatures": {},
"unsigned": {},
},
- room_version=RoomVersions.MSC3821,
+ room_version=RoomVersions.V11,
)
def test_relations(self) -> None:
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index ee48f9e546..66215af2b8 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -41,7 +41,6 @@ class DeviceTestCase(unittest.HomeserverTestCase):
self.appservice_api = mock.Mock()
hs = self.setup_test_homeserver(
"server",
- federation_http_client=None,
application_service_api=self.appservice_api,
)
handler = hs.get_device_handler()
@@ -401,7 +400,7 @@ class DeviceTestCase(unittest.HomeserverTestCase):
class DehydrationTestCase(unittest.HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver("server", federation_http_client=None)
+ hs = self.setup_test_homeserver("server")
handler = hs.get_device_handler()
assert isinstance(handler, DeviceHandler)
self.handler = handler
diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py
index bf0862ed54..5f11d5df11 100644
--- a/tests/handlers/test_federation.py
+++ b/tests/handlers/test_federation.py
@@ -57,7 +57,7 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase):
]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver(federation_http_client=None)
+ hs = self.setup_test_homeserver()
self.handler = hs.get_federation_handler()
self.store = hs.get_datastores().main
return hs
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index 19f5322317..fd66d573d2 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -993,7 +993,6 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
hs = self.setup_test_homeserver(
"server",
- federation_http_client=None,
federation_sender=Mock(spec=FederationSender),
)
return hs
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 8d8584609b..54eeec228e 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -587,17 +587,16 @@ class RegistrationTestCase(unittest.HomeserverTestCase):
self.assertFalse(self.get_success(d))
def test_invalid_user_id(self) -> None:
- invalid_user_id = "+abcd"
+ invalid_user_id = "^abcd"
self.get_failure(
self.handler.register_user(localpart=invalid_user_id), SynapseError
)
- @override_config({"experimental_features": {"msc4009_e164_mxids": True}})
- def text_extended_user_ids(self) -> None:
- """+ should be allowed according to MSC4009."""
- valid_user_id = "+1234"
+ def test_special_chars(self) -> None:
+ """Ensure that characters which are allowed in Matrix IDs work."""
+ valid_user_id = "a1234_-./=+"
user_id = self.get_success(self.handler.register_user(localpart=valid_user_id))
- self.assertEqual(user_id, valid_user_id)
+ self.assertEqual(user_id, f"@{valid_user_id}:test")
def test_invalid_user_id_length(self) -> None:
invalid_user_id = "x" * 256
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index 94518a7196..5da1d95f0b 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -17,6 +17,8 @@ import json
from typing import Dict, List, Set
from unittest.mock import ANY, Mock, call
+from netaddr import IPSet
+
from twisted.test.proto_helpers import MemoryReactor
from twisted.web.resource import Resource
@@ -24,6 +26,7 @@ from synapse.api.constants import EduTypes
from synapse.api.errors import AuthError
from synapse.federation.transport.server import TransportLayerServer
from synapse.handlers.typing import TypingWriterHandler
+from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
from synapse.server import HomeServer
from synapse.types import JsonDict, Requester, UserID, create_requester
from synapse.util import Clock
@@ -76,6 +79,13 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
# we mock out the federation client too
self.mock_federation_client = Mock(spec=["put_json"])
self.mock_federation_client.put_json.return_value = make_awaitable((200, "OK"))
+ self.mock_federation_client.agent = MatrixFederationAgent(
+ reactor,
+ tls_client_options_factory=None,
+ user_agent=b"SynapseInTrialTest/0.0.0",
+ ip_allowlist=None,
+ ip_blocklist=IPSet(),
+ )
# the tests assume that we are starting at unix time 1000
reactor.pump((1000,))
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index 15a7dc6818..9785dd698b 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -356,7 +356,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
support_user_id, ProfileInfo("I love support me", None)
)
)
- profile = self.get_success(self.store.get_user_in_directory(support_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(support_user_id))
self.assertIsNone(profile)
display_name = "display_name"
@@ -364,7 +364,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
self.get_success(
self.handler.handle_local_profile_change(regular_user_id, profile_info)
)
- profile = self.get_success(self.store.get_user_in_directory(regular_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(regular_user_id))
assert profile is not None
self.assertTrue(profile["display_name"] == display_name)
@@ -383,7 +383,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
)
# profile is in directory
- profile = self.get_success(self.store.get_user_in_directory(r_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(r_user_id))
assert profile is not None
self.assertTrue(profile["display_name"] == display_name)
@@ -392,7 +392,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
self.get_success(self.handler.handle_local_user_deactivated(r_user_id))
# profile is not in directory
- profile = self.get_success(self.store.get_user_in_directory(r_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(r_user_id))
self.assertIsNone(profile)
# update profile after deactivation
@@ -401,7 +401,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
)
# profile is furthermore not in directory
- profile = self.get_success(self.store.get_user_in_directory(r_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(r_user_id))
self.assertIsNone(profile)
def test_handle_local_profile_change_with_appservice_user(self) -> None:
@@ -411,7 +411,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
)
# profile is not in directory
- profile = self.get_success(self.store.get_user_in_directory(as_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(as_user_id))
self.assertIsNone(profile)
# update profile
@@ -421,13 +421,13 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
)
# profile is still not in directory
- profile = self.get_success(self.store.get_user_in_directory(as_user_id))
+ profile = self.get_success(self.store._get_user_in_directory(as_user_id))
self.assertIsNone(profile)
def test_handle_local_profile_change_with_appservice_sender(self) -> None:
# profile is not in directory
profile = self.get_success(
- self.store.get_user_in_directory(self.appservice.sender)
+ self.store._get_user_in_directory(self.appservice.sender)
)
self.assertIsNone(profile)
@@ -441,7 +441,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
# profile is still not in directory
profile = self.get_success(
- self.store.get_user_in_directory(self.appservice.sender)
+ self.store._get_user_in_directory(self.appservice.sender)
)
self.assertIsNone(profile)
diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py
index b5f4a60fe5..ab94f3f67a 100644
--- a/tests/http/test_matrixfederationclient.py
+++ b/tests/http/test_matrixfederationclient.py
@@ -11,8 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Generator
-from unittest.mock import Mock
+from typing import Any, Dict, Generator
+from unittest.mock import ANY, Mock, create_autospec
from netaddr import IPSet
from parameterized import parameterized
@@ -21,10 +21,12 @@ from twisted.internet import defer
from twisted.internet.defer import Deferred, TimeoutError
from twisted.internet.error import ConnectingCancelledError, DNSLookupError
from twisted.test.proto_helpers import MemoryReactor, StringTransport
-from twisted.web.client import ResponseNeverReceived
+from twisted.web.client import Agent, ResponseNeverReceived
from twisted.web.http import HTTPChannel
+from twisted.web.http_headers import Headers
-from synapse.api.errors import RequestSendFailed
+from synapse.api.errors import HttpResponseException, RequestSendFailed
+from synapse.config._base import ConfigError
from synapse.http.matrixfederationclient import (
ByteParser,
MatrixFederationHttpClient,
@@ -39,7 +41,9 @@ from synapse.logging.context import (
from synapse.server import HomeServer
from synapse.util import Clock
+from tests.replication._base import BaseMultiWorkerStreamTestCase
from tests.server import FakeTransport
+from tests.test_utils import FakeResponse
from tests.unittest import HomeserverTestCase, override_config
@@ -658,3 +662,275 @@ class FederationClientTests(HomeserverTestCase):
self.assertEqual(self.cl.max_short_retry_delay_seconds, 7)
self.assertEqual(self.cl.max_long_retries, 20)
self.assertEqual(self.cl.max_short_retries, 5)
+
+
+class FederationClientProxyTests(BaseMultiWorkerStreamTestCase):
+ def default_config(self) -> Dict[str, Any]:
+ conf = super().default_config()
+ conf["instance_map"] = {
+ "main": {"host": "testserv", "port": 8765},
+ "federation_sender": {"host": "testserv", "port": 1001},
+ }
+ return conf
+
+ @override_config(
+ {
+ "outbound_federation_restricted_to": ["federation_sender"],
+ "worker_replication_secret": "secret",
+ }
+ )
+ def test_proxy_requests_through_federation_sender_worker(self) -> None:
+ """
+ Test that all outbound federation requests go through the `federation_sender`
+ worker
+ """
+ # Mock out the `MatrixFederationHttpClient` of the `federation_sender` instance
+ # so we can act like some remote server responding to requests
+ mock_client_on_federation_sender = Mock()
+ mock_agent_on_federation_sender = create_autospec(Agent, spec_set=True)
+ mock_client_on_federation_sender.agent = mock_agent_on_federation_sender
+
+ # Create the `federation_sender` worker
+ self.make_worker_hs(
+ "synapse.app.generic_worker",
+ {"worker_name": "federation_sender"},
+ federation_http_client=mock_client_on_federation_sender,
+ )
+
+ # Fake `remoteserv:8008` responding to requests
+ mock_agent_on_federation_sender.request.side_effect = (
+ lambda *args, **kwargs: defer.succeed(
+ FakeResponse.json(
+ payload={
+ "foo": "bar",
+ }
+ )
+ )
+ )
+
+ # This federation request from the main process should be proxied through the
+ # `federation_sender` worker off to the remote server
+ test_request_from_main_process_d = defer.ensureDeferred(
+ self.hs.get_federation_http_client().get_json("remoteserv:8008", "foo/bar")
+ )
+
+ # Pump the reactor so our deferred goes through the motions
+ self.pump()
+
+ # Make sure that the request was proxied through the `federation_sender` worker
+ mock_agent_on_federation_sender.request.assert_called_once_with(
+ b"GET",
+ b"matrix-federation://remoteserv:8008/foo/bar",
+ headers=ANY,
+ bodyProducer=ANY,
+ )
+
+ # Make sure the response is as expected back on the main worker
+ res = self.successResultOf(test_request_from_main_process_d)
+ self.assertEqual(res, {"foo": "bar"})
+
+ @override_config(
+ {
+ "outbound_federation_restricted_to": ["federation_sender"],
+ "worker_replication_secret": "secret",
+ }
+ )
+ def test_proxy_request_with_network_error_through_federation_sender_worker(
+ self,
+ ) -> None:
+ """
+ Test that when the outbound federation request fails with a network related
+ error, a sensible error makes its way back to the main process.
+ """
+ # Mock out the `MatrixFederationHttpClient` of the `federation_sender` instance
+ # so we can act like some remote server responding to requests
+ mock_client_on_federation_sender = Mock()
+ mock_agent_on_federation_sender = create_autospec(Agent, spec_set=True)
+ mock_client_on_federation_sender.agent = mock_agent_on_federation_sender
+
+ # Create the `federation_sender` worker
+ self.make_worker_hs(
+ "synapse.app.generic_worker",
+ {"worker_name": "federation_sender"},
+ federation_http_client=mock_client_on_federation_sender,
+ )
+
+ # Fake `remoteserv:8008` responding to requests
+ mock_agent_on_federation_sender.request.side_effect = (
+ lambda *args, **kwargs: defer.fail(ResponseNeverReceived("fake error"))
+ )
+
+ # This federation request from the main process should be proxied through the
+ # `federation_sender` worker off to the remote server
+ test_request_from_main_process_d = defer.ensureDeferred(
+ self.hs.get_federation_http_client().get_json("remoteserv:8008", "foo/bar")
+ )
+
+ # Pump the reactor so our deferred goes through the motions. We pump with 10
+ # seconds (0.1 * 100) so the `MatrixFederationHttpClient` runs out of retries
+ # and finally passes along the error response.
+ self.pump(0.1)
+
+ # Make sure that the request was proxied through the `federation_sender` worker
+ mock_agent_on_federation_sender.request.assert_called_with(
+ b"GET",
+ b"matrix-federation://remoteserv:8008/foo/bar",
+ headers=ANY,
+ bodyProducer=ANY,
+ )
+
+ # Make sure we get some sort of error back on the main worker
+ failure_res = self.failureResultOf(test_request_from_main_process_d)
+ self.assertIsInstance(failure_res.value, RequestSendFailed)
+ self.assertIsInstance(failure_res.value.inner_exception, HttpResponseException)
+ self.assertEqual(failure_res.value.inner_exception.code, 502)
+
+ @override_config(
+ {
+ "outbound_federation_restricted_to": ["federation_sender"],
+ "worker_replication_secret": "secret",
+ }
+ )
+ def test_proxy_requests_and_discards_hop_by_hop_headers(self) -> None:
+ """
+ Test to make sure hop-by-hop headers and addional headers defined in the
+ `Connection` header are discarded when proxying requests
+ """
+ # Mock out the `MatrixFederationHttpClient` of the `federation_sender` instance
+ # so we can act like some remote server responding to requests
+ mock_client_on_federation_sender = Mock()
+ mock_agent_on_federation_sender = create_autospec(Agent, spec_set=True)
+ mock_client_on_federation_sender.agent = mock_agent_on_federation_sender
+
+ # Create the `federation_sender` worker
+ self.make_worker_hs(
+ "synapse.app.generic_worker",
+ {"worker_name": "federation_sender"},
+ federation_http_client=mock_client_on_federation_sender,
+ )
+
+ # Fake `remoteserv:8008` responding to requests
+ mock_agent_on_federation_sender.request.side_effect = lambda *args, **kwargs: defer.succeed(
+ FakeResponse(
+ code=200,
+ body=b'{"foo": "bar"}',
+ headers=Headers(
+ {
+ "Content-Type": ["application/json"],
+ "Connection": ["close, X-Foo, X-Bar"],
+ # Should be removed because it's defined in the `Connection` header
+ "X-Foo": ["foo"],
+ "X-Bar": ["bar"],
+ # Should be removed because it's a hop-by-hop header
+ "Proxy-Authorization": "abcdef",
+ }
+ ),
+ )
+ )
+
+ # This federation request from the main process should be proxied through the
+ # `federation_sender` worker off to the remote server
+ test_request_from_main_process_d = defer.ensureDeferred(
+ self.hs.get_federation_http_client().get_json_with_headers(
+ "remoteserv:8008", "foo/bar"
+ )
+ )
+
+ # Pump the reactor so our deferred goes through the motions
+ self.pump()
+
+ # Make sure that the request was proxied through the `federation_sender` worker
+ mock_agent_on_federation_sender.request.assert_called_once_with(
+ b"GET",
+ b"matrix-federation://remoteserv:8008/foo/bar",
+ headers=ANY,
+ bodyProducer=ANY,
+ )
+
+ res, headers = self.successResultOf(test_request_from_main_process_d)
+ header_names = set(headers.keys())
+
+ # Make sure the response does not include the hop-by-hop headers
+ self.assertNotIn(b"X-Foo", header_names)
+ self.assertNotIn(b"X-Bar", header_names)
+ self.assertNotIn(b"Proxy-Authorization", header_names)
+ # Make sure the response is as expected back on the main worker
+ self.assertEqual(res, {"foo": "bar"})
+
+ @override_config(
+ {
+ "outbound_federation_restricted_to": ["federation_sender"],
+ # `worker_replication_secret` is set here so that the test setup is able to pass
+ # but the actual homserver creation test is in the test body below
+ "worker_replication_secret": "secret",
+ }
+ )
+ def test_not_able_to_proxy_requests_through_federation_sender_worker_when_no_secret_configured(
+ self,
+ ) -> None:
+ """
+ Test that we aren't able to proxy any outbound federation requests when
+ `worker_replication_secret` is not configured.
+ """
+ with self.assertRaises(ConfigError):
+ # Create the `federation_sender` worker
+ self.make_worker_hs(
+ "synapse.app.generic_worker",
+ {
+ "worker_name": "federation_sender",
+ # Test that we aren't able to proxy any outbound federation requests
+ # when `worker_replication_secret` is not configured.
+ "worker_replication_secret": None,
+ },
+ )
+
+ @override_config(
+ {
+ "outbound_federation_restricted_to": ["federation_sender"],
+ "worker_replication_secret": "secret",
+ }
+ )
+ def test_not_able_to_proxy_requests_through_federation_sender_worker_when_wrong_auth_given(
+ self,
+ ) -> None:
+ """
+ Test that we aren't able to proxy any outbound federation requests when the
+ wrong authorization is given.
+ """
+ # Mock out the `MatrixFederationHttpClient` of the `federation_sender` instance
+ # so we can act like some remote server responding to requests
+ mock_client_on_federation_sender = Mock()
+ mock_agent_on_federation_sender = create_autospec(Agent, spec_set=True)
+ mock_client_on_federation_sender.agent = mock_agent_on_federation_sender
+
+ # Create the `federation_sender` worker
+ self.make_worker_hs(
+ "synapse.app.generic_worker",
+ {
+ "worker_name": "federation_sender",
+ # Test that we aren't able to proxy any outbound federation requests
+ # when `worker_replication_secret` is wrong.
+ "worker_replication_secret": "wrong",
+ },
+ federation_http_client=mock_client_on_federation_sender,
+ )
+
+ # This federation request from the main process should be proxied through the
+ # `federation_sender` worker off but will fail here because it's using the wrong
+ # authorization.
+ test_request_from_main_process_d = defer.ensureDeferred(
+ self.hs.get_federation_http_client().get_json("remoteserv:8008", "foo/bar")
+ )
+
+ # Pump the reactor so our deferred goes through the motions. We pump with 10
+ # seconds (0.1 * 100) so the `MatrixFederationHttpClient` runs out of retries
+ # and finally passes along the error response.
+ self.pump(0.1)
+
+ # Make sure that the request was *NOT* proxied through the `federation_sender`
+ # worker
+ mock_agent_on_federation_sender.request.assert_not_called()
+
+ failure_res = self.failureResultOf(test_request_from_main_process_d)
+ self.assertIsInstance(failure_res.value, HttpResponseException)
+ self.assertEqual(failure_res.value.code, 401)
diff --git a/tests/http/test_proxy.py b/tests/http/test_proxy.py
new file mode 100644
index 0000000000..0dc9ba8e05
--- /dev/null
+++ b/tests/http/test_proxy.py
@@ -0,0 +1,53 @@
+# Copyright 2023 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Set
+
+from parameterized import parameterized
+
+from synapse.http.proxy import parse_connection_header_value
+
+from tests.unittest import TestCase
+
+
+class ProxyTests(TestCase):
+ @parameterized.expand(
+ [
+ [b"close, X-Foo, X-Bar", {"Close", "X-Foo", "X-Bar"}],
+ # No whitespace
+ [b"close,X-Foo,X-Bar", {"Close", "X-Foo", "X-Bar"}],
+ # More whitespace
+ [b"close, X-Foo, X-Bar", {"Close", "X-Foo", "X-Bar"}],
+ # "close" directive in not the first position
+ [b"X-Foo, X-Bar, close", {"X-Foo", "X-Bar", "Close"}],
+ # Normalizes header capitalization
+ [b"keep-alive, x-fOo, x-bAr", {"Keep-Alive", "X-Foo", "X-Bar"}],
+ # Handles header names with whitespace
+ [
+ b"keep-alive, x foo, x bar",
+ {"Keep-Alive", "X foo", "X bar"},
+ ],
+ ]
+ )
+ def test_parse_connection_header_value(
+ self,
+ connection_header_value: bytes,
+ expected_extra_headers_to_remove: Set[str],
+ ) -> None:
+ """
+ Tests that the connection header value is parsed correctly
+ """
+ self.assertEqual(
+ expected_extra_headers_to_remove,
+ parse_connection_header_value(connection_header_value),
+ )
diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py
index e0ae5a88ff..8164b0b78e 100644
--- a/tests/http/test_proxyagent.py
+++ b/tests/http/test_proxyagent.py
@@ -33,7 +33,7 @@ from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web.http import HTTPChannel
from synapse.http.client import BlocklistingReactorWrapper
-from synapse.http.connectproxyclient import ProxyCredentials
+from synapse.http.connectproxyclient import BasicProxyCredentials
from synapse.http.proxyagent import ProxyAgent, parse_proxy
from tests.http import (
@@ -205,7 +205,7 @@ class ProxyParserTests(TestCase):
"""
proxy_cred = None
if expected_credentials:
- proxy_cred = ProxyCredentials(expected_credentials)
+ proxy_cred = BasicProxyCredentials(expected_credentials)
self.assertEqual(
(
expected_scheme,
diff --git a/tests/replication/_base.py b/tests/replication/_base.py
index eb9b1f1cd9..6712ac485d 100644
--- a/tests/replication/_base.py
+++ b/tests/replication/_base.py
@@ -22,6 +22,7 @@ from twisted.test.proto_helpers import MemoryReactor
from twisted.web.resource import Resource
from synapse.app.generic_worker import GenericWorkerServer
+from synapse.config.workers import InstanceTcpLocationConfig, InstanceUnixLocationConfig
from synapse.http.site import SynapseRequest, SynapseSite
from synapse.replication.http import ReplicationRestResource
from synapse.replication.tcp.client import ReplicationDataHandler
@@ -69,10 +70,10 @@ class BaseStreamTestCase(unittest.HomeserverTestCase):
# Make a new HomeServer object for the worker
self.reactor.lookups["testserv"] = "1.2.3.4"
self.worker_hs = self.setup_test_homeserver(
- federation_http_client=None,
homeserver_to_use=GenericWorkerServer,
config=self._get_worker_hs_config(),
reactor=self.reactor,
+ federation_http_client=None,
)
# Since we use sqlite in memory databases we need to make sure the
@@ -339,7 +340,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
# `_handle_http_replication_attempt` like we do with the master HS.
instance_name = worker_hs.get_instance_name()
instance_loc = worker_hs.config.worker.instance_map.get(instance_name)
- if instance_loc:
+ if instance_loc and isinstance(instance_loc, InstanceTcpLocationConfig):
# Ensure the host is one that has a fake DNS entry.
if instance_loc.host not in self.reactor.lookups:
raise Exception(
@@ -360,6 +361,10 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
instance_loc.port,
lambda: self._handle_http_replication_attempt(worker_hs, port),
)
+ elif instance_loc and isinstance(instance_loc, InstanceUnixLocationConfig):
+ raise Exception(
+ "Unix sockets are not supported for unit tests at this time."
+ )
store = worker_hs.get_datastores().main
store.db_pool._db_pool = self.database_pool._db_pool
@@ -380,6 +385,7 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase):
server_version_string="1",
max_request_body_size=8192,
reactor=self.reactor,
+ hs=worker_hs,
)
worker_hs.get_replication_command_handler().start_replication(worker_hs)
diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py
index 08703206a9..a324b4d31d 100644
--- a/tests/replication/test_federation_sender_shard.py
+++ b/tests/replication/test_federation_sender_shard.py
@@ -14,14 +14,18 @@
import logging
from unittest.mock import Mock
+from netaddr import IPSet
+
from synapse.api.constants import EventTypes, Membership
from synapse.events.builder import EventBuilderFactory
from synapse.handlers.typing import TypingWriterHandler
+from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
from synapse.rest.admin import register_servlets_for_client_rest_resource
from synapse.rest.client import login, room
from synapse.types import UserID, create_requester
from tests.replication._base import BaseMultiWorkerStreamTestCase
+from tests.server import get_clock
from tests.test_utils import make_awaitable
logger = logging.getLogger(__name__)
@@ -41,13 +45,25 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase):
room.register_servlets,
]
+ def setUp(self) -> None:
+ super().setUp()
+
+ reactor, _ = get_clock()
+ self.matrix_federation_agent = MatrixFederationAgent(
+ reactor,
+ tls_client_options_factory=None,
+ user_agent=b"SynapseInTrialTest/0.0.0",
+ ip_allowlist=None,
+ ip_blocklist=IPSet(),
+ )
+
def test_send_event_single_sender(self) -> None:
"""Test that using a single federation sender worker correctly sends a
new event.
"""
mock_client = Mock(spec=["put_json"])
mock_client.put_json.return_value = make_awaitable({})
-
+ mock_client.agent = self.matrix_federation_agent
self.make_worker_hs(
"synapse.app.generic_worker",
{
@@ -78,6 +94,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase):
"""
mock_client1 = Mock(spec=["put_json"])
mock_client1.put_json.return_value = make_awaitable({})
+ mock_client1.agent = self.matrix_federation_agent
self.make_worker_hs(
"synapse.app.generic_worker",
{
@@ -92,6 +109,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase):
mock_client2 = Mock(spec=["put_json"])
mock_client2.put_json.return_value = make_awaitable({})
+ mock_client2.agent = self.matrix_federation_agent
self.make_worker_hs(
"synapse.app.generic_worker",
{
@@ -145,6 +163,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase):
"""
mock_client1 = Mock(spec=["put_json"])
mock_client1.put_json.return_value = make_awaitable({})
+ mock_client1.agent = self.matrix_federation_agent
self.make_worker_hs(
"synapse.app.generic_worker",
{
@@ -159,6 +178,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase):
mock_client2 = Mock(spec=["put_json"])
mock_client2.put_json.return_value = make_awaitable({})
+ mock_client2.agent = self.matrix_federation_agent
self.make_worker_hs(
"synapse.app.generic_worker",
{
diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py
index a17a1bb1d8..6f7b4bf642 100644
--- a/tests/rest/admin/test_user.py
+++ b/tests/rest/admin/test_user.py
@@ -2472,7 +2472,7 @@ class UserRestTestCase(unittest.HomeserverTestCase):
"""
# is in user directory
- profile = self.get_success(self.store.get_user_in_directory(self.other_user))
+ profile = self.get_success(self.store._get_user_in_directory(self.other_user))
assert profile is not None
self.assertTrue(profile["display_name"] == "User")
@@ -2489,7 +2489,7 @@ class UserRestTestCase(unittest.HomeserverTestCase):
self.assertTrue(channel.json_body["deactivated"])
# is not in user directory
- profile = self.get_success(self.store.get_user_in_directory(self.other_user))
+ profile = self.get_success(self.store._get_user_in_directory(self.other_user))
self.assertIsNone(profile)
# Set new displayname user
@@ -2506,7 +2506,7 @@ class UserRestTestCase(unittest.HomeserverTestCase):
self.assertEqual("Foobar", channel.json_body["displayname"])
# is not in user directory
- profile = self.get_success(self.store.get_user_in_directory(self.other_user))
+ profile = self.get_success(self.store._get_user_in_directory(self.other_user))
self.assertIsNone(profile)
def test_reactivate_user(self) -> None:
diff --git a/tests/rest/client/test_presence.py b/tests/rest/client/test_presence.py
index dcbb125a3b..e12098102b 100644
--- a/tests/rest/client/test_presence.py
+++ b/tests/rest/client/test_presence.py
@@ -40,7 +40,6 @@ class PresenceTestCase(unittest.HomeserverTestCase):
hs = self.setup_test_homeserver(
"red",
- federation_http_client=None,
federation_client=Mock(),
presence_handler=self.presence_handler,
)
diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py
index b43e95292c..6028886bd6 100644
--- a/tests/rest/client/test_redactions.py
+++ b/tests/rest/client/test_redactions.py
@@ -20,6 +20,8 @@ from synapse.api.room_versions import RoomVersions
from synapse.rest import admin
from synapse.rest.client import login, room, sync
from synapse.server import HomeServer
+from synapse.storage._base import db_to_json
+from synapse.storage.database import LoggingTransaction
from synapse.types import JsonDict
from synapse.util import Clock
@@ -573,7 +575,7 @@ class RedactionsTestCase(HomeserverTestCase):
room_id = self.helper.create_room_as(
self.mod_user_id,
tok=self.mod_access_token,
- room_version=RoomVersions.MSC2176.identifier,
+ room_version=RoomVersions.V11.identifier,
)
# Create an event.
@@ -597,5 +599,20 @@ class RedactionsTestCase(HomeserverTestCase):
redact_event = timeline[-1]
self.assertEqual(redact_event["type"], EventTypes.Redaction)
# The redacts key should be in the content.
- self.assertNotIn("redacts", redact_event)
self.assertEquals(redact_event["content"]["redacts"], event_id)
+
+ # It should also be copied as the top-level redacts field for backwards
+ # compatibility.
+ self.assertEquals(redact_event["redacts"], event_id)
+
+ # But it isn't actually part of the event.
+ def get_event(txn: LoggingTransaction) -> JsonDict:
+ return db_to_json(
+ main_datastore._fetch_event_rows(txn, [event_id])[event_id].json
+ )
+
+ main_datastore = self.hs.get_datastores().main
+ event_json = self.get_success(
+ main_datastore.db_pool.runInteraction("get_event", get_event)
+ )
+ self.assertNotIn("redacts", event_json)
diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py
index f1b4e1ad2f..d013e75d55 100644
--- a/tests/rest/client/test_rooms.py
+++ b/tests/rest/client/test_rooms.py
@@ -67,8 +67,6 @@ class RoomBase(unittest.HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
self.hs = self.setup_test_homeserver(
"red",
- federation_http_client=None,
- federation_client=Mock(),
)
self.hs.get_federation_handler = Mock() # type: ignore[assignment]
diff --git a/tests/server.py b/tests/server.py
index a12c3e3b9a..c84a524e8c 100644
--- a/tests/server.py
+++ b/tests/server.py
@@ -53,6 +53,7 @@ from twisted.internet.interfaces import (
IConnector,
IConsumer,
IHostnameResolver,
+ IListeningPort,
IProducer,
IProtocol,
IPullProducer,
@@ -62,7 +63,7 @@ from twisted.internet.interfaces import (
IResolverSimple,
ITransport,
)
-from twisted.internet.protocol import ClientFactory, DatagramProtocol
+from twisted.internet.protocol import ClientFactory, DatagramProtocol, Factory
from twisted.python import threadpool
from twisted.python.failure import Failure
from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock
@@ -523,6 +524,35 @@ class ThreadedMemoryReactorClock(MemoryReactorClock):
"""
self._tcp_callbacks[(host, port)] = callback
+ def connectUNIX(
+ self,
+ address: str,
+ factory: ClientFactory,
+ timeout: float = 30,
+ checkPID: int = 0,
+ ) -> IConnector:
+ """
+ Unix sockets aren't supported for unit tests yet. Make it obvious to any
+ developer trying it out that they will need to do some work before being able
+ to use it in tests.
+ """
+ raise Exception("Unix sockets are not implemented for tests yet, sorry.")
+
+ def listenUNIX(
+ self,
+ address: str,
+ factory: Factory,
+ backlog: int = 50,
+ mode: int = 0o666,
+ wantPID: int = 0,
+ ) -> IListeningPort:
+ """
+ Unix sockets aren't supported for unit tests yet. Make it obvious to any
+ developer trying it out that they will need to do some work before being able
+ to use it in tests.
+ """
+ raise Exception("Unix sockets are not implemented for tests, sorry")
+
def connectTCP(
self,
host: str,
diff --git a/tests/storage/test_e2e_room_keys.py b/tests/storage/test_e2e_room_keys.py
index 9cb326d90a..f6df31aba4 100644
--- a/tests/storage/test_e2e_room_keys.py
+++ b/tests/storage/test_e2e_room_keys.py
@@ -31,7 +31,7 @@ room_key: RoomKey = {
class E2eRoomKeysHandlerTestCase(unittest.HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver("server", federation_http_client=None)
+ hs = self.setup_test_homeserver("server")
self.store = hs.get_datastores().main
return hs
diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py
index 857e2caf2e..0282673167 100644
--- a/tests/storage/test_purge.py
+++ b/tests/storage/test_purge.py
@@ -27,7 +27,7 @@ class PurgeTests(HomeserverTestCase):
servlets = [room.register_servlets]
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver("server", federation_http_client=None)
+ hs = self.setup_test_homeserver("server")
return hs
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
diff --git a/tests/storage/test_rollback_worker.py b/tests/storage/test_rollback_worker.py
index 6861d3a6c9..809c9f175d 100644
--- a/tests/storage/test_rollback_worker.py
+++ b/tests/storage/test_rollback_worker.py
@@ -45,9 +45,7 @@ def fake_listdir(filepath: str) -> List[str]:
class WorkerSchemaTests(HomeserverTestCase):
def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
- hs = self.setup_test_homeserver(
- federation_http_client=None, homeserver_to_use=GenericWorkerServer
- )
+ hs = self.setup_test_homeserver(homeserver_to_use=GenericWorkerServer)
return hs
def default_config(self) -> JsonDict:
diff --git a/tests/test_server.py b/tests/test_server.py
index dc491e06ed..36162cd1f5 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -38,7 +38,7 @@ from tests.http.server._base import test_disconnect
from tests.server import (
FakeChannel,
FakeSite,
- ThreadedMemoryReactorClock,
+ get_clock,
make_request,
setup_test_homeserver,
)
@@ -46,12 +46,11 @@ from tests.server import (
class JsonResourceTests(unittest.TestCase):
def setUp(self) -> None:
- self.reactor = ThreadedMemoryReactorClock()
- self.hs_clock = Clock(self.reactor)
+ reactor, clock = get_clock()
+ self.reactor = reactor
self.homeserver = setup_test_homeserver(
self.addCleanup,
- federation_http_client=None,
- clock=self.hs_clock,
+ clock=clock,
reactor=self.reactor,
)
@@ -209,7 +208,13 @@ class JsonResourceTests(unittest.TestCase):
class OptionsResourceTests(unittest.TestCase):
def setUp(self) -> None:
- self.reactor = ThreadedMemoryReactorClock()
+ reactor, clock = get_clock()
+ self.reactor = reactor
+ self.homeserver = setup_test_homeserver(
+ self.addCleanup,
+ clock=clock,
+ reactor=self.reactor,
+ )
class DummyResource(Resource):
isLeaf = True
@@ -242,6 +247,7 @@ class OptionsResourceTests(unittest.TestCase):
"1.0",
max_request_body_size=4096,
reactor=self.reactor,
+ hs=self.homeserver,
)
# render the request and return the channel
@@ -344,7 +350,8 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase):
await self.callback(request)
def setUp(self) -> None:
- self.reactor = ThreadedMemoryReactorClock()
+ reactor, _ = get_clock()
+ self.reactor = reactor
def test_good_response(self) -> None:
async def callback(request: SynapseRequest) -> None:
@@ -462,9 +469,9 @@ class DirectServeJsonResourceCancellationTests(unittest.TestCase):
"""Tests for `DirectServeJsonResource` cancellation."""
def setUp(self) -> None:
- self.reactor = ThreadedMemoryReactorClock()
- self.clock = Clock(self.reactor)
- self.resource = CancellableDirectServeJsonResource(self.clock)
+ reactor, clock = get_clock()
+ self.reactor = reactor
+ self.resource = CancellableDirectServeJsonResource(clock)
self.site = FakeSite(self.resource, self.reactor)
def test_cancellable_disconnect(self) -> None:
@@ -496,9 +503,9 @@ class DirectServeHtmlResourceCancellationTests(unittest.TestCase):
"""Tests for `DirectServeHtmlResource` cancellation."""
def setUp(self) -> None:
- self.reactor = ThreadedMemoryReactorClock()
- self.clock = Clock(self.reactor)
- self.resource = CancellableDirectServeHtmlResource(self.clock)
+ reactor, clock = get_clock()
+ self.reactor = reactor
+ self.resource = CancellableDirectServeHtmlResource(clock)
self.site = FakeSite(self.resource, self.reactor)
def test_cancellable_disconnect(self) -> None:
diff --git a/tests/unittest.py b/tests/unittest.py
index c73195b32b..b0721e060c 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -358,6 +358,7 @@ class HomeserverTestCase(TestCase):
server_version_string="1",
max_request_body_size=4096,
reactor=self.reactor,
+ hs=self.hs,
)
from tests.rest.client.utils import RestHelper
|
