diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index b32d7566a5..11ea8ef10c 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -17,18 +17,21 @@ import logging
from mock import Mock
import treq
+from zope.interface import implementer
from twisted.internet import defer
+from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions
from twisted.internet.protocol import Factory
from twisted.protocols.tls import TLSMemoryBIOFactory
-from twisted.test.ssl_helpers import ServerTLSContext
from twisted.web.http import HTTPChannel
+from twisted.web.iweb import IPolicyForHTTPS
from synapse.crypto.context_factory import ClientTLSOptionsFactory
from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent
from synapse.http.federation.srv_resolver import Server
from synapse.util.logcontext import LoggingContext
+from tests.http import ServerTLSContext
from tests.server import FakeTransport, ThreadedMemoryReactorClock
from tests.unittest import TestCase
@@ -44,6 +47,7 @@ class MatrixFederationAgentTests(TestCase):
self.agent = MatrixFederationAgent(
reactor=self.reactor,
tls_client_options_factory=ClientTLSOptionsFactory(None),
+ _well_known_tls_policy=TrustingTLSPolicyForHTTPS(),
_srv_resolver=self.mock_resolver,
)
@@ -65,10 +69,14 @@ class MatrixFederationAgentTests(TestCase):
# Normally this would be done by the TCP socket code in Twisted, but we are
# stubbing that out here.
client_protocol = client_factory.buildProtocol(None)
- client_protocol.makeConnection(FakeTransport(server_tls_protocol, self.reactor))
+ client_protocol.makeConnection(
+ FakeTransport(server_tls_protocol, self.reactor, client_protocol),
+ )
# tell the server tls protocol to send its stuff back to the client, too
- server_tls_protocol.makeConnection(FakeTransport(client_protocol, self.reactor))
+ server_tls_protocol.makeConnection(
+ FakeTransport(client_protocol, self.reactor, server_tls_protocol),
+ )
# give the reactor a pump to get the TLS juices flowing.
self.reactor.pump((0.1,))
@@ -101,9 +109,48 @@ class MatrixFederationAgentTests(TestCase):
try:
fetch_res = yield fetch_d
defer.returnValue(fetch_res)
+ except Exception as e:
+ logger.info("Fetch of %s failed: %s", uri.decode("ascii"), e)
+ raise
finally:
_check_logcontext(context)
+ def _handle_well_known_connection(self, client_factory, expected_sni, target_server):
+ """Handle an outgoing HTTPs connection: wire it up to a server, check that the
+ request is for a .well-known, and send the response.
+
+ Args:
+ client_factory (IProtocolFactory): outgoing connection
+ expected_sni (bytes): SNI that we expect the outgoing connection to send
+ target_server (bytes): target server that we should redirect to in the
+ .well-known response.
+ """
+ # make the connection for .well-known
+ well_known_server = self._make_connection(
+ client_factory,
+ expected_sni=expected_sni,
+ )
+ # check the .well-known request and send a response
+ self.assertEqual(len(well_known_server.requests), 1)
+ request = well_known_server.requests[0]
+ self._send_well_known_response(request, target_server)
+
+ def _send_well_known_response(self, request, target_server):
+ """Check that an incoming request looks like a valid .well-known request, and
+ send back the response.
+ """
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/.well-known/matrix/server')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'testserv'],
+ )
+ # send back a response
+ request.write(b'{ "m.server": "%s" }' % (target_server,))
+ request.finish()
+
+ self.reactor.pump((0.1, ))
+
def test_get(self):
"""
happy-path test of a GET request with an explicit port
@@ -131,6 +178,10 @@ class MatrixFederationAgentTests(TestCase):
request = http_server.requests[0]
self.assertEqual(request.method, b'GET')
self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'testserv:8448']
+ )
content = request.content.read()
self.assertEqual(content, b'')
@@ -162,11 +213,7 @@ class MatrixFederationAgentTests(TestCase):
"""
Test the behaviour when the server name contains an explicit IP (with no port)
"""
-
- # the SRV lookup will return an empty list (XXX: why do we even do an SRV lookup?)
- self.mock_resolver.resolve_service.side_effect = lambda _: []
-
- # then there will be a getaddrinfo on the IP
+ # there will be a getaddrinfo on the IP
self.reactor.lookups["1.2.3.4"] = "1.2.3.4"
test_d = self._make_get_request(b"matrix://1.2.3.4/foo/bar")
@@ -174,15 +221,52 @@ class MatrixFederationAgentTests(TestCase):
# Nothing happened yet
self.assertNoResult(test_d)
- self.mock_resolver.resolve_service.assert_called_once_with(
- b"_matrix._tcp.1.2.3.4",
+ # Make sure treq is trying to connect
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 8448)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=None,
)
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'1.2.3.4'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
+ def test_get_ipv6_address(self):
+ """
+ Test the behaviour when the server name contains an explicit IPv6 address
+ (with no port)
+ """
+
+ # there will be a getaddrinfo on the IP
+ self.reactor.lookups["::1"] = "::1"
+
+ test_d = self._make_get_request(b"matrix://[::1]/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
# Make sure treq is trying to connect
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 1)
(host, port, client_factory, _timeout, _bindAddress) = clients[0]
- self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(host, '::1')
self.assertEqual(port, 8448)
# make a test server, and wire up the client
@@ -195,15 +279,60 @@ class MatrixFederationAgentTests(TestCase):
request = http_server.requests[0]
self.assertEqual(request.method, b'GET')
self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'[::1]'],
+ )
# finish the request
request.finish()
self.reactor.pump((0.1,))
self.successResultOf(test_d)
- def test_get_hostname_no_srv(self):
+ def test_get_ipv6_address_with_port(self):
"""
- Test the behaviour when the server name has no port, and no SRV record
+ Test the behaviour when the server name contains an explicit IPv6 address
+ (with explicit port)
+ """
+
+ # there will be a getaddrinfo on the IP
+ self.reactor.lookups["::1"] = "::1"
+
+ test_d = self._make_get_request(b"matrix://[::1]:80/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ # Make sure treq is trying to connect
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '::1')
+ self.assertEqual(port, 80)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=None,
+ )
+
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'[::1]:80'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
+ def test_get_no_srv_no_well_known(self):
+ """
+ Test the behaviour when the server name has no port, no SRV, and no well-known
"""
self.mock_resolver.resolve_service.side_effect = lambda _: []
@@ -218,11 +347,24 @@ class MatrixFederationAgentTests(TestCase):
b"_matrix._tcp.testserv",
)
- # Make sure treq is trying to connect
+ # there should be an attempt to connect on port 443 for the .well-known
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 1)
(host, port, client_factory, _timeout, _bindAddress) = clients[0]
self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ # fonx the connection
+ client_factory.clientConnectionFailed(None, Exception("nope"))
+
+ # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
+ # .well-known request fails.
+ self.reactor.pump((0.4,))
+
+ # we should fall back to a direct connection
+ self.assertEqual(len(clients), 2)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[1]
+ self.assertEqual(host, '1.2.3.4')
self.assertEqual(port, 8448)
# make a test server, and wire up the client
@@ -235,6 +377,71 @@ class MatrixFederationAgentTests(TestCase):
request = http_server.requests[0]
self.assertEqual(request.method, b'GET')
self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'testserv'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
+ def test_get_well_known(self):
+ """Test the behaviour when the server name has no port and no SRV record, but
+ the .well-known redirects elsewhere
+ """
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: []
+ self.reactor.lookups["testserv"] = "1.2.3.4"
+ self.reactor.lookups["target-server"] = "1::f"
+
+ test_d = self._make_get_request(b"matrix://testserv/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.testserv",
+ )
+ self.mock_resolver.resolve_service.reset_mock()
+
+ # there should be an attempt to connect on port 443 for the .well-known
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ self._handle_well_known_connection(
+ client_factory, expected_sni=b"testserv", target_server=b"target-server",
+ )
+
+ # there should be another SRV lookup
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.target-server",
+ )
+
+ # now we should get a connection to the target server
+ self.assertEqual(len(clients), 2)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[1]
+ self.assertEqual(host, '1::f')
+ self.assertEqual(port, 8448)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=b'target-server',
+ )
+
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'target-server'],
+ )
# finish the request
request.finish()
@@ -246,7 +453,7 @@ class MatrixFederationAgentTests(TestCase):
Test the behaviour when there is a single SRV record
"""
self.mock_resolver.resolve_service.side_effect = lambda _: [
- Server(host="srvtarget", port=8443)
+ Server(host=b"srvtarget", port=8443)
]
self.reactor.lookups["srvtarget"] = "1.2.3.4"
@@ -276,6 +483,178 @@ class MatrixFederationAgentTests(TestCase):
request = http_server.requests[0]
self.assertEqual(request.method, b'GET')
self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'testserv'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
+ def test_get_well_known_srv(self):
+ """Test the behaviour when the server name has no port and no SRV record, but
+ the .well-known redirects to a place where there is a SRV.
+ """
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: []
+ self.reactor.lookups["testserv"] = "1.2.3.4"
+ self.reactor.lookups["srvtarget"] = "5.6.7.8"
+
+ test_d = self._make_get_request(b"matrix://testserv/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.testserv",
+ )
+ self.mock_resolver.resolve_service.reset_mock()
+
+ # there should be an attempt to connect on port 443 for the .well-known
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: [
+ Server(host=b"srvtarget", port=8443),
+ ]
+
+ self._handle_well_known_connection(
+ client_factory, expected_sni=b"testserv", target_server=b"target-server",
+ )
+
+ # there should be another SRV lookup
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.target-server",
+ )
+
+ # now we should get a connection to the target of the SRV record
+ self.assertEqual(len(clients), 2)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[1]
+ self.assertEqual(host, '5.6.7.8')
+ self.assertEqual(port, 8443)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=b'target-server',
+ )
+
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'target-server'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
+ def test_idna_servername(self):
+ """test the behaviour when the server name has idna chars in"""
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: []
+
+ # the resolver is always called with the IDNA hostname as a native string.
+ self.reactor.lookups["xn--bcher-kva.com"] = "1.2.3.4"
+
+ # this is idna for bücher.com
+ test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.xn--bcher-kva.com",
+ )
+
+ # there should be an attempt to connect on port 443 for the .well-known
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ # fonx the connection
+ client_factory.clientConnectionFailed(None, Exception("nope"))
+
+ # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
+ # .well-known request fails.
+ self.reactor.pump((0.4,))
+
+ # We should fall back to port 8448
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 2)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[1]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 8448)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=b'xn--bcher-kva.com',
+ )
+
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'xn--bcher-kva.com'],
+ )
+
+ # finish the request
+ request.finish()
+ self.reactor.pump((0.1,))
+ self.successResultOf(test_d)
+
+ def test_idna_srv_target(self):
+ """test the behaviour when the target of a SRV record has idna chars"""
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: [
+ Server(host=b"xn--trget-3qa.com", port=8443) # târget.com
+ ]
+ self.reactor.lookups["xn--trget-3qa.com"] = "1.2.3.4"
+
+ test_d = self._make_get_request(b"matrix://xn--bcher-kva.com/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.xn--bcher-kva.com",
+ )
+
+ # Make sure treq is trying to connect
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 8443)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(
+ client_factory,
+ expected_sni=b'xn--bcher-kva.com',
+ )
+
+ self.assertEqual(len(http_server.requests), 1)
+ request = http_server.requests[0]
+ self.assertEqual(request.method, b'GET')
+ self.assertEqual(request.path, b'/foo/bar')
+ self.assertEqual(
+ request.requestHeaders.getRawHeaders(b'host'),
+ [b'xn--bcher-kva.com'],
+ )
# finish the request
request.finish()
@@ -313,3 +692,11 @@ def _build_test_server():
def _log_request(request):
"""Implements Factory.log, which is expected by Request.finish"""
logger.info("Completed request %s", request)
+
+
+@implementer(IPolicyForHTTPS)
+class TrustingTLSPolicyForHTTPS(object):
+ """An IPolicyForHTTPS which doesn't do any certificate verification"""
+ def creatorForNetloc(self, hostname, port):
+ certificateOptions = OpenSSLCertificateOptions()
+ return ClientTLSOptions(hostname, certificateOptions.getContext())
|
