diff --git a/CHANGES.md b/CHANGES.md
index 4dea0f6319..523cdb1153 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,14 @@
+Synapse 1.0.0rc2 (2019-06-10)
+=============================
+
+Bugfixes
+--------
+
+- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392))
+- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415))
+- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417))
+
+
Synapse 1.0.0rc1 (2019-06-07)
=============================
diff --git a/MANIFEST.in b/MANIFEST.in
index ad1523e387..2c59c7bdc2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -18,8 +18,10 @@ recursive-include docs *
recursive-include scripts *
recursive-include scripts-dev *
recursive-include synapse *.pyi
-recursive-include tests *.pem
recursive-include tests *.py
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
recursive-include synapse/res *
recursive-include synapse/static *.css
diff --git a/changelog.d/5392.bugfix b/changelog.d/5392.bugfix
deleted file mode 100644
index 295a7cfce1..0000000000
--- a/changelog.d/5392.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Remove redundant warning about key server response validation.
diff --git a/changelog.d/5415.bugfix b/changelog.d/5415.bugfix
deleted file mode 100644
index 83629e193d..0000000000
--- a/changelog.d/5415.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 77a4cfc3a5..8dc07fe73c 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@ try:
except ImportError:
pass
-__version__ = "1.0.0rc1"
+__version__ = "1.0.0rc2"
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 59ea087e66..2bc5cc3807 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -15,10 +15,13 @@
import logging
+import idna
+from service_identity import VerificationError
+from service_identity.pyopenssl import verify_hostname, verify_ip_address
from zope.interface import implementer
from OpenSSL import SSL, crypto
-from twisted.internet._sslverify import ClientTLSOptions, _defaultCurveName
+from twisted.internet._sslverify import _defaultCurveName
from twisted.internet.abstract import isIPAddress, isIPv6Address
from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
from twisted.internet.ssl import CertificateOptions, ContextFactory, platformTrust
@@ -56,79 +59,19 @@ class ServerContextFactory(ContextFactory):
return self._context
-def _idnaBytes(text):
- """
- Convert some text typed by a human into some ASCII bytes. This is a
- copy of twisted.internet._idna._idnaBytes. For documentation, see the
- twisted documentation.
- """
- try:
- import idna
- except ImportError:
- return text.encode("idna")
- else:
- return idna.encode(text)
-
-
-def _tolerateErrors(wrapped):
- """
- Wrap up an info_callback for pyOpenSSL so that if something goes wrong
- the error is immediately logged and the connection is dropped if possible.
- This is a copy of twisted.internet._sslverify._tolerateErrors. For
- documentation, see the twisted documentation.
- """
-
- def infoCallback(connection, where, ret):
- try:
- return wrapped(connection, where, ret)
- except: # noqa: E722, taken from the twisted implementation
- f = Failure()
- logger.exception("Error during info_callback")
- connection.get_app_data().failVerification(f)
-
- return infoCallback
+class ClientTLSOptionsFactory(object):
+ """Factory for Twisted SSLClientConnectionCreators that are used to make connections
+ to remote servers for federation.
+ Uses one of two OpenSSL context objects for all connections, depending on whether
+ we should do SSL certificate verification.
-@implementer(IOpenSSLClientConnectionCreator)
-class ClientTLSOptionsNoVerify(object):
- """
- Client creator for TLS without certificate identity verification. This is a
- copy of twisted.internet._sslverify.ClientTLSOptions with the identity
- verification left out. For documentation, see the twisted documentation.
+ get_options decides whether we should do SSL certificate verification and
+ constructs an SSLClientConnectionCreator factory accordingly.
"""
- def __init__(self, hostname, ctx):
- self._ctx = ctx
-
- if isIPAddress(hostname) or isIPv6Address(hostname):
- self._hostnameBytes = hostname.encode('ascii')
- self._sendSNI = False
- else:
- self._hostnameBytes = _idnaBytes(hostname)
- self._sendSNI = True
-
- ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
-
- def clientConnectionForTLS(self, tlsProtocol):
- context = self._ctx
- connection = SSL.Connection(context, None)
- connection.set_app_data(tlsProtocol)
- return connection
-
- def _identityVerifyingInfoCallback(self, connection, where, ret):
- # Literal IPv4 and IPv6 addresses are not permitted
- # as host names according to the RFCs
- if where & SSL.SSL_CB_HANDSHAKE_START and self._sendSNI:
- connection.set_tlsext_host_name(self._hostnameBytes)
-
-
-class ClientTLSOptionsFactory(object):
- """Factory for Twisted ClientTLSOptions that are used to make connections
- to remote servers for federation."""
-
def __init__(self, config):
self._config = config
- self._options_noverify = CertificateOptions()
# Check if we're using a custom list of a CA certificates
trust_root = config.federation_ca_trust_root
@@ -136,11 +79,13 @@ class ClientTLSOptionsFactory(object):
# Use CA root certs provided by OpenSSL
trust_root = platformTrust()
- self._options_verify = CertificateOptions(trustRoot=trust_root)
+ self._verify_ssl_context = CertificateOptions(trustRoot=trust_root).getContext()
+ self._verify_ssl_context.set_info_callback(self._context_info_cb)
- def get_options(self, host):
- # Use _makeContext so that we get a fresh OpenSSL CTX each time.
+ self._no_verify_ssl_context = CertificateOptions().getContext()
+ self._no_verify_ssl_context.set_info_callback(self._context_info_cb)
+ def get_options(self, host):
# Check if certificate verification has been enabled
should_verify = self._config.federation_verify_certificates
@@ -151,6 +96,93 @@ class ClientTLSOptionsFactory(object):
should_verify = False
break
- if should_verify:
- return ClientTLSOptions(host, self._options_verify._makeContext())
- return ClientTLSOptionsNoVerify(host, self._options_noverify._makeContext())
+ ssl_context = (
+ self._verify_ssl_context if should_verify else self._no_verify_ssl_context
+ )
+
+ return SSLClientConnectionCreator(host, ssl_context, should_verify)
+
+ @staticmethod
+ def _context_info_cb(ssl_connection, where, ret):
+ """The 'information callback' for our openssl context object."""
+ # we assume that the app_data on the connection object has been set to
+ # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator)
+ tls_protocol = ssl_connection.get_app_data()
+ try:
+ # ... we further assume that SSLClientConnectionCreator has set the
+ # '_synapse_tls_verifier' attribute to a ConnectionVerifier object.
+ tls_protocol._synapse_tls_verifier.verify_context_info_cb(
+ ssl_connection, where
+ )
+ except: # noqa: E722, taken from the twisted implementation
+ logger.exception("Error during info_callback")
+ f = Failure()
+ tls_protocol.failVerification(f)
+
+
+@implementer(IOpenSSLClientConnectionCreator)
+class SSLClientConnectionCreator(object):
+ """Creates openssl connection objects for client connections.
+
+ Replaces twisted.internet.ssl.ClientTLSOptions
+ """
+
+ def __init__(self, hostname, ctx, verify_certs):
+ self._ctx = ctx
+ self._verifier = ConnectionVerifier(hostname, verify_certs)
+
+ def clientConnectionForTLS(self, tls_protocol):
+ context = self._ctx
+ connection = SSL.Connection(context, None)
+
+ # as per twisted.internet.ssl.ClientTLSOptions, we set the application
+ # data to our TLSMemoryBIOProtocol...
+ connection.set_app_data(tls_protocol)
+
+ # ... and we also gut-wrench a '_synapse_tls_verifier' attribute into the
+ # tls_protocol so that the SSL context's info callback has something to
+ # call to do the cert verification.
+ setattr(tls_protocol, "_synapse_tls_verifier", self._verifier)
+ return connection
+
+
+class ConnectionVerifier(object):
+ """Set the SNI, and do cert verification
+
+ This is a thing which is attached to the TLSMemoryBIOProtocol, and is called by
+ the ssl context's info callback.
+ """
+
+ # This code is based on twisted.internet.ssl.ClientTLSOptions.
+
+ def __init__(self, hostname, verify_certs):
+ self._verify_certs = verify_certs
+
+ if isIPAddress(hostname) or isIPv6Address(hostname):
+ self._hostnameBytes = hostname.encode("ascii")
+ self._is_ip_address = True
+ else:
+ # twisted's ClientTLSOptions falls back to the stdlib impl here if
+ # idna is not installed, but points out that lacks support for
+ # IDNA2008 (http://bugs.python.org/issue17305).
+ #
+ # We can rely on having idna.
+ self._hostnameBytes = idna.encode(hostname)
+ self._is_ip_address = False
+
+ self._hostnameASCII = self._hostnameBytes.decode("ascii")
+
+ def verify_context_info_cb(self, ssl_connection, where):
+ if where & SSL.SSL_CB_HANDSHAKE_START and not self._is_ip_address:
+ ssl_connection.set_tlsext_host_name(self._hostnameBytes)
+
+ if where & SSL.SSL_CB_HANDSHAKE_DONE and self._verify_certs:
+ try:
+ if self._is_ip_address:
+ verify_ip_address(ssl_connection, self._hostnameASCII)
+ else:
+ verify_hostname(ssl_connection, self._hostnameASCII)
+ except VerificationError:
+ f = Failure()
+ tls_protocol = ssl_connection.get_app_data()
+ tls_protocol.failVerification(f)
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index c78f2cb15e..6efd81f204 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -44,7 +44,10 @@ REQUIREMENTS = [
"canonicaljson>=1.1.3",
"signedjson>=1.0.0",
"pynacl>=1.2.1",
- "service_identity>=16.0.0",
+ "idna>=2",
+
+ # validating SSL certs for IP addresses requires service_identity 18.1.
+ "service_identity>=18.1.0",
# our logcontext handling relies on the ability to cancel inlineCallbacks
# (https://twistedmatrix.com/trac/ticket/4632) which landed in Twisted 18.7.
diff --git a/tests/http/__init__.py b/tests/http/__init__.py
index 851fc0eb33..2d5dba6464 100644
--- a/tests/http/__init__.py
+++ b/tests/http/__init__.py
@@ -13,28 +13,122 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path
+import subprocess
+
+from zope.interface import implementer
from OpenSSL import SSL
+from OpenSSL.SSL import Connection
+from twisted.internet.interfaces import IOpenSSLServerConnectionCreator
+
+
+def get_test_ca_cert_file():
+ """Get the path to the test CA cert
+
+ The keypair is generated with:
+
+ openssl genrsa -out ca.key 2048
+ openssl req -new -x509 -key ca.key -days 3650 -out ca.crt \
+ -subj '/CN=synapse test CA'
+ """
+ return os.path.join(os.path.dirname(__file__), "ca.crt")
+
+
+def get_test_key_file():
+ """get the path to the test key
+
+ The key file is made with:
+
+ openssl genrsa -out server.key 2048
+ """
+ return os.path.join(os.path.dirname(__file__), "server.key")
+
+
+cert_file_count = 0
+
+CONFIG_TEMPLATE = b"""\
+[default]
+basicConstraints = CA:FALSE
+keyUsage=nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = %(sanentries)s
+"""
+
+
+def create_test_cert_file(sanlist):
+ """build an x509 certificate file
+
+ Args:
+ sanlist: list[bytes]: a list of subjectAltName values for the cert
+
+ Returns:
+ str: the path to the file
+ """
+ global cert_file_count
+ csr_filename = "server.csr"
+ cnf_filename = "server.%i.cnf" % (cert_file_count,)
+ cert_filename = "server.%i.crt" % (cert_file_count,)
+ cert_file_count += 1
+
+ # first build a CSR
+ subprocess.check_call(
+ [
+ "openssl",
+ "req",
+ "-new",
+ "-key",
+ get_test_key_file(),
+ "-subj",
+ "/",
+ "-out",
+ csr_filename,
+ ]
+ )
+ # now a config file describing the right SAN entries
+ sanentries = b",".join(sanlist)
+ with open(cnf_filename, "wb") as f:
+ f.write(CONFIG_TEMPLATE % {b"sanentries": sanentries})
-def get_test_cert_file():
- """get the path to the test cert"""
+ # finally the cert
+ ca_key_filename = os.path.join(os.path.dirname(__file__), "ca.key")
+ ca_cert_filename = get_test_ca_cert_file()
+ subprocess.check_call(
+ [
+ "openssl",
+ "x509",
+ "-req",
+ "-in",
+ csr_filename,
+ "-CA",
+ ca_cert_filename,
+ "-CAkey",
+ ca_key_filename,
+ "-set_serial",
+ "1",
+ "-extfile",
+ cnf_filename,
+ "-out",
+ cert_filename,
+ ]
+ )
- # the cert file itself is made with:
- #
- # openssl req -x509 -newkey rsa:4096 -keyout server.pem -out server.pem -days 36500 \
- # -nodes -subj '/CN=testserv'
- return os.path.join(os.path.dirname(__file__), 'server.pem')
+ return cert_filename
-class ServerTLSContext(object):
- """A TLS Context which presents our test cert."""
+@implementer(IOpenSSLServerConnectionCreator)
+class TestServerTLSConnectionFactory(object):
+ """An SSL connection creator which returns connections which present a certificate
+ signed by our test CA."""
- def __init__(self):
- self.filename = get_test_cert_file()
+ def __init__(self, sanlist):
+ """
+ Args:
+ sanlist: list[bytes]: a list of subjectAltName values for the cert
+ """
+ self._cert_file = create_test_cert_file(sanlist)
- def getContext(self):
+ def serverConnectionForTLS(self, tlsProtocol):
ctx = SSL.Context(SSL.TLSv1_METHOD)
- ctx.use_certificate_file(self.filename)
- ctx.use_privatekey_file(self.filename)
- return ctx
+ ctx.use_certificate_file(self._cert_file)
+ ctx.use_privatekey_file(get_test_key_file())
+ return Connection(ctx, None)
diff --git a/tests/http/ca.crt b/tests/http/ca.crt
new file mode 100644
index 0000000000..730f81e99c
--- /dev/null
+++ b/tests/http/ca.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDCjCCAfKgAwIBAgIJAPwHIHgH/jtjMA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNV
+BAMMD3N5bmFwc2UgdGVzdCBDQTAeFw0xOTA2MTAxMTI2NDdaFw0yOTA2MDcxMTI2
+NDdaMBoxGDAWBgNVBAMMD3N5bmFwc2UgdGVzdCBDQTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAOZOXCKuylf9jHzJXpU2nS+XEKrnGPgs2SAhQKrzBxg3
+/d8KT2Zsfsj1i3G7oGu7B0ZKO6qG5AxOPCmSMf9/aiSHFilfSh+r8rCpJyWMev2c
+/w/xmhoFHgn+H90NnqlXvWb5y1YZCE3gWaituQSaa93GPKacRqXCgIrzjPUuhfeT
+uwFQt4iyUhMNBYEy3aw4IuIHdyBqi4noUhR2ZeuflLJ6PswdJ8mEiAvxCbBGPerq
+idhWcZwlo0fKu4u1uu5B8TnTsMg2fJgL6c5olBG90Urt22gA6anfP5W/U1ZdVhmB
+T3Rv5SJMkGyMGE6sEUetLFyb2GJpgGD7ePkUCZr+IMMCAwEAAaNTMFEwHQYDVR0O
+BBYEFLg7nTCYsvQXWTyS6upLc0YTlIwRMB8GA1UdIwQYMBaAFLg7nTCYsvQXWTyS
+6upLc0YTlIwRMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADqx
+GX4Ul5OGQlcG+xTt4u3vMCeqGo8mh1AnJ7zQbyRmwjJiNxJVX+/EcqFSTsmkBNoe
+xdYITI7Z6dyoiKw99yCZDE7gALcyACEU7r0XY7VY/hebAaX6uLaw1sZKKAIC04lD
+KgCu82tG85n60Qyud5SiZZF0q1XVq7lbvOYVdzVZ7k8Vssy5p9XnaLJLMggYeOiX
+psHIQjvYGnTTEBZZHzWOrc0WGThd69wxTOOkAbCsoTPEwZL8BGUsdtLWtvhp452O
+npvaUBzKg39R5X3KTdhB68XptiQfzbQkd3FtrwNuYPUywlsg55Bxkv85n57+xDO3
+D9YkgUqEp0RGUXQgCsQ=
+-----END CERTIFICATE-----
diff --git a/tests/http/ca.key b/tests/http/ca.key
new file mode 100644
index 0000000000..5c99cae186
--- /dev/null
+++ b/tests/http/ca.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpgIBAAKCAQEA5k5cIq7KV/2MfMlelTadL5cQqucY+CzZICFAqvMHGDf93wpP
+Zmx+yPWLcbuga7sHRko7qobkDE48KZIx/39qJIcWKV9KH6vysKknJYx6/Zz/D/Ga
+GgUeCf4f3Q2eqVe9ZvnLVhkITeBZqK25BJpr3cY8ppxGpcKAivOM9S6F95O7AVC3
+iLJSEw0FgTLdrDgi4gd3IGqLiehSFHZl65+Usno+zB0nyYSIC/EJsEY96uqJ2FZx
+nCWjR8q7i7W67kHxOdOwyDZ8mAvpzmiUEb3RSu3baADpqd8/lb9TVl1WGYFPdG/l
+IkyQbIwYTqwRR60sXJvYYmmAYPt4+RQJmv4gwwIDAQABAoIBAQCFuFG+wYYy+MCt
+Y65LLN6vVyMSWAQjdMbM5QHLQDiKU1hQPIhFjBFBVXCVpL9MTde3dDqYlKGsk3BT
+ItNs6eoTM2wmsXE0Wn4bHNvh7WMsBhACjeFP4lDCtI6DpvjMkmkidT8eyoIL1Yu5
+aMTYa2Dd79AfXPWYIQrJowfhBBY83KuW5fmYnKKDVLqkT9nf2dgmmQz85RgtNiZC
+zFkIsNmPqH1zRbcw0wORfOBrLFvsMc4Tt8EY5Wz3NnH8Zfgf8Q3MgARH1yspz3Vp
+B+EYHbsK17xZ+P59KPiX3yefvyYWEUjFF7ymVsVnDxLugYl4pXwWUpm19GxeDvFk
+cgBUD5OBAoGBAP7lBdCp6lx6fYtxdxUm3n4MMQmYcac4qZdeBIrvpFMnvOBBuixl
+eavcfFmFdwgAr8HyVYiu9ynac504IYvmtYlcpUmiRBbmMHbvLQEYHl7FYFKNz9ej
+2ue4oJE3RsPdLsD3xIlc+xN8oT1j0knyorwsHdj0Sv77eZzZS9XZZfJzAoGBAOdO
+CibYmoNqK/mqDHkp6PgsnbQGD5/CvPF/BLUWV1QpHxLzUQQeoBOQW5FatHe1H5zi
+mbq3emBefVmsCLrRIJ4GQu4vsTMfjcpGLwviWmaK6pHbGPt8IYeEQ2MNyv59EtA2
+pQy4dX7/Oe6NLAR1UEQjXmCuXf+rxnxF3VJd1nRxAoGBANb9eusl9fusgSnVOTjJ
+AQ7V36KVRv9hZoG6liBNwo80zDVmms4JhRd1MBkd3mkMkzIF4SkZUnWlwLBSANGM
+dX/3eZ5i1AVwgF5Am/f5TNxopDbdT/o1RVT/P8dcFT7s1xuBn+6wU0F7dFBgWqVu
+lt4aY85zNrJcj5XBHhqwdDGLAoGBAIksPNUAy9F3m5C6ih8o/aKAQx5KIeXrBUZq
+v43tK+kbYfRJHBjHWMOBbuxq0G/VmGPf9q9GtGqGXuxZG+w+rYtJx1OeMQZShjIZ
+ITl5CYeahrXtK4mo+fF2PMh3m5UE861LWuKKWhPwpJiWXC5grDNcjlHj1pcTdeip
+PjHkuJPhAoGBAIh35DptqqdicOd3dr/+/m2YQywY8aSpMrR0bC06aAkscD7oq4tt
+s/jwl0UlHIrEm/aMN7OnGIbpfkVdExfGKYaa5NRlgOwQpShwLufIo/c8fErd2zb8
+K3ptlwBxMrayMXpS3DP78r83Z0B8/FSK2guelzdRJ3ftipZ9io1Gss1C
+-----END RSA PRIVATE KEY-----
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index 05880a1048..ecce473b01 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -17,12 +17,14 @@ import logging
from mock import Mock
import treq
+from service_identity import VerificationError
from zope.interface import implementer
from twisted.internet import defer
from twisted.internet._sslverify import ClientTLSOptions, OpenSSLCertificateOptions
from twisted.internet.protocol import Factory
from twisted.protocols.tls import TLSMemoryBIOFactory
+from twisted.web._newclient import ResponseNeverReceived
from twisted.web.http import HTTPChannel
from twisted.web.http_headers import Headers
from twisted.web.iweb import IPolicyForHTTPS
@@ -37,13 +39,29 @@ from synapse.http.federation.srv_resolver import Server
from synapse.util.caches.ttlcache import TTLCache
from synapse.util.logcontext import LoggingContext
-from tests.http import ServerTLSContext
+from tests.http import TestServerTLSConnectionFactory, get_test_ca_cert_file
from tests.server import FakeTransport, ThreadedMemoryReactorClock
from tests.unittest import TestCase
from tests.utils import default_config
logger = logging.getLogger(__name__)
+test_server_connection_factory = None
+
+
+def get_connection_factory():
+ # this needs to happen once, but not until we are ready to run the first test
+ global test_server_connection_factory
+ if test_server_connection_factory is None:
+ test_server_connection_factory = TestServerTLSConnectionFactory(sanlist=[
+ b'DNS:testserv',
+ b'DNS:target-server',
+ b'DNS:xn--bcher-kva.com',
+ b'IP:1.2.3.4',
+ b'IP:::1',
+ ])
+ return test_server_connection_factory
+
class MatrixFederationAgentTests(TestCase):
def setUp(self):
@@ -53,12 +71,11 @@ class MatrixFederationAgentTests(TestCase):
self.well_known_cache = TTLCache("test_cache", timer=self.reactor.seconds)
- # for now, we disable cert verification for the test, since the cert we
- # present will not be trusted. We should do better here, though.
config_dict = default_config("test", parse=False)
- config_dict["federation_verify_certificates"] = False
- config_dict["trusted_key_servers"] = []
- config = HomeServerConfig()
+ config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()]
+ # config_dict["trusted_key_servers"] = []
+
+ self._config = config = HomeServerConfig()
config.parse_config_dict(config_dict)
self.agent = MatrixFederationAgent(
@@ -77,7 +94,7 @@ class MatrixFederationAgentTests(TestCase):
"""
# build the test server
- server_tls_protocol = _build_test_server()
+ server_tls_protocol = _build_test_server(get_connection_factory())
# now, tell the client protocol factory to build the client protocol (it will be a
# _WrappingProtocol, around a TLSMemoryBIOProtocol, around an
@@ -328,6 +345,88 @@ class MatrixFederationAgentTests(TestCase):
self.reactor.pump((0.1,))
self.successResultOf(test_d)
+ def test_get_hostname_bad_cert(self):
+ """
+ Test the behaviour when the certificate on the server doesn't match the hostname
+ """
+ self.mock_resolver.resolve_service.side_effect = lambda _: []
+ self.reactor.lookups["testserv1"] = "1.2.3.4"
+
+ test_d = self._make_get_request(b"matrix://testserv1/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ # No SRV record lookup yet
+ self.mock_resolver.resolve_service.assert_not_called()
+
+ # there should be an attempt to connect on port 443 for the .well-known
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ # fonx the connection
+ client_factory.clientConnectionFailed(None, Exception("nope"))
+
+ # attemptdelay on the hostnameendpoint is 0.3, so takes that long before the
+ # .well-known request fails.
+ self.reactor.pump((0.4,))
+
+ # now there should be a SRV lookup
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.testserv1"
+ )
+
+ # we should fall back to a direct connection
+ self.assertEqual(len(clients), 2)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[1]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 8448)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(client_factory, expected_sni=b'testserv1')
+
+ # there should be no requests
+ self.assertEqual(len(http_server.requests), 0)
+
+ # ... and the request should have failed
+ e = self.failureResultOf(test_d, ResponseNeverReceived)
+ failure_reason = e.value.reasons[0]
+ self.assertIsInstance(failure_reason.value, VerificationError)
+
+ def test_get_ip_address_bad_cert(self):
+ """
+ Test the behaviour when the server name contains an explicit IP, but
+ the server cert doesn't cover it
+ """
+ # there will be a getaddrinfo on the IP
+ self.reactor.lookups["1.2.3.5"] = "1.2.3.5"
+
+ test_d = self._make_get_request(b"matrix://1.2.3.5/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ # Make sure treq is trying to connect
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.5')
+ self.assertEqual(port, 8448)
+
+ # make a test server, and wire up the client
+ http_server = self._make_connection(client_factory, expected_sni=None)
+
+ # there should be no requests
+ self.assertEqual(len(http_server.requests), 0)
+
+ # ... and the request should have failed
+ e = self.failureResultOf(test_d, ResponseNeverReceived)
+ failure_reason = e.value.reasons[0]
+ self.assertIsInstance(failure_reason.value, VerificationError)
+
def test_get_no_srv_no_well_known(self):
"""
Test the behaviour when the server name has no port, no SRV, and no well-known
@@ -585,6 +684,49 @@ class MatrixFederationAgentTests(TestCase):
self.reactor.pump((0.1,))
self.successResultOf(test_d)
+ def test_get_well_known_unsigned_cert(self):
+ """Test the behaviour when the .well-known server presents a cert
+ not signed by a CA
+ """
+
+ # we use the same test server as the other tests, but use an agent
+ # with _well_known_tls_policy left to the default, which will not
+ # trust it (since the presented cert is signed by a test CA)
+
+ self.mock_resolver.resolve_service.side_effect = lambda _: []
+ self.reactor.lookups["testserv"] = "1.2.3.4"
+
+ agent = MatrixFederationAgent(
+ reactor=self.reactor,
+ tls_client_options_factory=ClientTLSOptionsFactory(self._config),
+ _srv_resolver=self.mock_resolver,
+ _well_known_cache=self.well_known_cache,
+ )
+
+ test_d = agent.request(b"GET", b"matrix://testserv/foo/bar")
+
+ # Nothing happened yet
+ self.assertNoResult(test_d)
+
+ # there should be an attempt to connect on port 443 for the .well-known
+ clients = self.reactor.tcpClients
+ self.assertEqual(len(clients), 1)
+ (host, port, client_factory, _timeout, _bindAddress) = clients[0]
+ self.assertEqual(host, '1.2.3.4')
+ self.assertEqual(port, 443)
+
+ http_proto = self._make_connection(
+ client_factory, expected_sni=b"testserv",
+ )
+
+ # there should be no requests
+ self.assertEqual(len(http_proto.requests), 0)
+
+ # and there should be a SRV lookup instead
+ self.mock_resolver.resolve_service.assert_called_once_with(
+ b"_matrix._tcp.testserv"
+ )
+
def test_get_hostname_srv(self):
"""
Test the behaviour when there is a single SRV record
@@ -918,11 +1060,17 @@ def _check_logcontext(context):
raise AssertionError("Expected logcontext %s but was %s" % (context, current))
-def _build_test_server():
+def _build_test_server(connection_creator):
"""Construct a test server
This builds an HTTP channel, wrapped with a TLSMemoryBIOProtocol
+ Args:
+ connection_creator (IOpenSSLServerConnectionCreator): thing to build
+ SSL connections
+ sanlist (list[bytes]): list of the SAN entries for the cert returned
+ by the server
+
Returns:
TLSMemoryBIOProtocol
"""
@@ -931,7 +1079,7 @@ def _build_test_server():
server_factory.log = _log_request
server_tls_factory = TLSMemoryBIOFactory(
- ServerTLSContext(), isClient=False, wrappedFactory=server_factory
+ connection_creator, isClient=False, wrappedFactory=server_factory
)
return server_tls_factory.buildProtocol(None)
@@ -944,7 +1092,8 @@ def _log_request(request):
@implementer(IPolicyForHTTPS)
class TrustingTLSPolicyForHTTPS(object):
- """An IPolicyForHTTPS which doesn't do any certificate verification"""
+ """An IPolicyForHTTPS which checks that the certificate belongs to the
+ right server, but doesn't check the certificate chain."""
def creatorForNetloc(self, hostname, port):
certificateOptions = OpenSSLCertificateOptions()
diff --git a/tests/http/server.key b/tests/http/server.key
new file mode 100644
index 0000000000..c53ee02b21
--- /dev/null
+++ b/tests/http/server.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAvUAWLOE6TEp3FYSfEnJMwYtJg3KIW5BjiAOOvFVOVQfJ5eEa
+vzyJ1Z+8DUgLznFnUkAeD9GjPvP7awl3NPJKLQSMkV5Tp+ea4YyV+Aa4R7flROEa
+zCGvmleydZw0VqN1atVZ0ikEoglM/APJQd70ec7KSR3QoxaV2/VNCHmyAPdP+0WI
+llV54VXX1CZrWSHaCSn1gzo3WjnGbxTOCQE5Z4k5hqJAwLWWhxDv+FX/jD38Sq3H
+gMFNpXJv6FYwwaKU8awghHdSY/qlBPE/1rU83vIBFJ3jW6I1WnQDfCQ69of5vshK
+N4v4hok56ScwdUnk8lw6xvJx1Uav/XQB9qGh4QIDAQABAoIBAQCHLO5p8hotAgdb
+JFZm26N9nxrMPBOvq0ucjEX4ucnwrFaGzynGrNwa7TRqHCrqs0/EjS2ryOacgbL0
+eldeRy26SASLlN+WD7UuI7e+6DXabDzj3RHB+tGuIbPDk+ZCeBDXVTsKBOhdQN1v
+KNkpJrJjCtSsMxKiWvCBow353srJKqCDZcF5NIBYBeDBPMoMbfYn5dJ9JhEf+2h4
+0iwpnWDX1Vqf46pCRa0hwEyMXycGeV2CnfJSyV7z52ZHQrvkz8QspSnPpnlCnbOE
+UAvc8kZ5e8oZE7W+JfkK38vHbEGM1FCrBmrC/46uUGMRpZfDferGs91RwQVq/F0n
+JN9hLzsBAoGBAPh2pm9Xt7a4fWSkX0cDgjI7PT2BvLUjbRwKLV+459uDa7+qRoGE
+sSwb2QBqmQ1kbr9JyTS+Ld8dyUTsGHZK+YbTieAxI3FBdKsuFtcYJO/REN0vik+6
+fMaBHPvDHSU2ioq7spZ4JBFskzqs38FvZ0lX7aa3fguMk8GMLnofQ8QxAoGBAML9
+o5sJLN9Tk9bv2aFgnERgfRfNjjV4Wd99TsktnCD04D1GrP2eDSLfpwFlCnguck6b
+jxikqcolsNhZH4dgYHqRNj+IljSdl+sYZiygO6Ld0XU+dEFO86N3E9NzZhKcQ1at
+85VdwNPCS7JM2fIxEvS9xfbVnsmK6/37ZZ5iI7yxAoGBALw2vRtJGmy60pojfd1A
+hibhAyINnlKlFGkSOI7zdgeuRTf6l9BTIRclvTt4hJpFgzM6hMWEbyE94hJoupsZ
+bm443o/LCWsox2VI05p6urhD6f9znNWKkiyY78izY+elqksvpjgfqEresaTYAeP5
+LQe9KNSK2VuMUP1j4G04M9BxAoGAWe8ITZJuytZOgrz/YIohqPvj1l2tcIYA1a6C
+7xEFSMIIxtpZIWSLZIFJEsCakpHBkPX4iwIveZfmt/JrM1JFTWK6ZZVGyh/BmOIZ
+Bg4lU1oBqJTUo+aZQtTCJS29b2n5OPpkNYkXTdP4e9UsVKNDvfPlYZJneUeEzxDr
+bqCPIRECgYA544KMwrWxDQZg1dsKWgdVVKx80wEFZAiQr9+0KF6ch6Iu7lwGJHFY
+iI6O85paX41qeC/Fo+feIWJVJU2GvG6eBsbO4bmq+KSg4NkABJSYxodgBp9ftNeD
+jo1tfw+gudlNe5jXHu7oSX93tqGjR4Cnlgan/KtfkB96yHOumGmOhQ==
+-----END RSA PRIVATE KEY-----
diff --git a/tests/http/server.pem b/tests/http/server.pem
deleted file mode 100644
index 0584cf1a80..0000000000
--- a/tests/http/server.pem
+++ /dev/null
@@ -1,81 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCgF43/3lAgJ+p0
-x7Rn8UcL8a4fctvdkikvZrCngw96LkB34Evfq8YGWlOVjU+f9naUJLAKMatmAfEN
-r+rMX4VOXmpTwuu6iLtqwreUrRFMESyrmvQxa15p+y85gkY0CFmXMblv6ORbxHTG
-ncBGwST4WK4Poewcgt6jcISFCESTUKu1zc3cw1ANIDRyDLB5K44KwIe36dcKckyN
-Kdtv4BJ+3fcIZIkPJH62zqCypgFF1oiFt40uJzClxgHdJZlKYpgkfnDTckw4Y/Mx
-9k8BbE310KAzUNMV9H7I1eEolzrNr66FQj1eN64X/dqO8lTbwCqAd4diCT4sIUk0
-0SVsAUjNd3g8j651hx+Qb1t8fuOjrny8dmeMxtUgIBHoQcpcj76R55Fs7KZ9uar0
-8OFTyGIze51W1jG2K/7/5M1zxIqrA+7lsXu5OR81s7I+Ng/UUAhiHA/z+42/aiNa
-qEuk6tqj3rHfLctnCbtZ+JrRNqSSwEi8F0lMA021ivEd2eJV+284OyJjhXOmKHrX
-QADHrmS7Sh4syTZvRNm9n+qWID0KdDr2Sji/KnS3Enp44HDQ4xriT6/xhwEGsyuX
-oH5aAkdLznulbWkHBbyx1SUQSTLpOqzaioF9m1vRrLsFvrkrY3D253mPJ5eU9HM/
-dilduFcUgj4rz+6cdXUAh+KK/v95zwIDAQABAoICAFG5tJPaOa0ws0/KYx5s3YgL
-aIhFalhCNSQtmCDrlwsYcXDA3/rfBchYdDL0YKGYgBBAal3J3WXFt/j0xThvyu2m
-5UC9UPl4s7RckrsjXqEmY1d3UxGnbhtMT19cUdpeKN42VCP9EBaIw9Rg07dLAkSF
-gNYaIx6q8F0fI4eGIPvTQtUcqur4CfWpaxyNvckdovV6M85/YXfDwbCOnacPDGIX
-jfSK3i0MxGMuOHr6o8uzKR6aBUh6WStHWcw7VXXTvzdiFNbckmx3Gb93rf1b/LBw
-QFfx+tBKcC62gKroCOzXso/0sL9YTVeSD/DJZOiJwSiz3Dj/3u1IUMbVvfTU8wSi
-CYS7Z+jHxwSOCSSNTXm1wO/MtDsNKbI1+R0cohr/J9pOMQvrVh1+2zSDOFvXAQ1S
-yvjn+uqdmijRoV2VEGVHd+34C+ci7eJGAhL/f92PohuuFR2shUETgGWzpACZSJwg
-j1d90Hs81hj07vWRb+xCeDh00vimQngz9AD8vYvv/S4mqRGQ6TZdfjLoUwSTg0JD
-6sQgRXX026gQhLhn687vLKZfHwzQPZkpQdxOR0dTZ/ho/RyGGRJXH4kN4cA2tPr+
-AKYQ29YXGlEzGG7OqikaZcprNWG6UFgEpuXyBxCgp9r4ladZo3J+1Rhgus8ZYatd
-uO98q3WEBmP6CZ2n32mBAoIBAQDS/c/ybFTos0YpGHakwdmSfj5OOQJto2y8ywfG
-qDHwO0ebcpNnS1+MA+7XbKUQb/3Iq7iJljkkzJG2DIJ6rpKynYts1ViYpM7M/t0T
-W3V1gvUcUL62iqkgws4pnpWmubFkqV31cPSHcfIIclnzeQ1aOEGsGHNAvhty0ciC
-DnkJACbqApvopFLOR5f6UFTtKExE+hDH0WqgpsCAKJ1L4g6pBzZatI32/CN9JEVU
-tDbxLV75hHlFFjUrG7nT1rPyr/gI8Ceh9/2xeXPfjJUR0PrG3U1nwLqUCZkvFzO6
-XpN2+A+/v4v5xqMjKDKDFy1oq6SCMomwv/viw6wl/84TMbolAoIBAQDCPiMecnR8
-REik6tqVzQO/uSe9ZHjz6J15t5xdwaI6HpSwLlIkQPkLTjyXtFpemK5DOYRxrJvQ
-remfrZrN2qtLlb/DKpuGPWRsPOvWCrSuNEp48ivUehtclljrzxAFfy0sM+fWeJ48
-nTnR+td9KNhjNtZixzWdAy/mE+jdaMsXVnk66L73Uz+2WsnvVMW2R6cpCR0F2eP/
-B4zDWRqlT2w47sePAB81mFYSQLvPC6Xcgg1OqMubfiizJI49c8DO6Jt+FFYdsxhd
-kG52Eqa/Net6rN3ueiS6yXL5TU3Y6g96bPA2KyNCypucGcddcBfqaiVx/o4AH6yT
-NrdsrYtyvk/jAoIBAQDHUwKVeeRJJbvdbQAArCV4MI155n+1xhMe1AuXkCQFWGtQ
-nlBE4D72jmyf1UKnIbW2Uwv15xY6/ouVWYIWlj9+QDmMaozVP7Uiko+WDuwLRNl8
-k4dn+dzHV2HejbPBG2JLv3lFOx23q1zEwArcaXrExaq9Ayg2fKJ/uVHcFAIiD6Oz
-pR1XDY4w1A/uaN+iYFSVQUyDCQLbnEz1hej73CaPZoHh9Pq83vxD5/UbjVjuRTeZ
-L55FNzKpc/r89rNvTPBcuUwnxplDhYKDKVNWzn9rSXwrzTY2Tk8J3rh+k4RqevSd
-6D47jH1n5Dy7/TRn0ueKHGZZtTUnyEUkbOJo3ayFAoIBAHKDyZaQqaX9Z8p6fwWj
-yVsFoK0ih8BcWkLBAdmwZ6DWGJjJpjmjaG/G3ygc9s4gO1R8m12dAnuDnGE8KzDD
-gwtbrKM2Alyg4wyA2hTlWOH/CAzH0RlCJ9Fs/d1/xJVJBeuyajLiB3/6vXTS6qnq
-I7BSSxAPG8eGcn21LSsjNeB7ZZtaTgNnu/8ZBUYo9yrgkWc67TZe3/ChldYxOOlO
-qqHh/BqNWtjxB4VZTp/g4RbgQVInZ2ozdXEv0v/dt0UEk29ANAjsZif7F3RayJ2f
-/0TilzCaJ/9K9pKNhaClVRy7Dt8QjYg6BIWCGSw4ApF7pLnQ9gySn95mersCkVzD
-YDsCggEAb0E/TORjQhKfNQvahyLfQFm151e+HIoqBqa4WFyfFxe/IJUaLH/JSSFw
-VohbQqPdCmaAeuQ8ERL564DdkcY5BgKcax79fLLCOYP5bT11aQx6uFpfl2Dcm6Z9
-QdCRI4jzPftsd5fxLNH1XtGyC4t6vTic4Pji2O71WgWzx0j5v4aeDY4sZQeFxqCV
-/q7Ee8hem1Rn5RFHu14FV45RS4LAWl6wvf5pQtneSKzx8YL0GZIRRytOzdEfnGKr
-FeUlAj5uL+5/p0ZEgM7gPsEBwdm8scF79qSUn8UWSoXNeIauF9D4BDg8RZcFFxka
-KILVFsq3cQC+bEnoM4eVbjEQkGs1RQ==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIIE/jCCAuagAwIBAgIJANFtVaGvJWZlMA0GCSqGSIb3DQEBCwUAMBMxETAPBgNV
-BAMMCHRlc3RzZXJ2MCAXDTE5MDEyNzIyMDIzNloYDzIxMTkwMTAzMjIwMjM2WjAT
-MREwDwYDVQQDDAh0ZXN0c2VydjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC
-ggIBAKAXjf/eUCAn6nTHtGfxRwvxrh9y292SKS9msKeDD3ouQHfgS9+rxgZaU5WN
-T5/2dpQksAoxq2YB8Q2v6sxfhU5ealPC67qIu2rCt5StEUwRLKua9DFrXmn7LzmC
-RjQIWZcxuW/o5FvEdMadwEbBJPhYrg+h7ByC3qNwhIUIRJNQq7XNzdzDUA0gNHIM
-sHkrjgrAh7fp1wpyTI0p22/gEn7d9whkiQ8kfrbOoLKmAUXWiIW3jS4nMKXGAd0l
-mUpimCR+cNNyTDhj8zH2TwFsTfXQoDNQ0xX0fsjV4SiXOs2vroVCPV43rhf92o7y
-VNvAKoB3h2IJPiwhSTTRJWwBSM13eDyPrnWHH5BvW3x+46OufLx2Z4zG1SAgEehB
-ylyPvpHnkWzspn25qvTw4VPIYjN7nVbWMbYr/v/kzXPEiqsD7uWxe7k5HzWzsj42
-D9RQCGIcD/P7jb9qI1qoS6Tq2qPesd8ty2cJu1n4mtE2pJLASLwXSUwDTbWK8R3Z
-4lX7bzg7ImOFc6YoetdAAMeuZLtKHizJNm9E2b2f6pYgPQp0OvZKOL8qdLcSenjg
-cNDjGuJPr/GHAQazK5egfloCR0vOe6VtaQcFvLHVJRBJMuk6rNqKgX2bW9GsuwW+
-uStjcPbneY8nl5T0cz92KV24VxSCPivP7px1dQCH4or+/3nPAgMBAAGjUzBRMB0G
-A1UdDgQWBBQcQZpzLzTk5KdS/Iz7sGCV7gTd/zAfBgNVHSMEGDAWgBQcQZpzLzTk
-5KdS/Iz7sGCV7gTd/zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IC
-AQAr/Pgha57jqYsDDX1LyRrVdqoVBpLBeB7x/p9dKYm7S6tBTDFNMZ0SZyQP8VEG
-7UoC9/OQ9nCdEMoR7ZKpQsmipwcIqpXHS6l4YOkf5EEq5jpMgvlEesHmBJJeJew/
-FEPDl1bl8d0tSrmWaL3qepmwzA+2lwAAouWk2n+rLiP8CZ3jZeoTXFqYYrUlEqO9
-fHMvuWqTV4KCSyNY+GWCrnHetulgKHlg+W2J1mZnrCKcBhWf9C2DesTJO+JldIeM
-ornTFquSt21hZi+k3aySuMn2N3MWiNL8XsZVsAnPSs0zA+2fxjJkShls8Gc7cCvd
-a6XrNC+PY6pONguo7rEU4HiwbvnawSTngFFglmH/ImdA/HkaAekW6o82aI8/UxFx
-V9fFMO3iKDQdOrg77hI1bx9RlzKNZZinE2/Pu26fWd5d2zqDWCjl8ykGQRAfXgYN
-H3BjgyXLl+ao5/pOUYYtzm3ruTXTgRcy5hhL6hVTYhSrf9vYh4LNIeXNKnZ78tyG
-TX77/kU2qXhBGCFEUUMqUNV/+ITir2lmoxVjknt19M07aGr8C7SgYt6Rs+qDpMiy
-JurgvRh8LpVq4pHx1efxzxCFmo58DMrG40I0+CF3y/niNpOb1gp2wAqByRiORkds
-f0ytW6qZ0TpHbD6gOtQLYDnhx3ISuX+QYSekVwQUpffeWQ==
------END CERTIFICATE-----
|
