diff options
-rw-r--r-- | changelog.d/4483.feature | 1 | ||||
-rw-r--r-- | changelog.d/4498.misc | 1 | ||||
-rw-r--r-- | changelog.d/4509.removal | 1 | ||||
-rw-r--r-- | changelog.d/4511.feature | 1 | ||||
-rw-r--r-- | changelog.d/4512.bugfix | 1 | ||||
-rw-r--r-- | synapse/api/constants.py | 2 | ||||
-rw-r--r-- | synapse/config/consent_config.py | 14 | ||||
-rw-r--r-- | synapse/config/server.py | 2 | ||||
-rw-r--r-- | synapse/config/tls.py | 61 | ||||
-rw-r--r-- | synapse/events/__init__.py | 110 | ||||
-rw-r--r-- | synapse/events/builder.py | 12 | ||||
-rw-r--r-- | synapse/events/utils.py | 3 | ||||
-rw-r--r-- | synapse/http/federation/matrix_federation_agent.py | 33 | ||||
-rw-r--r-- | synapse/rest/consent/consent_resource.py | 11 | ||||
-rw-r--r-- | tests/config/test_generate.py | 2 | ||||
-rw-r--r-- | tests/config/test_tls.py | 75 | ||||
-rw-r--r-- | tests/http/federation/test_matrix_federation_agent.py | 1 |
17 files changed, 234 insertions, 97 deletions
diff --git a/changelog.d/4483.feature b/changelog.d/4483.feature new file mode 100644 index 0000000000..9538c64f08 --- /dev/null +++ b/changelog.d/4483.feature @@ -0,0 +1 @@ +Add support for room version 3 diff --git a/changelog.d/4498.misc b/changelog.d/4498.misc new file mode 100644 index 0000000000..8b3bc94a34 --- /dev/null +++ b/changelog.d/4498.misc @@ -0,0 +1 @@ +Clarify documentation for the `public_baseurl` config param diff --git a/changelog.d/4509.removal b/changelog.d/4509.removal new file mode 100644 index 0000000000..9165009813 --- /dev/null +++ b/changelog.d/4509.removal @@ -0,0 +1 @@ +Synapse no longer generates self-signed TLS certificates when generating a configuration file. diff --git a/changelog.d/4511.feature b/changelog.d/4511.feature new file mode 100644 index 0000000000..bda713adf9 --- /dev/null +++ b/changelog.d/4511.feature @@ -0,0 +1 @@ +Implement MSC1708 (.well-known routing for server-server federation) \ No newline at end of file diff --git a/changelog.d/4512.bugfix b/changelog.d/4512.bugfix new file mode 100644 index 0000000000..7a1345c4ac --- /dev/null +++ b/changelog.d/4512.bugfix @@ -0,0 +1 @@ +Fix a bug where setting a relative consent directory path would cause a crash. \ No newline at end of file diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 51ee078bc3..b248e193fa 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -126,10 +126,12 @@ class EventFormatVersions(object): independently from the room version. """ V1 = 1 + V2 = 2 KNOWN_EVENT_FORMAT_VERSIONS = { EventFormatVersions.V1, + EventFormatVersions.V2, } diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index f193a090ae..9f2e85342f 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from os import path + +from synapse.config import ConfigError + from ._base import Config DEFAULT_CONFIG = """\ @@ -85,7 +89,15 @@ class ConsentConfig(Config): if consent_config is None: return self.user_consent_version = str(consent_config["version"]) - self.user_consent_template_dir = consent_config["template_dir"] + self.user_consent_template_dir = self.abspath( + consent_config["template_dir"] + ) + if not path.isdir(self.user_consent_template_dir): + raise ConfigError( + "Could not find template directory '%s'" % ( + self.user_consent_template_dir, + ), + ) self.user_consent_server_notice_content = consent_config.get( "server_notice_content", ) diff --git a/synapse/config/server.py b/synapse/config/server.py index 22dcc87d8a..268a43ff00 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -261,7 +261,7 @@ class ServerConfig(Config): # enter into the 'custom HS URL' field on their client. If you # use synapse with a reverse proxy, this should be the URL to reach # synapse via the proxy. - # public_baseurl: https://example.com:8448/ + # public_baseurl: https://example.com/ # Set the soft limit on the number of file descriptors synapse can use # Zero is used to indicate synapse should set the soft limit to the diff --git a/synapse/config/tls.py b/synapse/config/tls.py index a75e233aa0..734f612db7 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -15,6 +15,7 @@ import logging import os +import warnings from datetime import datetime from hashlib import sha256 @@ -39,8 +40,8 @@ class TlsConfig(Config): self.acme_bind_addresses = acme_config.get("bind_addresses", ["127.0.0.1"]) self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30) - self.tls_certificate_file = os.path.abspath(config.get("tls_certificate_path")) - self.tls_private_key_file = os.path.abspath(config.get("tls_private_key_path")) + self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) + self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) self._original_tls_fingerprints = config["tls_fingerprints"] self.tls_fingerprints = list(self._original_tls_fingerprints) self.no_tls = config.get("no_tls", False) @@ -94,6 +95,16 @@ class TlsConfig(Config): """ self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file) + # Check if it is self-signed, and issue a warning if so. + if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject(): + warnings.warn( + ( + "Self-signed TLS certificates will not be accepted by Synapse 1.0. " + "Please either provide a valid certificate, or use Synapse's ACME " + "support to provision one." + ) + ) + if not self.no_tls: self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) @@ -118,10 +129,11 @@ class TlsConfig(Config): return ( """\ # PEM encoded X509 certificate for TLS. - # You can replace the self-signed certificate that synapse - # autogenerates on launch with your own SSL certificate + key pair - # if you like. Any required intermediary certificates can be - # appended after the primary certificate in hierarchical order. + # This certificate, as of Synapse 1.0, will need to be a valid + # and verifiable certificate, with a root that is available in + # the root store of other servers you wish to federate to. Any + # required intermediary certificates can be appended after the + # primary certificate in hierarchical order. tls_certificate_path: "%(tls_certificate_path)s" # PEM encoded private key for TLS @@ -183,40 +195,3 @@ class TlsConfig(Config): def read_tls_private_key(self, private_key_path): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - - def generate_files(self, config): - tls_certificate_path = config["tls_certificate_path"] - tls_private_key_path = config["tls_private_key_path"] - - if not self.path_exists(tls_private_key_path): - with open(tls_private_key_path, "wb") as private_key_file: - tls_private_key = crypto.PKey() - tls_private_key.generate_key(crypto.TYPE_RSA, 2048) - private_key_pem = crypto.dump_privatekey( - crypto.FILETYPE_PEM, tls_private_key - ) - private_key_file.write(private_key_pem) - else: - with open(tls_private_key_path) as private_key_file: - private_key_pem = private_key_file.read() - tls_private_key = crypto.load_privatekey( - crypto.FILETYPE_PEM, private_key_pem - ) - - if not self.path_exists(tls_certificate_path): - with open(tls_certificate_path, "wb") as certificate_file: - cert = crypto.X509() - subject = cert.get_subject() - subject.CN = config["server_name"] - - cert.set_serial_number(1000) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) - cert.set_issuer(cert.get_subject()) - cert.set_pubkey(tls_private_key) - - cert.sign(tls_private_key, 'sha256') - - cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - - certificate_file.write(cert_pem) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 3fe52aaa45..627c55c404 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,11 +19,9 @@ from distutils.util import strtobool import six -from synapse.api.constants import ( - KNOWN_EVENT_FORMAT_VERSIONS, - KNOWN_ROOM_VERSIONS, - EventFormatVersions, -) +from unpaddedbase64 import encode_base64 + +from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions, RoomVersions from synapse.util.caches import intern_dict from synapse.util.frozenutils import freeze @@ -225,22 +224,91 @@ class FrozenEvent(EventBase): rejected_reason=rejected_reason, ) - @staticmethod - def from_event(event): - e = FrozenEvent( - event.get_pdu_json() + def __str__(self): + return self.__repr__() + + def __repr__(self): + return "<FrozenEvent event_id='%s', type='%s', state_key='%s'>" % ( + self.get("event_id", None), + self.get("type", None), + self.get("state_key", None), ) - e.internal_metadata = event.internal_metadata - return e +class FrozenEventV2(EventBase): + format_version = EventFormatVersions.V2 # All events of this type are V2 + + def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None): + event_dict = dict(event_dict) + + # Signatures is a dict of dicts, and this is faster than doing a + # copy.deepcopy + signatures = { + name: {sig_id: sig for sig_id, sig in sigs.items()} + for name, sigs in event_dict.pop("signatures", {}).items() + } + + assert "event_id" not in event_dict + + unsigned = dict(event_dict.pop("unsigned", {})) + + # We intern these strings because they turn up a lot (especially when + # caching). + event_dict = intern_dict(event_dict) + + if USE_FROZEN_DICTS: + frozen_dict = freeze(event_dict) + else: + frozen_dict = event_dict + + self._event_id = None + self.type = event_dict["type"] + if "state_key" in event_dict: + self.state_key = event_dict["state_key"] + + super(FrozenEventV2, self).__init__( + frozen_dict, + signatures=signatures, + unsigned=unsigned, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + @property + def event_id(self): + # We have to import this here as otherwise we get an import loop which + # is hard to break. + from synapse.crypto.event_signing import compute_event_reference_hash + + if self._event_id: + return self._event_id + self._event_id = "$" + encode_base64(compute_event_reference_hash(self)[1]) + return self._event_id + + def prev_event_ids(self): + """Returns the list of prev event IDs. The order matches the order + specified in the event, though there is no meaning to it. + + Returns: + list[str]: The list of event IDs of this event's prev_events + """ + return self.prev_events + + def auth_event_ids(self): + """Returns the list of auth event IDs. The order matches the order + specified in the event, though there is no meaning to it. + + Returns: + list[str]: The list of event IDs of this event's auth_events + """ + return self.auth_events def __str__(self): return self.__repr__() def __repr__(self): - return "<FrozenEvent event_id='%s', type='%s', state_key='%s'>" % ( - self.get("event_id", None), + return "<FrozenEventV2 event_id='%s', type='%s', state_key='%s'>" % ( + self.event_id, self.get("type", None), self.get("state_key", None), ) @@ -259,7 +327,13 @@ def room_version_to_event_format(room_version): # We should have already checked version, so this should not happen raise RuntimeError("Unrecognized room version %s" % (room_version,)) - return EventFormatVersions.V1 + if room_version in ( + RoomVersions.V1, RoomVersions.V2, RoomVersions.VDH_TEST, + RoomVersions.STATE_V2_TEST, + ): + return EventFormatVersions.V1 + else: + raise RuntimeError("Unrecognized room version %s" % (room_version,)) def event_type_from_format_version(format_version): @@ -273,8 +347,12 @@ def event_type_from_format_version(format_version): type: A type that can be initialized as per the initializer of `FrozenEvent` """ - if format_version not in KNOWN_EVENT_FORMAT_VERSIONS: + + if format_version == EventFormatVersions.V1: + return FrozenEvent + elif format_version == EventFormatVersions.V2: + return FrozenEventV2 + else: raise Exception( "No event format %r" % (format_version,) ) - return FrozenEvent diff --git a/synapse/events/builder.py b/synapse/events/builder.py index fb0683cea8..06e01be918 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -21,6 +21,7 @@ from synapse.api.constants import ( KNOWN_EVENT_FORMAT_VERSIONS, KNOWN_ROOM_VERSIONS, MAX_DEPTH, + EventFormatVersions, ) from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.types import EventID @@ -109,8 +110,12 @@ class EventBuilder(object): self, state_ids, ) - auth_events = yield self._store.add_event_hashes(auth_ids) - prev_events = yield self._store.add_event_hashes(prev_event_ids) + if self.format_version == EventFormatVersions.V1: + auth_events = yield self._store.add_event_hashes(auth_ids) + prev_events = yield self._store.add_event_hashes(prev_event_ids) + else: + auth_events = auth_ids + prev_events = prev_event_ids old_depth = yield self._store.get_max_depth_of( prev_event_ids, @@ -228,7 +233,8 @@ def create_local_event_from_event_dict(clock, hostname, signing_key, time_now = int(clock.time_msec()) - event_dict["event_id"] = _create_event_id(clock, hostname) + if format_version == EventFormatVersions.V1: + event_dict["event_id"] = _create_event_id(clock, hostname) event_dict["origin"] = hostname event_dict["origin_server_ts"] = time_now diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 63f693f259..07fccdd8f9 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -267,6 +267,7 @@ def serialize_event(e, time_now_ms, as_client_event=True, Returns: dict """ + # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -276,6 +277,8 @@ def serialize_event(e, time_now_ms, as_client_event=True, # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} + d["event_id"] = e.event_id + if "age_ts" in d["unsigned"]: d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 07c72c9351..f81fcd4301 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import cgi import json import logging @@ -263,37 +262,31 @@ class MatrixFederationAgent(object): # FIXME: add a cache uri = b"https://%s/.well-known/matrix/server" % (server_name, ) - logger.info("Fetching %s", uri.decode("ascii")) + uri_str = uri.decode("ascii") + logger.info("Fetching %s", uri_str) try: response = yield make_deferred_yieldable( self._well_known_agent.request(b"GET", uri), ) except Exception as e: - logger.info( - "Connection error fetching %s: %s", - uri.decode("ascii"), e, - ) + logger.info("Connection error fetching %s: %s", uri_str, e) defer.returnValue(None) body = yield make_deferred_yieldable(readBody(response)) if response.code != 200: - logger.info( - "Error response %i from %s: %s", - response.code, uri.decode("ascii"), body, - ) + logger.info("Error response %i from %s", response.code, uri_str) defer.returnValue(None) - content_types = response.headers.getRawHeaders(u'content-type') - if content_types is None: - raise Exception("no content-type header on .well-known response") - content_type, _opts = cgi.parse_header(content_types[-1]) - if content_type != 'application/json': - raise Exception("content-type not application/json on .well-known response") - parsed_body = json.loads(body.decode('utf-8')) - logger.info("Response from .well-known: %s", parsed_body) - if not isinstance(parsed_body, dict) or "m.server" not in parsed_body: - raise Exception("invalid .well-known response") + try: + parsed_body = json.loads(body.decode('utf-8')) + logger.info("Response from .well-known: %s", parsed_body) + if not isinstance(parsed_body, dict): + raise Exception("not a dict") + if "m.server" not in parsed_body: + raise Exception("Missing key 'm.server'") + except Exception as e: + raise Exception("invalid .well-known response from %s: %s" % (uri_str, e,)) defer.returnValue(parsed_body["m.server"].encode("ascii")) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 80611cfe84..008d4edae5 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -101,16 +101,7 @@ class ConsentResource(Resource): "missing in config file.", ) - # daemonize changes the cwd to /, so make the path absolute now. - consent_template_directory = path.abspath( - hs.config.user_consent_template_dir, - ) - if not path.isdir(consent_template_directory): - raise ConfigError( - "Could not find template directory '%s'" % ( - consent_template_directory, - ), - ) + consent_template_directory = hs.config.user_consent_template_dir loader = jinja2.FileSystemLoader(consent_template_directory) self._jinja_env = jinja2.Environment( diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py index b5ad99348d..795b4c298d 100644 --- a/tests/config/test_generate.py +++ b/tests/config/test_generate.py @@ -50,8 +50,6 @@ class ConfigGenerationTestCase(unittest.TestCase): "homeserver.yaml", "lemurs.win.log.config", "lemurs.win.signing.key", - "lemurs.win.tls.crt", - "lemurs.win.tls.key", ] ), set(os.listdir(self.dir)), diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py new file mode 100644 index 0000000000..4ccaf35603 --- /dev/null +++ b/tests/config/test_tls.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from synapse.config.tls import TlsConfig + +from tests.unittest import TestCase + + +class TLSConfigTests(TestCase): + + def test_warn_self_signed(self): + """ + Synapse will give a warning when it loads a self-signed certificate. + """ + config_dir = self.mktemp() + os.mkdir(config_dir) + with open(os.path.join(config_dir, "cert.pem"), 'w') as f: + f.write("""-----BEGIN CERTIFICATE----- +MIID6DCCAtACAws9CjANBgkqhkiG9w0BAQUFADCBtzELMAkGA1UEBhMCVFIxDzAN +BgNVBAgMBsOHb3J1bTEUMBIGA1UEBwwLQmHFn21ha8OnxLExEjAQBgNVBAMMCWxv +Y2FsaG9zdDEcMBoGA1UECgwTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECwwb +QXV0b21hdGVkIFRlc3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1 +cml0eUB0d2lzdGVkbWF0cml4LmNvbTAgFw0xNzA3MTIxNDAxNTNaGA8yMTE3MDYx +ODE0MDE1M1owgbcxCzAJBgNVBAYTAlRSMQ8wDQYDVQQIDAbDh29ydW0xFDASBgNV +BAcMC0JhxZ9tYWvDp8SxMRIwEAYDVQQDDAlsb2NhbGhvc3QxHDAaBgNVBAoME1R3 +aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNVBAsMG0F1dG9tYXRlZCBUZXN0aW5nIEF1 +dGhvcml0eTEpMCcGCSqGSIb3DQEJARYac2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5j +b20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDwT6kbqtMUI0sMkx4h +I+L780dA59KfksZCqJGmOsMD6hte9EguasfkZzvCF3dk3NhwCjFSOvKx6rCwiteo +WtYkVfo+rSuVNmt7bEsOUDtuTcaxTzIFB+yHOYwAaoz3zQkyVW0c4pzioiLCGCmf +FLdiDBQGGp74tb+7a0V6kC3vMLFoM3L6QWq5uYRB5+xLzlPJ734ltyvfZHL3Us6p +cUbK+3WTWvb4ER0W2RqArAj6Bc/ERQKIAPFEiZi9bIYTwvBH27OKHRz+KoY/G8zY ++l+WZoJqDhupRAQAuh7O7V/y6bSP+KNxJRie9QkZvw1PSaGSXtGJI3WWdO12/Ulg +epJpAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAJXEq5P9xwvP9aDkXIqzcD0L8sf8 +ewlhlxTQdeqt2Nace0Yk18lIo2oj1t86Y8jNbpAnZJeI813Rr5M7FbHCXoRc/SZG +I8OtG1xGwcok53lyDuuUUDexnK4O5BkjKiVlNPg4HPim5Kuj2hRNFfNt/F2BVIlj +iZupikC5MT1LQaRwidkSNxCku1TfAyueiBwhLnFwTmIGNnhuDCutEVAD9kFmcJN2 +SznugAcPk4doX2+rL+ila+ThqgPzIkwTUHtnmjI0TI6xsDUlXz5S3UyudrE2Qsfz +s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= +-----END CERTIFICATE-----""") + + config = { + "tls_certificate_path": os.path.join(config_dir, "cert.pem"), + "no_tls": True, + "tls_fingerprints": [] + } + + t = TlsConfig() + t.read_config(config) + t.read_certificate_from_disk() + + warnings = self.flushWarnings() + self.assertEqual(len(warnings), 1) + self.assertEqual( + warnings[0]["message"], + ( + "Self-signed TLS certificates will not be accepted by " + "Synapse 1.0. Please either provide a valid certificate, " + "or use Synapse's ACME support to provision one." + ) + ) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index bd80dd0cb6..11ea8ef10c 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -146,7 +146,6 @@ class MatrixFederationAgentTests(TestCase): [b'testserv'], ) # send back a response - request.responseHeaders.setRawHeaders(b'Content-Type', [b'application/json']) request.write(b'{ "m.server": "%s" }' % (target_server,)) request.finish() |