diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 62c633146f..5b0ca312e2 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -213,16 +213,17 @@ def refresh_certificate(hs):
Refresh the TLS certificates that Synapse is using by re-reading them from
disk and updating the TLS context factories to use them.
"""
- logging.info("Loading certificate from disk...")
- hs.config.read_certificate_from_disk()
+
+ if not hs.config.has_tls_listener():
+ # attempt to reload the certs for the good of the tls_fingerprints
+ hs.config.read_certificate_from_disk(require_cert_and_key=False)
+ return
+
+ hs.config.read_certificate_from_disk(require_cert_and_key=True)
hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
- hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- hs.config
- )
- logging.info("Certificate loaded.")
if hs._listening_services:
- logging.info("Updating context factories...")
+ logger.info("Updating context factories...")
for i in hs._listening_services:
# When you listenSSL, it doesn't make an SSL port but a TCP one with
# a TLS wrapping factory around the factory you actually want to get
@@ -237,7 +238,7 @@ def refresh_certificate(hs):
False,
i.factory.wrappedFactory
)
- logging.info("Context factories updated.")
+ logger.info("Context factories updated.")
def start(hs, listeners=None):
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index b4476bf16e..dbd98d394f 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -90,11 +90,6 @@ class SynapseHomeServer(HomeServer):
tls = listener_config.get("tls", False)
site_tag = listener_config.get("tag", port)
- if tls and config.no_tls:
- raise ConfigError(
- "Listener on port %i has TLS enabled, but no_tls is set" % (port,),
- )
-
resources = {}
for res in listener_config["resources"]:
for name in res["names"]:
diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 5858fb92b4..5aec43b702 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -257,7 +257,7 @@ class Config(object):
"--keys-directory",
metavar="DIRECTORY",
help="Used with 'generate-*' options to specify where files such as"
- " certs and signing keys should be stored in, unless explicitly"
+ " signing keys should be stored, unless explicitly"
" specified in the config.",
)
config_parser.add_argument(
@@ -313,16 +313,11 @@ class Config(object):
print(
(
"A config file has been generated in %r for server name"
- " %r with corresponding SSL keys and self-signed"
- " certificates. Please review this file and customise it"
+ " %r. Please review this file and customise it"
" to your needs."
)
% (config_path, server_name)
)
- print(
- "If this server name is incorrect, you will need to"
- " regenerate the SSL certificates"
- )
return
else:
print(
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 5aad062c36..727fdc54d8 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -42,7 +42,7 @@ from .voip import VoipConfig
from .workers import WorkerConfig
-class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
+class HomeServerConfig(ServerConfig, TlsConfig, DatabaseConfig, LoggingConfig,
RatelimitConfig, ContentRepositoryConfig, CaptchaConfig,
VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig,
AppServiceConfig, KeyConfig, SAML2Config, CasConfig,
diff --git a/synapse/config/server.py b/synapse/config/server.py
index f0a60cc712..c5c3aac8ed 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -24,6 +24,14 @@ from ._base import Config, ConfigError
logger = logging.Logger(__name__)
+# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes
+# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen
+# on IPv6 when '::' is set.
+#
+# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in
+# in the list.
+DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0']
+
class ServerConfig(Config):
@@ -118,16 +126,34 @@ class ServerConfig(Config):
self.public_baseurl += '/'
self.start_pushers = config.get("start_pushers", True)
- self.listeners = config.get("listeners", [])
+ self.listeners = []
+ for listener in config.get("listeners", []):
+ if not isinstance(listener.get("port", None), int):
+ raise ConfigError(
+ "Listener configuration is lacking a valid 'port' option"
+ )
+
+ if listener.setdefault("tls", False):
+ # no_tls is not really supported any more, but let's grandfather it in
+ # here.
+ if config.get("no_tls", False):
+ logger.info(
+ "Ignoring TLS-enabled listener on port %i due to no_tls"
+ )
+ continue
- for listener in self.listeners:
bind_address = listener.pop("bind_address", None)
bind_addresses = listener.setdefault("bind_addresses", [])
+ # if bind_address was specified, add it to the list of addresses
if bind_address:
bind_addresses.append(bind_address)
- elif not bind_addresses:
- bind_addresses.append('')
+
+ # if we still have an empty list of addresses, use the default list
+ if not bind_addresses:
+ bind_addresses.extend(DEFAULT_BIND_ADDRESSES)
+
+ self.listeners.append(listener)
if not self.web_client_location:
_warn_if_webclient_configured(self.listeners)
@@ -136,6 +162,9 @@ class ServerConfig(Config):
bind_port = config.get("bind_port")
if bind_port:
+ if config.get("no_tls", False):
+ raise ConfigError("no_tls is incompatible with bind_port")
+
self.listeners = []
bind_host = config.get("bind_host", "")
gzip_responses = config.get("gzip_responses", True)
@@ -182,6 +211,7 @@ class ServerConfig(Config):
"port": manhole,
"bind_addresses": ["127.0.0.1"],
"type": "manhole",
+ "tls": False,
})
metrics_port = config.get("metrics_port")
@@ -207,6 +237,9 @@ class ServerConfig(Config):
_check_resource_config(self.listeners)
+ def has_tls_listener(self):
+ return any(l["tls"] for l in self.listeners)
+
def default_config(self, server_name, data_dir_path, **kwargs):
_, bind_port = parse_and_validate_server_name(server_name)
if bind_port is not None:
@@ -295,84 +328,106 @@ class ServerConfig(Config):
# List of ports that Synapse should listen on, their purpose and their
# configuration.
+ #
+ # Options for each listener include:
+ #
+ # port: the TCP port to bind to
+ #
+ # bind_addresses: a list of local addresses to listen on. The default is
+ # 'all local interfaces'.
+ #
+ # type: the type of listener. Normally 'http', but other valid options are:
+ # 'manhole' (see docs/manhole.md),
+ # 'metrics' (see docs/metrics-howto.rst),
+ # 'replication' (see docs/workers.rst).
+ #
+ # tls: set to true to enable TLS for this listener. Will use the TLS
+ # key/cert specified in tls_private_key_path / tls_certificate_path.
+ #
+ # x_forwarded: Only valid for an 'http' listener. Set to true to use the
+ # X-Forwarded-For header as the client IP. Useful when Synapse is
+ # behind a reverse-proxy.
+ #
+ # resources: Only valid for an 'http' listener. A list of resources to host
+ # on this port. Options for each resource are:
+ #
+ # names: a list of names of HTTP resources. See below for a list of
+ # valid resource names.
+ #
+ # compress: set to true to enable HTTP comression for this resource.
+ #
+ # additional_resources: Only valid for an 'http' listener. A map of
+ # additional endpoints which should be loaded via dynamic modules.
+ #
+ # Valid resource names are:
+ #
+ # client: the client-server API (/_matrix/client). Also implies 'media' and
+ # 'static'.
+ #
+ # consent: user consent forms (/_matrix/consent). See
+ # docs/consent_tracking.md.
+ #
+ # federation: the server-server API (/_matrix/federation). Also implies
+ # 'media', 'keys', 'openid'
+ #
+ # keys: the key discovery API (/_matrix/keys).
+ #
+ # media: the media API (/_matrix/media).
+ #
+ # metrics: the metrics interface. See docs/metrics-howto.rst.
+ #
+ # openid: OpenID authentication.
+ #
+ # replication: the HTTP replication API (/_synapse/replication). See
+ # docs/workers.rst.
+ #
+ # static: static resources under synapse/static (/_matrix/static). (Mostly
+ # useful for 'fallback authentication'.)
+ #
+ # webclient: A web client. Requires web_client_location to be set.
+ #
listeners:
- # Main HTTPS listener
- # For when matrix traffic is sent directly to synapse.
- -
- # The port to listen for HTTPS requests on.
- port: %(bind_port)s
-
- # Local addresses to listen on.
- # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6
- # addresses by default. For most other OSes, this will only listen
- # on IPv6.
- bind_addresses:
- - '::'
- - '0.0.0.0'
-
- # This is a 'http' listener, allows us to specify 'resources'.
+ # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
+ #
+ # Disabled by default. To enable it, uncomment the following. (Note that you
+ # will also need to give Synapse a TLS key and certificate: see the TLS section
+ # below.)
+ #
+ # - port: %(bind_port)s
+ # type: http
+ # tls: true
+ # resources:
+ # - names: [client, federation]
+
+ # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy
+ # that unwraps TLS.
+ #
+ # If you plan to use a reverse proxy, please see
+ # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
+ #
+ - port: %(unsecure_port)s
+ tls: false
+ bind_addresses: ['::1', '127.0.0.1']
type: http
+ x_forwarded: true
- tls: true
-
- # Use the X-Forwarded-For (XFF) header as the client IP and not the
- # actual client IP.
- x_forwarded: false
-
- # List of HTTP resources to serve on this listener.
resources:
- -
- # List of resources to host on this listener.
- names:
- - client # The client-server APIs, both v1 and v2
- # - webclient # A web client. Requires web_client_location to be set.
-
- # Should synapse compress HTTP responses to clients that support it?
- # This should be disabled if running synapse behind a load balancer
- # that can do automatic compression.
- compress: true
-
- - names: [federation] # Federation APIs
+ - names: [client, federation]
compress: false
- # # If federation is disabled synapse can still expose the open ID endpoint
- # # to allow integrations to authenticate users
- # - names: [openid]
- # compress: false
-
- # optional list of additional endpoints which can be loaded via
- # dynamic modules
+ # example additonal_resources:
+ #
# additional_resources:
# "/_matrix/my/custom/endpoint":
# module: my_module.CustomRequestHandler
# config: {}
- # Unsecure HTTP listener,
- # For when matrix traffic passes through loadbalancer that unwraps TLS.
- - port: %(unsecure_port)s
- tls: false
- bind_addresses: ['::', '0.0.0.0']
- type: http
-
- x_forwarded: false
-
- resources:
- - names: [client]
- compress: true
- - names: [federation]
- compress: false
- # # If federation is disabled synapse can still expose the open ID endpoint
- # # to allow integrations to authenticate users
- # - names: [openid]
- # compress: false
-
# Turn on the twisted ssh manhole service on localhost on the given
# port.
# - port: 9000
# bind_addresses: ['::1', '127.0.0.1']
# type: manhole
-
# Homeserver blocking
#
# How to reach the server admin, used in ResourceLimitError
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 9fcc79816d..5fb3486db1 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -23,9 +23,9 @@ from unpaddedbase64 import encode_base64
from OpenSSL import crypto
-from synapse.config._base import Config
+from synapse.config._base import Config, ConfigError
-logger = logging.getLogger()
+logger = logging.getLogger(__name__)
class TlsConfig(Config):
@@ -45,13 +45,25 @@ class TlsConfig(Config):
self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
+
+ if self.has_tls_listener():
+ if not self.tls_certificate_file:
+ raise ConfigError(
+ "tls_certificate_path must be specified if TLS-enabled listeners are "
+ "configured."
+ )
+ if not self.tls_private_key_file:
+ raise ConfigError(
+ "tls_certificate_path must be specified if TLS-enabled listeners are "
+ "configured."
+ )
+
self._original_tls_fingerprints = config.get("tls_fingerprints", [])
if self._original_tls_fingerprints is None:
self._original_tls_fingerprints = []
self.tls_fingerprints = list(self._original_tls_fingerprints)
- self.no_tls = config.get("no_tls", False)
# This config option applies to non-federation HTTP clients
# (e.g. for talking to recaptcha, identity servers, and such)
@@ -106,36 +118,40 @@ class TlsConfig(Config):
days_remaining = (expires_on - now).days
return days_remaining
- def read_certificate_from_disk(self):
- """
- Read the certificates from disk.
+ def read_certificate_from_disk(self, require_cert_and_key):
"""
- self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file)
+ Read the certificates and private key from disk.
- # Check if it is self-signed, and issue a warning if so.
- if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject():
- warnings.warn(
- (
- "Self-signed TLS certificates will not be accepted by Synapse 1.0. "
- "Please either provide a valid certificate, or use Synapse's ACME "
- "support to provision one."
+ Args:
+ require_cert_and_key (bool): set to True to throw an error if the certificate
+ and key file are not given
+ """
+ if require_cert_and_key:
+ self.tls_private_key = self.read_tls_private_key()
+ self.tls_certificate = self.read_tls_certificate()
+ elif self.tls_certificate_file:
+ # we only need the certificate for the tls_fingerprints. Reload it if we
+ # can, but it's not a fatal error if we can't.
+ try:
+ self.tls_certificate = self.read_tls_certificate()
+ except Exception as e:
+ logger.info(
+ "Unable to read TLS certificate (%s). Ignoring as no "
+ "tls listeners enabled.", e,
)
- )
-
- if not self.no_tls:
- self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file)
self.tls_fingerprints = list(self._original_tls_fingerprints)
- # Check that our own certificate is included in the list of fingerprints
- # and include it if it is not.
- x509_certificate_bytes = crypto.dump_certificate(
- crypto.FILETYPE_ASN1, self.tls_certificate
- )
- sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
- sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
- if sha256_fingerprint not in sha256_fingerprints:
- self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
+ if self.tls_certificate:
+ # Check that our own certificate is included in the list of fingerprints
+ # and include it if it is not.
+ x509_certificate_bytes = crypto.dump_certificate(
+ crypto.FILETYPE_ASN1, self.tls_certificate
+ )
+ sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
+ sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
+ if sha256_fingerprint not in sha256_fingerprints:
+ self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
def default_config(self, config_dir_path, server_name, **kwargs):
base_key_name = os.path.join(config_dir_path, server_name)
@@ -151,6 +167,8 @@ class TlsConfig(Config):
return (
"""\
+ ## TLS ##
+
# PEM-encoded X509 certificate for TLS.
# This certificate, as of Synapse 1.0, will need to be a valid and verifiable
# certificate, signed by a recognised Certificate Authority.
@@ -158,10 +176,10 @@ class TlsConfig(Config):
# See 'ACME support' below to enable auto-provisioning this certificate via
# Let's Encrypt.
#
- tls_certificate_path: "%(tls_certificate_path)s"
+ # tls_certificate_path: "%(tls_certificate_path)s"
# PEM-encoded private key for TLS
- tls_private_key_path: "%(tls_private_key_path)s"
+ # tls_private_key_path: "%(tls_private_key_path)s"
# ACME support: This will configure Synapse to request a valid TLS certificate
# for your configured `server_name` via Let's Encrypt.
@@ -186,7 +204,7 @@ class TlsConfig(Config):
#
acme:
# ACME support is disabled by default. Uncomment the following line
- # to enable it.
+ # (and tls_certificate_path and tls_private_key_path above) to enable it.
#
# enabled: true
@@ -211,13 +229,6 @@ class TlsConfig(Config):
#
# reprovision_threshold: 30
- # If your server runs behind a reverse-proxy which terminates TLS connections
- # (for both client and federation connections), it may be useful to disable
- # All TLS support for incoming connections. Setting no_tls to True will
- # do so (and avoid the need to give synapse a TLS private key).
- #
- # no_tls: True
-
# List of allowed TLS fingerprints for this server to publish along
# with the signing keys for this server. Other matrix servers that
# make HTTPS requests to this server will check that the TLS
@@ -250,10 +261,38 @@ class TlsConfig(Config):
% locals()
)
- def read_tls_certificate(self, cert_path):
- cert_pem = self.read_file(cert_path, "tls_certificate")
- return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+ def read_tls_certificate(self):
+ """Reads the TLS certificate from the configured file, and returns it
+
+ Also checks if it is self-signed, and warns if so
+
+ Returns:
+ OpenSSL.crypto.X509: the certificate
+ """
+ cert_path = self.tls_certificate_file
+ logger.info("Loading TLS certificate from %s", cert_path)
+ cert_pem = self.read_file(cert_path, "tls_certificate_path")
+ cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+
+ # Check if it is self-signed, and issue a warning if so.
+ if cert.get_issuer() == cert.get_subject():
+ warnings.warn(
+ (
+ "Self-signed TLS certificates will not be accepted by Synapse 1.0. "
+ "Please either provide a valid certificate, or use Synapse's ACME "
+ "support to provision one."
+ )
+ )
+
+ return cert
+
+ def read_tls_private_key(self):
+ """Reads the TLS private key from the configured file, and returns it
- def read_tls_private_key(self, private_key_path):
- private_key_pem = self.read_file(private_key_path, "tls_private_key")
+ Returns:
+ OpenSSL.crypto.PKey: the private key
+ """
+ private_key_path = self.tls_private_key_file
+ logger.info("Loading TLS key from %s", private_key_path)
+ private_key_pem = self.read_file(private_key_path, "tls_private_key_path")
return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem)
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 286ad80100..85f2848fb1 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -43,9 +43,7 @@ class ServerContextFactory(ContextFactory):
logger.exception("Failed to enable elliptic curve for TLS")
context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
context.use_certificate_chain_file(config.tls_certificate_file)
-
- if not config.no_tls:
- context.use_privatekey(config.tls_private_key)
+ context.use_privatekey(config.tls_private_key)
# https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
context.set_cipher_list(
diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py
index 42b040375f..7bc174070e 100644
--- a/synapse/handlers/e2e_room_keys.py
+++ b/synapse/handlers/e2e_room_keys.py
@@ -19,7 +19,13 @@ from six import iteritems
from twisted.internet import defer
-from synapse.api.errors import NotFoundError, RoomKeysVersionError, StoreError
+from synapse.api.errors import (
+ Codes,
+ NotFoundError,
+ RoomKeysVersionError,
+ StoreError,
+ SynapseError,
+)
from synapse.util.async_helpers import Linearizer
logger = logging.getLogger(__name__)
@@ -267,7 +273,7 @@ class E2eRoomKeysHandler(object):
version(str): Optional; if None gives the most recent version
otherwise a historical one.
Raises:
- StoreError: code 404 if the requested backup version doesn't exist
+ NotFoundError: if the requested backup version doesn't exist
Returns:
A deferred of a info dict that gives the info about the new version.
@@ -279,7 +285,13 @@ class E2eRoomKeysHandler(object):
"""
with (yield self._upload_linearizer.queue(user_id)):
- res = yield self.store.get_e2e_room_keys_version_info(user_id, version)
+ try:
+ res = yield self.store.get_e2e_room_keys_version_info(user_id, version)
+ except StoreError as e:
+ if e.code == 404:
+ raise NotFoundError("Unknown backup version")
+ else:
+ raise
defer.returnValue(res)
@defer.inlineCallbacks
@@ -290,8 +302,60 @@ class E2eRoomKeysHandler(object):
user_id(str): the user whose current backup version we're deleting
version(str): the version id of the backup being deleted
Raises:
- StoreError: code 404 if this backup version doesn't exist
+ NotFoundError: if this backup version doesn't exist
"""
with (yield self._upload_linearizer.queue(user_id)):
- yield self.store.delete_e2e_room_keys_version(user_id, version)
+ try:
+ yield self.store.delete_e2e_room_keys_version(user_id, version)
+ except StoreError as e:
+ if e.code == 404:
+ raise NotFoundError("Unknown backup version")
+ else:
+ raise
+
+ @defer.inlineCallbacks
+ def update_version(self, user_id, version, version_info):
+ """Update the info about a given version of the user's backup
+
+ Args:
+ user_id(str): the user whose current backup version we're updating
+ version(str): the backup version we're updating
+ version_info(dict): the new information about the backup
+ Raises:
+ NotFoundError: if the requested backup version doesn't exist
+ Returns:
+ A deferred of an empty dict.
+ """
+ if "version" not in version_info:
+ raise SynapseError(
+ 400,
+ "Missing version in body",
+ Codes.MISSING_PARAM
+ )
+ if version_info["version"] != version:
+ raise SynapseError(
+ 400,
+ "Version in body does not match",
+ Codes.INVALID_PARAM
+ )
+ with (yield self._upload_linearizer.queue(user_id)):
+ try:
+ old_info = yield self.store.get_e2e_room_keys_version_info(
+ user_id, version
+ )
+ except StoreError as e:
+ if e.code == 404:
+ raise NotFoundError("Unknown backup version")
+ else:
+ raise
+ if old_info["algorithm"] != version_info["algorithm"]:
+ raise SynapseError(
+ 400,
+ "Algorithm does not match",
+ Codes.INVALID_PARAM
+ )
+
+ yield self.store.update_e2e_room_keys_version(user_id, version, version_info)
+
+ defer.returnValue({})
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 5ee4d528d2..3c24bf3805 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -168,7 +168,7 @@ class MatrixFederationHttpClient(object):
requests.
"""
- def __init__(self, hs):
+ def __init__(self, hs, tls_client_options_factory):
self.hs = hs
self.signing_key = hs.config.signing_key[0]
self.server_name = hs.hostname
@@ -176,7 +176,7 @@ class MatrixFederationHttpClient(object):
self.agent = MatrixFederationAgent(
hs.get_reactor(),
- hs.tls_client_options_factory,
+ tls_client_options_factory,
)
self.clock = hs.get_clock()
self._store = hs.get_datastore()
diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py
index ab3f1bd21a..220a0de30b 100644
--- a/synapse/rest/client/v2_alpha/room_keys.py
+++ b/synapse/rest/client/v2_alpha/room_keys.py
@@ -380,6 +380,40 @@ class RoomKeysVersionServlet(RestServlet):
)
defer.returnValue((200, {}))
+ @defer.inlineCallbacks
+ def on_PUT(self, request, version):
+ """
+ Update the information about a given version of the user's room_keys backup.
+
+ POST /room_keys/version/12345 HTTP/1.1
+ Content-Type: application/json
+ {
+ "algorithm": "m.megolm_backup.v1",
+ "auth_data": {
+ "public_key": "abcdefg",
+ "signatures": {
+ "ed25519:something": "hijklmnop"
+ }
+ },
+ "version": "42"
+ }
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ {}
+ """
+ requester = yield self.auth.get_user_by_req(request, allow_guest=False)
+ user_id = requester.user.to_string()
+ info = parse_json_object_from_request(request)
+
+ if version is None:
+ raise SynapseError(400, "No version specified to update", Codes.MISSING_PARAM)
+
+ yield self.e2e_room_keys_handler.update_version(
+ user_id, version, info
+ )
+ defer.returnValue((200, {}))
+
def register_servlets(hs, http_server):
RoomKeysServlet(hs).register(http_server)
diff --git a/synapse/server.py b/synapse/server.py
index a2cf8a91cd..8615b67ad4 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -31,6 +31,7 @@ from synapse.api.filtering import Filtering
from synapse.api.ratelimiting import Ratelimiter
from synapse.appservice.api import ApplicationServiceApi
from synapse.appservice.scheduler import ApplicationServiceScheduler
+from synapse.crypto import context_factory
from synapse.crypto.keyring import Keyring
from synapse.events.builder import EventBuilderFactory
from synapse.events.spamcheck import SpamChecker
@@ -367,7 +368,10 @@ class HomeServer(object):
return PusherPool(self)
def build_http_client(self):
- return MatrixFederationHttpClient(self)
+ tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+ self.config
+ )
+ return MatrixFederationHttpClient(self, tls_client_options_factory)
def build_db_pool(self):
name = self.db_config["name"]
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index 091d7116c5..9c21362226 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -66,6 +66,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
)
self.register_background_update_handler(
+ "user_ips_analyze",
+ self._analyze_user_ip,
+ )
+
+ self.register_background_update_handler(
"user_ips_remove_dupes",
self._remove_user_ip_dupes,
)
@@ -109,6 +114,25 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
defer.returnValue(1)
@defer.inlineCallbacks
+ def _analyze_user_ip(self, progress, batch_size):
+ # Background update to analyze user_ips table before we run the
+ # deduplication background update. The table may not have been analyzed
+ # for ages due to the table locks.
+ #
+ # This will lock out the naive upserts to user_ips while it happens, but
+ # the analyze should be quick (28GB table takes ~10s)
+ def user_ips_analyze(txn):
+ txn.execute("ANALYZE user_ips")
+
+ yield self.runInteraction(
+ "user_ips_analyze", user_ips_analyze
+ )
+
+ yield self._end_background_update("user_ips_analyze")
+
+ defer.returnValue(1)
+
+ @defer.inlineCallbacks
def _remove_user_ip_dupes(self, progress, batch_size):
# This works function works by scanning the user_ips table in batches
# based on `last_seen`. For each row in a batch it searches the rest of
@@ -167,12 +191,16 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
clause = "? <= last_seen AND last_seen < ?"
args = (begin_last_seen, end_last_seen)
+ # (Note: The DISTINCT in the inner query is important to ensure that
+ # the COUNT(*) is accurate, otherwise double counting may happen due
+ # to the join effectively being a cross product)
txn.execute(
"""
SELECT user_id, access_token, ip,
- MAX(device_id), MAX(user_agent), MAX(last_seen)
+ MAX(device_id), MAX(user_agent), MAX(last_seen),
+ COUNT(*)
FROM (
- SELECT user_id, access_token, ip
+ SELECT DISTINCT user_id, access_token, ip
FROM user_ips
WHERE {}
) c
@@ -186,7 +214,60 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
# We've got some duplicates
for i in res:
- user_id, access_token, ip, device_id, user_agent, last_seen = i
+ user_id, access_token, ip, device_id, user_agent, last_seen, count = i
+
+ # We want to delete the duplicates so we end up with only a
+ # single row.
+ #
+ # The naive way of doing this would be just to delete all rows
+ # and reinsert a constructed row. However, if there are a lot of
+ # duplicate rows this can cause the table to grow a lot, which
+ # can be problematic in two ways:
+ # 1. If user_ips is already large then this can cause the
+ # table to rapidly grow, potentially filling the disk.
+ # 2. Reinserting a lot of rows can confuse the table
+ # statistics for postgres, causing it to not use the
+ # correct indices for the query above, resulting in a full
+ # table scan. This is incredibly slow for large tables and
+ # can kill database performance. (This seems to mainly
+ # happen for the last query where the clause is simply `? <
+ # last_seen`)
+ #
+ # So instead we want to delete all but *one* of the duplicate
+ # rows. That is hard to do reliably, so we cheat and do a two
+ # step process:
+ # 1. Delete all rows with a last_seen strictly less than the
+ # max last_seen. This hopefully results in deleting all but
+ # one row the majority of the time, but there may be
+ # duplicate last_seen
+ # 2. If multiple rows remain, we fall back to the naive method
+ # and simply delete all rows and reinsert.
+ #
+ # Note that this relies on no new duplicate rows being inserted,
+ # but if that is happening then this entire process is futile
+ # anyway.
+
+ # Do step 1:
+
+ txn.execute(
+ """
+ DELETE FROM user_ips
+ WHERE user_id = ? AND access_token = ? AND ip = ? AND last_seen < ?
+ """,
+ (user_id, access_token, ip, last_seen)
+ )
+ if txn.rowcount == count - 1:
+ # We deleted all but one of the duplicate rows, i.e. there
+ # is exactly one remaining and so there is nothing left to
+ # do.
+ continue
+ elif txn.rowcount >= count:
+ raise Exception(
+ "We deleted more duplicate rows from 'user_ips' than expected",
+ )
+
+ # The previous step didn't delete enough rows, so we fallback to
+ # step 2:
# Drop all the duplicates
txn.execute(
diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py
index 45cebe61d1..9a3aec759e 100644
--- a/synapse/storage/e2e_room_keys.py
+++ b/synapse/storage/e2e_room_keys.py
@@ -298,6 +298,27 @@ class EndToEndRoomKeyStore(SQLBaseStore):
"create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn
)
+ def update_e2e_room_keys_version(self, user_id, version, info):
+ """Update a given backup version
+
+ Args:
+ user_id(str): the user whose backup version we're updating
+ version(str): the version ID of the backup version we're updating
+ info(dict): the new backup version info to store
+ """
+
+ return self._simple_update(
+ table="e2e_room_keys_versions",
+ keyvalues={
+ "user_id": user_id,
+ "version": version,
+ },
+ updatevalues={
+ "auth_data": json.dumps(info["auth_data"]),
+ },
+ desc="update_e2e_room_keys_version"
+ )
+
def delete_e2e_room_keys_version(self, user_id, version=None):
"""Delete a given backup version of the user's room keys.
Doesn't delete their actual key data.
diff --git a/synapse/storage/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/delta/53/user_ips_index.sql
index 4ca346c111..b812c5794f 100644
--- a/synapse/storage/schema/delta/53/user_ips_index.sql
+++ b/synapse/storage/schema/delta/53/user_ips_index.sql
@@ -13,9 +13,13 @@
* limitations under the License.
*/
--- delete duplicates
+ -- analyze user_ips, to help ensure the correct indices are used
INSERT INTO background_updates (update_name, progress_json) VALUES
- ('user_ips_remove_dupes', '{}');
+ ('user_ips_analyze', '{}');
+
+-- delete duplicates
+INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
+ ('user_ips_remove_dupes', '{}', 'user_ips_analyze');
-- add a new unique index to user_ips table
INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
@@ -23,4 +27,4 @@ INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-- drop the old original index
INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
- ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index');
\ No newline at end of file
+ ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index');
|
