diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index e8112d5f05..960e66dbdc 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -64,6 +64,8 @@ class Auth(object):
self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000)
register_cache("cache", "token_cache", self.token_cache)
+ self._account_validity = hs.config.account_validity
+
@defer.inlineCallbacks
def check_from_context(self, room_version, event, context, do_sig_check=True):
prev_state_ids = yield context.get_prev_state_ids(self.store)
@@ -226,6 +228,17 @@ class Auth(object):
token_id = user_info["token_id"]
is_guest = user_info["is_guest"]
+ # Deny the request if the user account has expired.
+ if self._account_validity.enabled:
+ user_id = user.to_string()
+ expiration_ts = yield self.store.get_expiration_ts_for_user(user_id)
+ if expiration_ts is not None and self.clock.time_msec() >= expiration_ts:
+ raise AuthError(
+ 403,
+ "User account has expired",
+ errcode=Codes.EXPIRED_ACCOUNT,
+ )
+
# device_id may not be present if get_user_by_access_token has been
# stubbed out.
device_id = user_info.get("device_id")
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index dd373fa4b8..0860b75905 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2017 Vector Creations Ltd
-# Copyright 2018 New Vector Ltd.
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 0b464834ce..ff89259dec 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd.
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -60,6 +60,7 @@ class Codes(object):
UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
+ EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
class CodeMessageException(RuntimeError):
diff --git a/synapse/api/urls.py b/synapse/api/urls.py
index 8102176653..cb71d80875 100644
--- a/synapse/api/urls.py
+++ b/synapse/api/urls.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd.
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index 1e9e686107..864f1eac48 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -114,7 +114,7 @@ class ClientReaderServer(HomeServer):
KeyChangesServlet(self).register(resource)
VoipRestServlet(self).register(resource)
PushRuleRestServlet(self).register(resource)
- VersionsRestServlet(self).register(resource)
+ VersionsRestServlet().register(resource)
resources.update({
"/_matrix/client": resource,
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 869c028d1f..79be977ea6 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -518,6 +518,7 @@ def run(hs):
uptime = 0
stats["homeserver"] = hs.config.server_name
+ stats["server_context"] = hs.config.server_context
stats["timestamp"] = now
stats["uptime_seconds"] = uptime
version = sys.version_info
@@ -558,7 +559,6 @@ def run(hs):
stats["database_engine"] = hs.get_datastore().database_engine_name
stats["database_server_version"] = hs.get_datastore().get_server_version()
-
logger.info("Reporting stats to matrix.org: %s" % (stats,))
try:
yield hs.get_simple_http_client().put_json(
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index 93d70cff14..342a6ce5fd 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -71,6 +71,12 @@ class EmailConfig(Config):
self.email_notif_from = email_config["notif_from"]
self.email_notif_template_html = email_config["notif_template_html"]
self.email_notif_template_text = email_config["notif_template_text"]
+ self.email_expiry_template_html = email_config.get(
+ "expiry_template_html", "notice_expiry.html",
+ )
+ self.email_expiry_template_text = email_config.get(
+ "expiry_template_text", "notice_expiry.txt",
+ )
template_dir = email_config.get("template_dir")
# we need an absolute path, because we change directory after starting (and
@@ -120,7 +126,7 @@ class EmailConfig(Config):
def default_config(self, config_dir_path, server_name, **kwargs):
return """
- # Enable sending emails for notification events
+ # Enable sending emails for notification events or expiry notices
# Defining a custom URL for Riot is only needed if email notifications
# should contain links to a self-hosted installation of Riot; when set
# the "app_name" setting is ignored.
@@ -142,6 +148,9 @@ class EmailConfig(Config):
# #template_dir: res/templates
# notif_template_html: notif_mail.html
# notif_template_text: notif_mail.txt
+ # # Templates for account expiry notices.
+ # expiry_template_html: notice_expiry.html
+ # expiry_template_text: notice_expiry.txt
# notif_for_new_users: True
# riot_base_url: "http://localhost/riot"
"""
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index f6b2b9ceee..1309bce3ee 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -20,6 +20,29 @@ from synapse.types import RoomAlias
from synapse.util.stringutils import random_string_with_symbols
+class AccountValidityConfig(Config):
+ def __init__(self, config, synapse_config):
+ self.enabled = config.get("enabled", False)
+ self.renew_by_email_enabled = ("renew_at" in config)
+
+ if self.enabled:
+ if "period" in config:
+ self.period = self.parse_duration(config["period"])
+ else:
+ raise ConfigError("'period' is required when using account validity")
+
+ if "renew_at" in config:
+ self.renew_at = self.parse_duration(config["renew_at"])
+
+ if "renew_email_subject" in config:
+ self.renew_email_subject = config["renew_email_subject"]
+ else:
+ self.renew_email_subject = "Renew your %(app)s account"
+
+ if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
+ raise ConfigError("Can't send renewal emails without 'public_baseurl'")
+
+
class RegistrationConfig(Config):
def read_config(self, config):
@@ -31,8 +54,13 @@ class RegistrationConfig(Config):
strtobool(str(config["disable_registration"]))
)
+ self.account_validity = AccountValidityConfig(
+ config.get("account_validity", {}), config,
+ )
+
self.registrations_require_3pid = config.get("registrations_require_3pid", [])
self.allowed_local_3pids = config.get("allowed_local_3pids", [])
+ self.enable_3pid_lookup = config.get("enable_3pid_lookup", True)
self.registration_shared_secret = config.get("registration_shared_secret")
self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
@@ -75,6 +103,32 @@ class RegistrationConfig(Config):
#
#enable_registration: false
+ # Optional account validity configuration. This allows for accounts to be denied
+ # any request after a given period.
+ #
+ # ``enabled`` defines whether the account validity feature is enabled. Defaults
+ # to False.
+ #
+ # ``period`` allows setting the period after which an account is valid
+ # after its registration. When renewing the account, its validity period
+ # will be extended by this amount of time. This parameter is required when using
+ # the account validity feature.
+ #
+ # ``renew_at`` is the amount of time before an account's expiry date at which
+ # Synapse will send an email to the account's email address with a renewal link.
+ # This needs the ``email`` and ``public_baseurl`` configuration sections to be
+ # filled.
+ #
+ # ``renew_email_subject`` is the subject of the email sent out with the renewal
+ # link. ``%%(app)s`` can be used as a placeholder for the ``app_name`` parameter
+ # from the ``email`` section.
+ #
+ #account_validity:
+ # enabled: True
+ # period: 6w
+ # renew_at: 1w
+ # renew_email_subject: "Renew your %%(app)s account"
+
# The user must provide all of the below types of 3PID when registering.
#
#registrations_require_3pid:
@@ -97,6 +151,10 @@ class RegistrationConfig(Config):
# - medium: msisdn
# pattern: '\\+44'
+ # Enable 3PIDs lookup requests to identity servers from this server.
+ #
+ #enable_3pid_lookup: true
+
# If set, allows registration of standard or admin accounts by anyone who
# has the shared secret, even if registration is otherwise disabled.
#
diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py
index 39b9eb29c2..aa6eac271f 100644
--- a/synapse/config/saml2_config.py
+++ b/synapse/config/saml2_config.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd.
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 08e4e45482..cdf1e4d286 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -37,6 +37,7 @@ class ServerConfig(Config):
def read_config(self, config):
self.server_name = config["server_name"]
+ self.server_context = config.get("server_context", None)
try:
parse_and_validate_server_name(self.server_name)
@@ -113,11 +114,13 @@ class ServerConfig(Config):
# FIXME: federation_domain_whitelist needs sytests
self.federation_domain_whitelist = None
federation_domain_whitelist = config.get(
- "federation_domain_whitelist", None
+ "federation_domain_whitelist", None,
)
- # turn the whitelist into a hash for speed of lookup
+
if federation_domain_whitelist is not None:
+ # turn the whitelist into a hash for speed of lookup
self.federation_domain_whitelist = {}
+
for domain in federation_domain_whitelist:
self.federation_domain_whitelist[domain] = True
@@ -484,6 +487,9 @@ class ServerConfig(Config):
#mau_limit_reserved_threepids:
# - medium: 'email'
# address: 'reserved_user@example.com'
+
+ # Used by phonehome stats to group together related servers.
+ #server_context: context
""" % locals()
def read_arguments(self, args):
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index f0014902da..72dd5926f9 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -24,8 +24,10 @@ import six
from unpaddedbase64 import encode_base64
from OpenSSL import crypto
+from twisted.internet._sslverify import Certificate, trustRootFromCertificates
from synapse.config._base import Config, ConfigError
+from synapse.util import glob_to_regex
logger = logging.getLogger(__name__)
@@ -70,6 +72,53 @@ class TlsConfig(Config):
self.tls_fingerprints = list(self._original_tls_fingerprints)
+ # Whether to verify certificates on outbound federation traffic
+ self.federation_verify_certificates = config.get(
+ "federation_verify_certificates", False,
+ )
+
+ # Whitelist of domains to not verify certificates for
+ fed_whitelist_entries = config.get(
+ "federation_certificate_verification_whitelist", [],
+ )
+
+ # Support globs (*) in whitelist values
+ self.federation_certificate_verification_whitelist = []
+ for entry in fed_whitelist_entries:
+ # Convert globs to regex
+ entry_regex = glob_to_regex(entry)
+ self.federation_certificate_verification_whitelist.append(entry_regex)
+
+ # List of custom certificate authorities for federation traffic validation
+ custom_ca_list = config.get(
+ "federation_custom_ca_list", None,
+ )
+
+ # Read in and parse custom CA certificates
+ self.federation_ca_trust_root = None
+ if custom_ca_list is not None:
+ if len(custom_ca_list) == 0:
+ # A trustroot cannot be generated without any CA certificates.
+ # Raise an error if this option has been specified without any
+ # corresponding certificates.
+ raise ConfigError("federation_custom_ca_list specified without "
+ "any certificate files")
+
+ certs = []
+ for ca_file in custom_ca_list:
+ logger.debug("Reading custom CA certificate file: %s", ca_file)
+ content = self.read_file(ca_file)
+
+ # Parse the CA certificates
+ try:
+ cert_base = Certificate.loadPEM(content)
+ certs.append(cert_base)
+ except Exception as e:
+ raise ConfigError("Error parsing custom CA certificate file %s: %s"
+ % (ca_file, e))
+
+ self.federation_ca_trust_root = trustRootFromCertificates(certs)
+
# This config option applies to non-federation HTTP clients
# (e.g. for talking to recaptcha, identity servers, and such)
# It should never be used in production, and is intended for
@@ -99,15 +148,15 @@ class TlsConfig(Config):
try:
with open(self.tls_certificate_file, 'rb') as f:
cert_pem = f.read()
- except Exception:
- logger.exception("Failed to read existing certificate off disk!")
- raise
+ except Exception as e:
+ raise ConfigError("Failed to read existing certificate file %s: %s"
+ % (self.tls_certificate_file, e))
try:
tls_certificate = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
- except Exception:
- logger.exception("Failed to parse existing certificate off disk!")
- raise
+ except Exception as e:
+ raise ConfigError("Failed to parse existing certificate file %s: %s"
+ % (self.tls_certificate_file, e))
if not allow_self_signed:
if tls_certificate.get_subject() == tls_certificate.get_issuer():
@@ -192,6 +241,40 @@ class TlsConfig(Config):
#
#tls_private_key_path: "%(tls_private_key_path)s"
+ # Whether to verify TLS certificates when sending federation traffic.
+ #
+ # This currently defaults to `false`, however this will change in
+ # Synapse 1.0 when valid federation certificates will be required.
+ #
+ #federation_verify_certificates: true
+
+ # Skip federation certificate verification on the following whitelist
+ # of domains.
+ #
+ # This setting should only be used in very specific cases, such as
+ # federation over Tor hidden services and similar. For private networks
+ # of homeservers, you likely want to use a private CA instead.
+ #
+ # Only effective if federation_verify_certicates is `true`.
+ #
+ #federation_certificate_verification_whitelist:
+ # - lon.example.com
+ # - *.domain.com
+ # - *.onion
+
+ # List of custom certificate authorities for federation traffic.
+ #
+ # This setting should only normally be used within a private network of
+ # homeservers.
+ #
+ # Note that this list will replace those that are provided by your
+ # operating environment. Certificates must be in PEM format.
+ #
+ #federation_custom_ca_list:
+ # - myCA1.pem
+ # - myCA2.pem
+ # - myCA3.pem
+
# ACME support: This will configure Synapse to request a valid TLS certificate
# for your configured `server_name` via Let's Encrypt.
#
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 49cbc7098f..59ea087e66 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -18,10 +18,10 @@ import logging
from zope.interface import implementer
from OpenSSL import SSL, crypto
-from twisted.internet._sslverify import _defaultCurveName
+from twisted.internet._sslverify import ClientTLSOptions, _defaultCurveName
from twisted.internet.abstract import isIPAddress, isIPv6Address
from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
-from twisted.internet.ssl import CertificateOptions, ContextFactory
+from twisted.internet.ssl import CertificateOptions, ContextFactory, platformTrust
from twisted.python.failure import Failure
logger = logging.getLogger(__name__)
@@ -90,7 +90,7 @@ def _tolerateErrors(wrapped):
@implementer(IOpenSSLClientConnectionCreator)
-class ClientTLSOptions(object):
+class ClientTLSOptionsNoVerify(object):
"""
Client creator for TLS without certificate identity verification. This is a
copy of twisted.internet._sslverify.ClientTLSOptions with the identity
@@ -127,9 +127,30 @@ class ClientTLSOptionsFactory(object):
to remote servers for federation."""
def __init__(self, config):
- # We don't use config options yet
- self._options = CertificateOptions(verify=False)
+ self._config = config
+ self._options_noverify = CertificateOptions()
+
+ # Check if we're using a custom list of a CA certificates
+ trust_root = config.federation_ca_trust_root
+ if trust_root is None:
+ # Use CA root certs provided by OpenSSL
+ trust_root = platformTrust()
+
+ self._options_verify = CertificateOptions(trustRoot=trust_root)
def get_options(self, host):
# Use _makeContext so that we get a fresh OpenSSL CTX each time.
- return ClientTLSOptions(host, self._options._makeContext())
+
+ # Check if certificate verification has been enabled
+ should_verify = self._config.federation_verify_certificates
+
+ # Check if we've disabled certificate verification for this host
+ if should_verify:
+ for regex in self._config.federation_certificate_verification_whitelist:
+ if regex.match(host):
+ should_verify = False
+ break
+
+ if should_verify:
+ return ClientTLSOptions(host, self._options_verify._makeContext())
+ return ClientTLSOptionsNoVerify(host, self._options_noverify._makeContext())
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 0207cd989a..ed2e994437 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2017, 2018 New Vector Ltd.
+# Copyright 2017, 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ from collections import namedtuple
from six import raise_from
from six.moves import urllib
+import nacl.signing
from signedjson.key import (
decode_verify_key_bytes,
encode_verify_key_base64,
@@ -274,10 +275,6 @@ class Keyring(object):
@defer.inlineCallbacks
def do_iterations():
with Measure(self.clock, "get_server_verify_keys"):
- # dict[str, dict[str, VerifyKey]]: results so far.
- # map server_name -> key_id -> VerifyKey
- merged_results = {}
-
# dict[str, set(str)]: keys to fetch for each server
missing_keys = {}
for verify_request in verify_requests:
@@ -287,29 +284,29 @@ class Keyring(object):
for fn in key_fetch_fns:
results = yield fn(missing_keys.items())
- merged_results.update(results)
# We now need to figure out which verify requests we have keys
# for and which we don't
missing_keys = {}
requests_missing_keys = []
for verify_request in verify_requests:
- server_name = verify_request.server_name
- result_keys = merged_results[server_name]
-
if verify_request.deferred.called:
# We've already called this deferred, which probably
# means that we've already found a key for it.
continue
+ server_name = verify_request.server_name
+
+ # see if any of the keys we got this time are sufficient to
+ # complete this VerifyKeyRequest.
+ result_keys = results.get(server_name, {})
for key_id in verify_request.key_ids:
- if key_id in result_keys:
+ key = result_keys.get(key_id)
+ if key:
with PreserveLoggingContext():
- verify_request.deferred.callback((
- server_name,
- key_id,
- result_keys[key_id],
- ))
+ verify_request.deferred.callback(
+ (server_name, key_id, key)
+ )
break
else:
# The else block is only reached if the loop above
@@ -343,27 +340,24 @@ class Keyring(object):
@defer.inlineCallbacks
def get_keys_from_store(self, server_name_and_key_ids):
"""
-
Args:
- server_name_and_key_ids (list[(str, iterable[str])]):
+ server_name_and_key_ids (iterable(Tuple[str, iterable[str]]):
list of (server_name, iterable[key_id]) tuples to fetch keys for
Returns:
- Deferred: resolves to dict[str, dict[str, VerifyKey]]: map from
+ Deferred: resolves to dict[str, dict[str, VerifyKey|None]]: map from
server_name -> key_id -> VerifyKey
"""
- res = yield logcontext.make_deferred_yieldable(defer.gatherResults(
- [
- run_in_background(
- self.store.get_server_verify_keys,
- server_name, key_ids,
- ).addCallback(lambda ks, server: (server, ks), server_name)
- for server_name, key_ids in server_name_and_key_ids
- ],
- consumeErrors=True,
- ).addErrback(unwrapFirstError))
-
- defer.returnValue(dict(res))
+ keys_to_fetch = (
+ (server_name, key_id)
+ for server_name, key_ids in server_name_and_key_ids
+ for key_id in key_ids
+ )
+ res = yield self.store.get_server_verify_keys(keys_to_fetch)
+ keys = {}
+ for (server_name, key_id), key in res.items():
+ keys.setdefault(server_name, {})[key_id] = key
+ defer.returnValue(keys)
@defer.inlineCallbacks
def get_keys_from_perspectives(self, server_name_and_key_ids):
@@ -494,11 +488,11 @@ class Keyring(object):
)
processed_response = yield self.process_v2_response(
- perspective_name, response, only_from_server=False
+ perspective_name, response
)
+ server_name = response["server_name"]
- for server_name, response_keys in processed_response.items():
- keys.setdefault(server_name, {}).update(response_keys)
+ keys.setdefault(server_name, {}).update(processed_response)
yield logcontext.make_deferred_yieldable(defer.gatherResults(
[
@@ -517,7 +511,7 @@ class Keyring(object):
@defer.inlineCallbacks
def get_server_verify_key_v2_direct(self, server_name, key_ids):
- keys = {}
+ keys = {} # type: dict[str, nacl.signing.VerifyKey]
for requested_key_id in key_ids:
if requested_key_id in keys:
@@ -542,6 +536,11 @@ class Keyring(object):
or server_name not in response[u"signatures"]):
raise KeyLookupError("Key response not signed by remote server")
+ if response["server_name"] != server_name:
+ raise KeyLookupError("Expected a response for server %r not %r" % (
+ server_name, response["server_name"]
+ ))
+
response_keys = yield self.process_v2_response(
from_server=server_name,
requested_ids=[requested_key_id],
@@ -550,24 +549,45 @@ class Keyring(object):
keys.update(response_keys)
- yield logcontext.make_deferred_yieldable(defer.gatherResults(
- [
- run_in_background(
- self.store_keys,
- server_name=key_server_name,
- from_server=server_name,
- verify_keys=verify_keys,
- )
- for key_server_name, verify_keys in keys.items()
- ],
- consumeErrors=True
- ).addErrback(unwrapFirstError))
-
- defer.returnValue(keys)
+ yield self.store_keys(
+ server_name=server_name,
+ from_server=server_name,
+ verify_keys=keys,
+ )
+ defer.returnValue({server_name: keys})
@defer.inlineCallbacks
- def process_v2_response(self, from_server, response_json,
- requested_ids=[], only_from_server=True):
+ def process_v2_response(
+ self, from_server, response_json, requested_ids=[],
+ ):
+ """Parse a 'Server Keys' structure from the result of a /key request
+
+ This is used to parse either the entirety of the response from
+ GET /_matrix/key/v2/server, or a single entry from the list returned by
+ POST /_matrix/key/v2/query.
+
+ Checks that each signature in the response that claims to come from the origin
+ server is valid. (Does not check that there actually is such a signature, for
+ some reason.)
+
+ Stores the json in server_keys_json so that it can be used for future responses
+ to /_matrix/key/v2/query.
+
+ Args:
+ from_server (str): the name of the server producing this result: either
+ the origin server for a /_matrix/key/v2/server request, or the notary
+ for a /_matrix/key/v2/query.
+
+ response_json (dict): the json-decoded Server Keys response object
+
+ requested_ids (iterable[str]): a list of the key IDs that were requested.
+ We will store the json for these key ids as well as any that are
+ actually in the response
+
+ Returns:
+ Deferred[dict[str, nacl.signing.VerifyKey]]:
+ map from key_id to key object
+ """
time_now_ms = self.clock.time_msec()
response_keys = {}
verify_keys = {}
@@ -589,15 +609,7 @@ class Keyring(object):
verify_key.time_added = time_now_ms
old_verify_keys[key_id] = verify_key
- results = {}
server_name = response_json["server_name"]
- if only_from_server:
- if server_name != from_server:
- raise KeyLookupError(
- "Expected a response for server %r not %r" % (
- from_server, server_name
- )
- )
for key_id in response_json["signatures"].get(server_name, {}):
if key_id not in response_json["verify_keys"]:
raise KeyLookupError(
@@ -633,7 +645,7 @@ class Keyring(object):
self.store.store_server_keys_json,
server_name=server_name,
key_id=key_id,
- from_server=server_name,
+ from_server=from_server,
ts_now_ms=time_now_ms,
ts_expires_ms=ts_valid_until_ms,
key_json_bytes=signed_key_json_bytes,
@@ -643,9 +655,7 @@ class Keyring(object):
consumeErrors=True,
).addErrback(unwrapFirstError))
- results[server_name] = response_keys
-
- defer.returnValue(results)
+ defer.returnValue(response_keys)
def store_keys(self, server_name, from_server, verify_keys):
"""Store a collection of verify keys for a given server
diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py
index 633e068eb8..6058077f75 100644
--- a/synapse/events/spamcheck.py
+++ b/synapse/events/spamcheck.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2017 New Vector Ltd.
+# Copyright 2017 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py
new file mode 100644
index 0000000000..261446517d
--- /dev/null
+++ b/synapse/handlers/account_validity.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import email.mime.multipart
+import email.utils
+import logging
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+from twisted.internet import defer
+
+from synapse.api.errors import StoreError
+from synapse.types import UserID
+from synapse.util import stringutils
+from synapse.util.logcontext import make_deferred_yieldable
+
+try:
+ from synapse.push.mailer import load_jinja2_templates
+except ImportError:
+ load_jinja2_templates = None
+
+logger = logging.getLogger(__name__)
+
+
+class AccountValidityHandler(object):
+ def __init__(self, hs):
+ self.hs = hs
+ self.store = self.hs.get_datastore()
+ self.sendmail = self.hs.get_sendmail()
+ self.clock = self.hs.get_clock()
+
+ self._account_validity = self.hs.config.account_validity
+
+ if self._account_validity.renew_by_email_enabled and load_jinja2_templates:
+ # Don't do email-specific configuration if renewal by email is disabled.
+ try:
+ app_name = self.hs.config.email_app_name
+
+ self._subject = self._account_validity.renew_email_subject % {
+ "app": app_name,
+ }
+
+ self._from_string = self.hs.config.email_notif_from % {
+ "app": app_name,
+ }
+ except Exception:
+ # If substitution failed, fall back to the bare strings.
+ self._subject = self._account_validity.renew_email_subject
+ self._from_string = self.hs.config.email_notif_from
+
+ self._raw_from = email.utils.parseaddr(self._from_string)[1]
+
+ self._template_html, self._template_text = load_jinja2_templates(
+ config=self.hs.config,
+ template_html_name=self.hs.config.email_expiry_template_html,
+ template_text_name=self.hs.config.email_expiry_template_text,
+ )
+
+ # Check the renewal emails to send and send them every 30min.
+ self.clock.looping_call(
+ self.send_renewal_emails,
+ 30 * 60 * 1000,
+ )
+
+ @defer.inlineCallbacks
+ def send_renewal_emails(self):
+ """Gets the list of users whose account is expiring in the amount of time
+ configured in the ``renew_at`` parameter from the ``account_validity``
+ configuration, and sends renewal emails to all of these users as long as they
+ have an email 3PID attached to their account.
+ """
+ expiring_users = yield self.store.get_users_expiring_soon()
+
+ if expiring_users:
+ for user in expiring_users:
+ yield self._send_renewal_email(
+ user_id=user["user_id"],
+ expiration_ts=user["expiration_ts_ms"],
+ )
+
+ @defer.inlineCallbacks
+ def send_renewal_email_to_user(self, user_id):
+ expiration_ts = yield self.store.get_expiration_ts_for_user(user_id)
+ yield self._send_renewal_email(user_id, expiration_ts)
+
+ @defer.inlineCallbacks
+ def _send_renewal_email(self, user_id, expiration_ts):
+ """Sends out a renewal email to every email address attached to the given user
+ with a unique link allowing them to renew their account.
+
+ Args:
+ user_id (str): ID of the user to send email(s) to.
+ expiration_ts (int): Timestamp in milliseconds for the expiration date of
+ this user's account (used in the email templates).
+ """
+ addresses = yield self._get_email_addresses_for_user(user_id)
+
+ # Stop right here if the user doesn't have at least one email address.
+ # In this case, they will have to ask their server admin to renew their
+ # account manually.
+ if not addresses:
+ return
+
+ try:
+ user_display_name = yield self.store.get_profile_displayname(
+ UserID.from_string(user_id).localpart
+ )
+ if user_display_name is None:
+ user_display_name = user_id
+ except StoreError:
+ user_display_name = user_id
+
+ renewal_token = yield self._get_renewal_token(user_id)
+ url = "%s_matrix/client/unstable/account_validity/renew?token=%s" % (
+ self.hs.config.public_baseurl,
+ renewal_token,
+ )
+
+ template_vars = {
+ "display_name": user_display_name,
+ "expiration_ts": expiration_ts,
+ "url": url,
+ }
+
+ html_text = self._template_html.render(**template_vars)
+ html_part = MIMEText(html_text, "html", "utf8")
+
+ plain_text = self._template_text.render(**template_vars)
+ text_part = MIMEText(plain_text, "plain", "utf8")
+
+ for address in addresses:
+ raw_to = email.utils.parseaddr(address)[1]
+
+ multipart_msg = MIMEMultipart('alternative')
+ multipart_msg['Subject'] = self._subject
+ multipart_msg['From'] = self._from_string
+ multipart_msg['To'] = address
+ multipart_msg['Date'] = email.utils.formatdate()
+ multipart_msg['Message-ID'] = email.utils.make_msgid()
+ multipart_msg.attach(text_part)
+ multipart_msg.attach(html_part)
+
+ logger.info("Sending renewal email to %s", address)
+
+ yield make_deferred_yieldable(self.sendmail(
+ self.hs.config.email_smtp_host,
+ self._raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
+ reactor=self.hs.get_reactor(),
+ port=self.hs.config.email_smtp_port,
+ requireAuthentication=self.hs.config.email_smtp_user is not None,
+ username=self.hs.config.email_smtp_user,
+ password=self.hs.config.email_smtp_pass,
+ requireTransportSecurity=self.hs.config.require_transport_security
+ ))
+
+ yield self.store.set_renewal_mail_status(
+ user_id=user_id,
+ email_sent=True,
+ )
+
+ @defer.inlineCallbacks
+ def _get_email_addresses_for_user(self, user_id):
+ """Retrieve the list of email addresses attached to a user's account.
+
+ Args:
+ user_id (str): ID of the user to lookup email addresses for.
+
+ Returns:
+ defer.Deferred[list[str]]: Email addresses for this account.
+ """
+ threepids = yield self.store.user_get_threepids(user_id)
+
+ addresses = []
+ for threepid in threepids:
+ if threepid["medium"] == "email":
+ addresses.append(threepid["address"])
+
+ defer.returnValue(addresses)
+
+ @defer.inlineCallbacks
+ def _get_renewal_token(self, user_id):
+ """Generates a 32-byte long random string that will be inserted into the
+ user's renewal email's unique link, then saves it into the database.
+
+ Args:
+ user_id (str): ID of the user to generate a string for.
+
+ Returns:
+ defer.Deferred[str]: The generated string.
+
+ Raises:
+ StoreError(500): Couldn't generate a unique string after 5 attempts.
+ """
+ attempts = 0
+ while attempts < 5:
+ try:
+ renewal_token = stringutils.random_string(32)
+ yield self.store.set_renewal_token_for_user(user_id, renewal_token)
+ defer.returnValue(renewal_token)
+ except StoreError:
+ attempts += 1
+ raise StoreError(500, "Couldn't generate a unique string as refresh string.")
+
+ @defer.inlineCallbacks
+ def renew_account(self, renewal_token):
+ """Renews the account attached to a given renewal token by pushing back the
+ expiration date by the current validity period in the server's configuration.
+
+ Args:
+ renewal_token (str): Token sent with the renewal request.
+ """
+ user_id = yield self.store.get_user_from_renewal_token(renewal_token)
+ logger.debug("Renewing an account for user %s", user_id)
+ yield self.renew_account_for_user(user_id)
+
+ @defer.inlineCallbacks
+ def renew_account_for_user(self, user_id, expiration_ts=None, email_sent=False):
+ """Renews the account attached to a given user by pushing back the
+ expiration date by the current validity period in the server's
+ configuration.
+
+ Args:
+ renewal_token (str): Token sent with the renewal request.
+ expiration_ts (int): New expiration date. Defaults to now + validity period.
+ email_sent (bool): Whether an email has been sent for this validity period.
+ Defaults to False.
+
+ Returns:
+ defer.Deferred[int]: New expiration date for this account, as a timestamp
+ in milliseconds since epoch.
+ """
+ if expiration_ts is None:
+ expiration_ts = self.clock.time_msec() + self._account_validity.period
+
+ yield self.store.set_account_validity_for_user(
+ user_id=user_id,
+ expiration_ts=expiration_ts,
+ email_sent=email_sent,
+ )
+
+ defer.returnValue(expiration_ts)
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index bd1285b15c..59d53f1050 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -828,6 +828,11 @@ class PresenceHandler(object):
if typ != EventTypes.Member:
continue
+ if event_id is None:
+ # state has been deleted, so this is not a join. We only care about
+ # joins.
+ continue
+
event = yield self.store.get_event(event_id)
if event.content.get("membership") != Membership.JOIN:
# We only care about joins
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 53e0103b5b..ad3df7cc7d 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -72,6 +72,7 @@ class RoomMemberHandler(object):
self.clock = hs.get_clock()
self.spam_checker = hs.get_spam_checker()
self._server_notices_mxid = self.config.server_notices_mxid
+ self._enable_lookup = hs.config.enable_3pid_lookup
# This is only used to get at ratelimit function, and
# maybe_kick_guest_users. It's fine there are multiple of these as
@@ -748,6 +749,10 @@ class RoomMemberHandler(object):
Returns:
str: the matrix ID of the 3pid, or None if it is not recognized.
"""
+ if not self._enable_lookup:
+ raise SynapseError(
+ 403, "Looking up third-party identifiers is denied from this server",
+ )
try:
data = yield self.simple_http_client.get_json(
"%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server,),
diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py
index 1334c630cc..b4cbe97b41 100644
--- a/synapse/http/federation/matrix_federation_agent.py
+++ b/synapse/http/federation/matrix_federation_agent.py
@@ -149,7 +149,7 @@ class MatrixFederationAgent(object):
tls_options = None
else:
tls_options = self._tls_client_options_factory.get_options(
- res.tls_server_name.decode("ascii")
+ res.tls_server_name.decode("ascii"),
)
# make sure that the Host header is set correctly
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 1eb5be0957..c269bcf4a4 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -521,11 +521,11 @@ def format_ts_filter(value, format):
return time.strftime(format, time.localtime(value / 1000))
-def load_jinja2_templates(config):
+def load_jinja2_templates(config, template_html_name, template_text_name):
"""Load the jinja2 email templates from disk
Returns:
- (notif_template_html, notif_template_text)
+ (template_html, template_text)
"""
logger.info("loading email templates from '%s'", config.email_template_dir)
loader = jinja2.FileSystemLoader(config.email_template_dir)
@@ -533,14 +533,10 @@ def load_jinja2_templates(config):
env.filters["format_ts"] = format_ts_filter
env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
- notif_template_html = env.get_template(
- config.email_notif_template_html
- )
- notif_template_text = env.get_template(
- config.email_notif_template_text
- )
+ template_html = env.get_template(template_html_name)
+ template_text = env.get_template(template_text_name)
- return notif_template_html, notif_template_text
+ return template_html, template_text
def _create_mxc_to_http_filter(config):
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
index b33f2a357b..14bc7823cf 100644
--- a/synapse/push/pusher.py
+++ b/synapse/push/pusher.py
@@ -44,7 +44,11 @@ class PusherFactory(object):
if hs.config.email_enable_notifs:
self.mailers = {} # app_name -> Mailer
- templates = load_jinja2_templates(hs.config)
+ templates = load_jinja2_templates(
+ config=hs.config,
+ template_html_name=hs.config.email_notif_template_html,
+ template_text_name=hs.config.email_notif_template_text,
+ )
self.notif_template_html, self.notif_template_text = templates
self.pusher_types["email"] = self._create_email_pusher
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index f71e21ff4d..779f36dbed 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -74,7 +74,9 @@ REQUIREMENTS = [
CONDITIONAL_REQUIREMENTS = {
"email.enable_notifs": ["Jinja2>=2.9", "bleach>=1.4.2"],
"matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"],
- "postgres": ["psycopg2>=2.6"],
+
+ # we use execute_batch, which arrived in psycopg 2.7.
+ "postgres": ["psycopg2>=2.7"],
# ConsentResource uses select_autoescape, which arrived in jinja 2.9
"resources.consent": ["Jinja2>=2.9"],
@@ -84,18 +86,22 @@ CONDITIONAL_REQUIREMENTS = {
"acme": ["txacme>=0.9.2"],
"saml2": ["pysaml2>=4.5.0"],
+ "systemd": ["systemd-python>=231"],
"url_preview": ["lxml>=3.5.0"],
"test": ["mock>=2.0", "parameterized"],
"sentry": ["sentry-sdk>=0.7.2"],
}
+ALL_OPTIONAL_REQUIREMENTS = set()
-def list_requirements():
- deps = set(REQUIREMENTS)
- for opt in CONDITIONAL_REQUIREMENTS.values():
- deps = set(opt) | deps
+for name, optional_deps in CONDITIONAL_REQUIREMENTS.items():
+ # Exclude systemd as it's a system-based requirement.
+ if name not in ["systemd"]:
+ ALL_OPTIONAL_REQUIREMENTS = set(optional_deps) | ALL_OPTIONAL_REQUIREMENTS
- return list(deps)
+
+def list_requirements():
+ return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
class DependencyException(Exception):
diff --git a/synapse/replication/slave/storage/keys.py b/synapse/replication/slave/storage/keys.py
index 8032f53fec..cc6f7f009f 100644
--- a/synapse/replication/slave/storage/keys.py
+++ b/synapse/replication/slave/storage/keys.py
@@ -13,22 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from synapse.storage import DataStore
-from synapse.storage.keys import KeyStore
+from synapse.storage import KeyStore
-from ._base import BaseSlavedStore, __func__
+# KeyStore isn't really safe to use from a worker, but for now we do so and hope that
+# the races it creates aren't too bad.
-
-class SlavedKeyStore(BaseSlavedStore):
- _get_server_verify_key = KeyStore.__dict__[
- "_get_server_verify_key"
- ]
-
- get_server_verify_keys = __func__(DataStore.get_server_verify_keys)
- store_server_verify_key = __func__(DataStore.store_server_verify_key)
-
- get_server_certificate = __func__(DataStore.get_server_certificate)
- store_server_certificate = __func__(DataStore.store_server_certificate)
-
- get_server_keys_json = __func__(DataStore.get_server_keys_json)
- store_server_keys_json = __func__(DataStore.store_server_keys_json)
+SlavedKeyStore = KeyStore
diff --git a/synapse/res/templates/mail-expiry.css b/synapse/res/templates/mail-expiry.css
new file mode 100644
index 0000000000..3dea486467
--- /dev/null
+++ b/synapse/res/templates/mail-expiry.css
@@ -0,0 +1,4 @@
+.noticetext {
+ margin-top: 10px;
+ margin-bottom: 10px;
+}
diff --git a/synapse/res/templates/notice_expiry.html b/synapse/res/templates/notice_expiry.html
new file mode 100644
index 0000000000..f0d7c66e1b
--- /dev/null
+++ b/synapse/res/templates/notice_expiry.html
@@ -0,0 +1,43 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <style type="text/css">
+ {% include 'mail.css' without context %}
+ {% include "mail-%s.css" % app_name ignore missing without context %}
+ {% include 'mail-expiry.css' without context %}
+ </style>
+ </head>
+ <body>
+ <table id="page">
+ <tr>
+ <td> </td>
+ <td id="inner">
+ <table class="header">
+ <tr>
+ <td>
+ <div class="salutation">Hi {{ display_name }},</div>
+ </td>
+ <td class="logo">
+ {% if app_name == "Riot" %}
+ <img src="http://riot.im/img/external/riot-logo-email.png" width="83" height="83" alt="[Riot]"/>
+ {% elif app_name == "Vector" %}
+ <img src="http://matrix.org/img/vector-logo-email.png" width="64" height="83" alt="[Vector]"/>
+ {% else %}
+ <img src="http://matrix.org/img/matrix-120x51.png" width="120" height="51" alt="[matrix]"/>
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <div class="noticetext">Your account will expire on {{ expiration_ts|format_ts("%d-%m-%Y") }}. This means that you will lose access to your account after this date.</div>
+ <div class="noticetext">To extend the validity of your account, please click on the link bellow (or copy and paste it into a new browser tab):</div>
+ <div class="noticetext"><a href="{{ url }}">{{ url }}</a></div>
+ </td>
+ </tr>
+ </table>
+ </td>
+ <td> </td>
+ </tr>
+ </table>
+ </body>
+</html>
diff --git a/synapse/res/templates/notice_expiry.txt b/synapse/res/templates/notice_expiry.txt
new file mode 100644
index 0000000000..41f1c4279c
--- /dev/null
+++ b/synapse/res/templates/notice_expiry.txt
@@ -0,0 +1,7 @@
+Hi {{ display_name }},
+
+Your account will expire on {{ expiration_ts|format_ts("%d-%m-%Y") }}. This means that you will lose access to your account after this date.
+
+To extend the validity of your account, please click on the link bellow (or copy and paste it to a new browser tab):
+
+{{ url }}
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 91f5247d52..a66885d349 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -33,6 +33,7 @@ from synapse.rest.client.v1 import (
from synapse.rest.client.v2_alpha import (
account,
account_data,
+ account_validity,
auth,
capabilities,
devices,
@@ -109,3 +110,4 @@ class ClientRestResource(JsonResource):
groups.register_servlets(hs, client_resource)
room_upgrade_rest_servlet.register_servlets(hs, client_resource)
capabilities.register_servlets(hs, client_resource)
+ account_validity.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 7d7a75fc30..0a1e233b23 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -809,6 +809,44 @@ class DeleteGroupAdminRestServlet(ClientV1RestServlet):
defer.returnValue((200, {}))
+class AccountValidityRenewServlet(ClientV1RestServlet):
+ PATTERNS = client_path_patterns("/admin/account_validity/validity$")
+
+ def __init__(self, hs):
+ """
+ Args:
+ hs (synapse.server.HomeServer): server
+ """
+ super(AccountValidityRenewServlet, self).__init__(hs)
+
+ self.hs = hs
+ self.account_activity_handler = hs.get_account_validity_handler()
+ self.auth = hs.get_auth()
+
+ @defer.inlineCallbacks
+ def on_POST(self, request):
+ requester = yield self.auth.get_user_by_req(request)
+ is_admin = yield self.auth.is_server_admin(requester.user)
+
+ if not is_admin:
+ raise AuthError(403, "You are not a server admin")
+
+ body = parse_json_object_from_request(request)
+
+ if "user_id" not in body:
+ raise SynapseError(400, "Missing property 'user_id' in the request body")
+
+ expiration_ts = yield self.account_activity_handler.renew_account_for_user(
+ body["user_id"], body.get("expiration_ts"),
+ not body.get("enable_renewal_emails", True),
+ )
+
+ res = {
+ "expiration_ts": expiration_ts,
+ }
+ defer.returnValue((200, res))
+
+
def register_servlets(hs, http_server):
WhoisRestServlet(hs).register(http_server)
PurgeMediaCacheRestServlet(hs).register(http_server)
@@ -825,3 +863,4 @@ def register_servlets(hs, http_server):
UserRegisterServlet(hs).register(http_server)
VersionServlet(hs).register(http_server)
DeleteGroupAdminRestServlet(hs).register(http_server)
+ AccountValidityRenewServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/account_validity.py b/synapse/rest/client/v2_alpha/account_validity.py
new file mode 100644
index 0000000000..fc8dbeb617
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/account_validity.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.errors import AuthError, SynapseError
+from synapse.http.server import finish_request
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_patterns
+
+logger = logging.getLogger(__name__)
+
+
+class AccountValidityRenewServlet(RestServlet):
+ PATTERNS = client_v2_patterns("/account_validity/renew$")
+ SUCCESS_HTML = b"<html><body>Your account has been successfully renewed.</body><html>"
+
+ def __init__(self, hs):
+ """
+ Args:
+ hs (synapse.server.HomeServer): server
+ """
+ super(AccountValidityRenewServlet, self).__init__()
+
+ self.hs = hs
+ self.account_activity_handler = hs.get_account_validity_handler()
+ self.auth = hs.get_auth()
+
+ @defer.inlineCallbacks
+ def on_GET(self, request):
+ if b"token" not in request.args:
+ raise SynapseError(400, "Missing renewal token")
+ renewal_token = request.args[b"token"][0]
+
+ yield self.account_activity_handler.renew_account(renewal_token.decode('utf8'))
+
+ request.setResponseCode(200)
+ request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+ request.setHeader(b"Content-Length", b"%d" % (
+ len(AccountValidityRenewServlet.SUCCESS_HTML),
+ ))
+ request.write(AccountValidityRenewServlet.SUCCESS_HTML)
+ finish_request(request)
+ defer.returnValue(None)
+
+
+class AccountValiditySendMailServlet(RestServlet):
+ PATTERNS = client_v2_patterns("/account_validity/send_mail$")
+
+ def __init__(self, hs):
+ """
+ Args:
+ hs (synapse.server.HomeServer): server
+ """
+ super(AccountValiditySendMailServlet, self).__init__()
+
+ self.hs = hs
+ self.account_activity_handler = hs.get_account_validity_handler()
+ self.auth = hs.get_auth()
+ self.account_validity = self.hs.config.account_validity
+
+ @defer.inlineCallbacks
+ def on_POST(self, request):
+ if not self.account_validity.renew_by_email_enabled:
+ raise AuthError(403, "Account renewal via email is disabled on this server.")
+
+ requester = yield self.auth.get_user_by_req(request)
+ user_id = requester.user.to_string()
+ yield self.account_activity_handler.send_renewal_email_to_user(user_id)
+
+ defer.returnValue((200, {}))
+
+
+def register_servlets(hs, http_server):
+ AccountValidityRenewServlet(hs).register(http_server)
+ AccountValiditySendMailServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 6d235262c8..dc3e265bcd 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -391,6 +391,13 @@ class RegisterRestServlet(RestServlet):
# the user-facing checks will probably already have happened in
# /register/email/requestToken when we requested a 3pid, but that's not
# guaranteed.
+ #
+ # Also check that we're not trying to register a 3pid that's already
+ # been registered.
+ #
+ # This has probably happened in /register/email/requestToken as well,
+ # but if a user hits this endpoint twice then clicks on each link from
+ # the two activation emails, they would register the same 3pid twice.
if auth_result:
for login_type in [LoginType.EMAIL_IDENTITY, LoginType.MSISDN]:
@@ -406,6 +413,17 @@ class RegisterRestServlet(RestServlet):
Codes.THREEPID_DENIED,
)
+ existingUid = yield self.store.get_user_id_by_threepid(
+ medium, address,
+ )
+
+ if existingUid is not None:
+ raise SynapseError(
+ 400,
+ "%s is already in use" % medium,
+ Codes.THREEPID_IN_USE,
+ )
+
if registered_user_id is not None:
logger.info(
"Already registered user ID %r for this session",
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 953d89bd82..2dcc8f74d6 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2019 New Vector Ltd.
+# Copyright 2019 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -191,6 +191,10 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam
# in that case.
logger.warning("Failed to write to consumer: %s %s", type(e), e)
+ # Unregister the producer, if it has one, so Twisted doesn't complain
+ if request.producer:
+ request.unregisterProducer()
+
finish_request(request)
diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py
index c0a4ae93e5..a7fa4f39af 100644
--- a/synapse/rest/well_known.py
+++ b/synapse/rest/well_known.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd.
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -68,6 +68,6 @@ class WellKnownResource(Resource):
request.setHeader(b"Content-Type", b"text/plain")
return b'.well-known not available'
- logger.error("returning: %s", r)
+ logger.debug("returning: %s", r)
request.setHeader(b"Content-Type", b"application/json")
return json.dumps(r).encode("utf-8")
diff --git a/synapse/server.py b/synapse/server.py
index dc8f1ccb8c..8c30ac2fa5 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -47,6 +47,7 @@ from synapse.federation.transport.client import TransportLayerClient
from synapse.groups.attestations import GroupAttestationSigning, GroupAttestionRenewer
from synapse.groups.groups_server import GroupsServerHandler
from synapse.handlers import Handlers
+from synapse.handlers.account_validity import AccountValidityHandler
from synapse.handlers.acme import AcmeHandler
from synapse.handlers.appservice import ApplicationServicesHandler
from synapse.handlers.auth import AuthHandler, MacaroonGenerator
@@ -183,6 +184,7 @@ class HomeServer(object):
'room_context_handler',
'sendmail',
'registration_handler',
+ 'account_validity_handler',
]
REQUIRED_ON_MASTER_STARTUP = [
@@ -506,6 +508,9 @@ class HomeServer(object):
def build_registration_handler(self):
return RegistrationHandler(self)
+ def build_account_validity_handler(self):
+ return AccountValidityHandler(self)
+
def remove_pusher(self, app_id, push_key, user_id):
return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index dfda39bbe0..7a7f841c6c 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1179,14 +1179,10 @@ class EventsStore(
"events",
"event_auth",
"event_json",
- "event_content_hashes",
- "event_destinations",
- "event_edge_hashes",
"event_edges",
"event_forward_extremities",
"event_reference_hashes",
"event_search",
- "event_signatures",
"event_to_state_groups",
"guest_access",
"history_visibility",
@@ -1857,16 +1853,12 @@ class EventsStore(
# Tables that should be pruned:
# event_auth
# event_backward_extremities
- # event_content_hashes
- # event_destinations
- # event_edge_hashes
# event_edges
# event_forward_extremities
# event_json
# event_push_actions
# event_reference_hashes
# event_search
- # event_signatures
# event_to_state_groups
# events
# rejections
@@ -2065,14 +2057,10 @@ class EventsStore(
"events",
"event_json",
"event_auth",
- "event_content_hashes",
- "event_destinations",
- "event_edge_hashes",
"event_edges",
"event_forward_extremities",
"event_reference_hashes",
"event_search",
- "event_signatures",
"rejections",
):
logger.info("[purge] removing events from %s", table)
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 030cd1e5a3..7036541792 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,17 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import hashlib
+import itertools
import logging
import six
from signedjson.key import decode_verify_key_bytes
-import OpenSSL
-from twisted.internet import defer
-
-from synapse.util.caches.descriptors import cachedInlineCallbacks
+from synapse.util import batch_iter
+from synapse.util.caches.descriptors import cached, cachedList
from ._base import SQLBaseStore
@@ -38,83 +37,52 @@ else:
class KeyStore(SQLBaseStore):
- """Persistence for signature verification keys and tls X.509 certificates
+ """Persistence for signature verification keys
"""
- @defer.inlineCallbacks
- def get_server_certificate(self, server_name):
- """Retrieve the TLS X.509 certificate for the given server
+ @cached()
+ def _get_server_verify_key(self, server_name_and_key_id):
+ raise NotImplementedError()
+
+ @cachedList(
+ cached_method_name="_get_server_verify_key", list_name="server_name_and_key_ids"
+ )
+ def get_server_verify_keys(self, server_name_and_key_ids):
+ """
Args:
- server_name (bytes): The name of the server.
+ server_name_and_key_ids (iterable[Tuple[str, str]]):
+ iterable of (server_name, key-id) tuples to fetch keys for
+
Returns:
- (OpenSSL.crypto.X509): The tls certificate.
+ Deferred: resolves to dict[Tuple[str, str], VerifyKey|None]:
+ map from (server_name, key_id) -> VerifyKey, or None if the key is
+ unknown
"""
- tls_certificate_bytes, = yield self._simple_select_one(
- table="server_tls_certificates",
- keyvalues={"server_name": server_name},
- retcols=("tls_certificate",),
- desc="get_server_certificate",
- )
- tls_certificate = OpenSSL.crypto.load_certificate(
- OpenSSL.crypto.FILETYPE_ASN1, tls_certificate_bytes
- )
- defer.returnValue(tls_certificate)
+ keys = {}
- def store_server_certificate(
- self, server_name, from_server, time_now_ms, tls_certificate
- ):
- """Stores the TLS X.509 certificate for the given server
- Args:
- server_name (str): The name of the server.
- from_server (str): Where the certificate was looked up
- time_now_ms (int): The time now in milliseconds
- tls_certificate (OpenSSL.crypto.X509): The X.509 certificate.
- """
- tls_certificate_bytes = OpenSSL.crypto.dump_certificate(
- OpenSSL.crypto.FILETYPE_ASN1, tls_certificate
- )
- fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest()
- return self._simple_upsert(
- table="server_tls_certificates",
- keyvalues={"server_name": server_name, "fingerprint": fingerprint},
- values={
- "from_server": from_server,
- "ts_added_ms": time_now_ms,
- "tls_certificate": db_binary_type(tls_certificate_bytes),
- },
- desc="store_server_certificate",
- )
+ def _get_keys(txn, batch):
+ """Processes a batch of keys to fetch, and adds the result to `keys`."""
- @cachedInlineCallbacks()
- def _get_server_verify_key(self, server_name, key_id):
- verify_key_bytes = yield self._simple_select_one_onecol(
- table="server_signature_keys",
- keyvalues={"server_name": server_name, "key_id": key_id},
- retcol="verify_key",
- desc="_get_server_verify_key",
- allow_none=True,
- )
+ # batch_iter always returns tuples so it's safe to do len(batch)
+ sql = (
+ "SELECT server_name, key_id, verify_key FROM server_signature_keys "
+ "WHERE 1=0"
+ ) + " OR (server_name=? AND key_id=?)" * len(batch)
- if verify_key_bytes:
- defer.returnValue(decode_verify_key_bytes(key_id, bytes(verify_key_bytes)))
+ txn.execute(sql, tuple(itertools.chain.from_iterable(batch)))
- @defer.inlineCallbacks
- def get_server_verify_keys(self, server_name, key_ids):
- """Retrieve the NACL verification key for a given server for the given
- key_ids
- Args:
- server_name (str): The name of the server.
- key_ids (iterable[str]): key_ids to try and look up.
- Returns:
- Deferred: resolves to dict[str, VerifyKey]: map from
- key_id to verification key.
- """
- keys = {}
- for key_id in key_ids:
- key = yield self._get_server_verify_key(server_name, key_id)
- if key:
- keys[key_id] = key
- defer.returnValue(keys)
+ for row in txn:
+ server_name, key_id, key_bytes = row
+ keys[(server_name, key_id)] = decode_verify_key_bytes(
+ key_id, bytes(key_bytes)
+ )
+
+ def _txn(txn):
+ for batch in batch_iter(server_name_and_key_ids, 50):
+ _get_keys(txn, batch)
+ return keys
+
+ return self.runInteraction("get_server_verify_keys", _txn)
def store_server_verify_key(
self, server_name, from_server, time_now_ms, verify_key
@@ -140,8 +108,11 @@ class KeyStore(SQLBaseStore):
"verify_key": db_binary_type(verify_key.encode()),
},
)
+ # invalidate takes a tuple corresponding to the params of
+ # _get_server_verify_key. _get_server_verify_key only takes one
+ # param, which is itself the 2-tuple (server_name, key_id).
txn.call_after(
- self._get_server_verify_key.invalidate, (server_name, key_id)
+ self._get_server_verify_key.invalidate, ((server_name, key_id),)
)
return self.runInteraction("store_server_verify_key", _txn)
@@ -188,8 +159,8 @@ class KeyStore(SQLBaseStore):
Args:
server_keys (list): List of (server_name, key_id, source) triplets.
Returns:
- Dict mapping (server_name, key_id, source) triplets to dicts with
- "ts_valid_until_ms" and "key_json" keys.
+ Deferred[dict[Tuple[str, str, str|None], list[dict]]]:
+ Dict mapping (server_name, key_id, source) triplets to lists of dicts
"""
def _get_server_keys_json_txn(txn):
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index e30b86c346..03a06a83d6 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -32,6 +32,7 @@ class RegistrationWorkerStore(SQLBaseStore):
super(RegistrationWorkerStore, self).__init__(db_conn, hs)
self.config = hs.config
+ self.clock = hs.get_clock()
@cached()
def get_user_by_id(self, user_id):
@@ -86,6 +87,162 @@ class RegistrationWorkerStore(SQLBaseStore):
"get_user_by_access_token", self._query_for_auth, token
)
+ @cachedInlineCallbacks()
+ def get_expiration_ts_for_user(self, user_id):
+ """Get the expiration timestamp for the account bearing a given user ID.
+
+ Args:
+ user_id (str): The ID of the user.
+ Returns:
+ defer.Deferred: None, if the account has no expiration timestamp,
+ otherwise int representation of the timestamp (as a number of
+ milliseconds since epoch).
+ """
+ res = yield self._simple_select_one_onecol(
+ table="account_validity",
+ keyvalues={"user_id": user_id},
+ retcol="expiration_ts_ms",
+ allow_none=True,
+ desc="get_expiration_ts_for_user",
+ )
+ defer.returnValue(res)
+
+ @defer.inlineCallbacks
+ def set_account_validity_for_user(self, user_id, expiration_ts, email_sent,
+ renewal_token=None):
+ """Updates the account validity properties of the given account, with the
+ given values.
+
+ Args:
+ user_id (str): ID of the account to update properties for.
+ expiration_ts (int): New expiration date, as a timestamp in milliseconds
+ since epoch.
+ email_sent (bool): True means a renewal email has been sent for this
+ account and there's no need to send another one for the current validity
+ period.
+ renewal_token (str): Renewal token the user can use to extend the validity
+ of their account. Defaults to no token.
+ """
+ def set_account_validity_for_user_txn(txn):
+ self._simple_update_txn(
+ txn=txn,
+ table="account_validity",
+ keyvalues={"user_id": user_id},
+ updatevalues={
+ "expiration_ts_ms": expiration_ts,
+ "email_sent": email_sent,
+ "renewal_token": renewal_token,
+ },
+ )
+ self._invalidate_cache_and_stream(
+ txn, self.get_expiration_ts_for_user, (user_id,),
+ )
+
+ yield self.runInteraction(
+ "set_account_validity_for_user",
+ set_account_validity_for_user_txn,
+ )
+
+ @defer.inlineCallbacks
+ def set_renewal_token_for_user(self, user_id, renewal_token):
+ """Defines a renewal token for a given user.
+
+ Args:
+ user_id (str): ID of the user to set the renewal token for.
+ renewal_token (str): Random unique string that will be used to renew the
+ user's account.
+
+ Raises:
+ StoreError: The provided token is already set for another user.
+ """
+ yield self._simple_update_one(
+ table="account_validity",
+ keyvalues={"user_id": user_id},
+ updatevalues={"renewal_token": renewal_token},
+ desc="set_renewal_token_for_user",
+ )
+
+ @defer.inlineCallbacks
+ def get_user_from_renewal_token(self, renewal_token):
+ """Get a user ID from a renewal token.
+
+ Args:
+ renewal_token (str): The renewal token to perform the lookup with.
+
+ Returns:
+ defer.Deferred[str]: The ID of the user to which the token belongs.
+ """
+ res = yield self._simple_select_one_onecol(
+ table="account_validity",
+ keyvalues={"renewal_token": renewal_token},
+ retcol="user_id",
+ desc="get_user_from_renewal_token",
+ )
+
+ defer.returnValue(res)
+
+ @defer.inlineCallbacks
+ def get_renewal_token_for_user(self, user_id):
+ """Get the renewal token associated with a given user ID.
+
+ Args:
+ user_id (str): The user ID to lookup a token for.
+
+ Returns:
+ defer.Deferred[str]: The renewal token associated with this user ID.
+ """
+ res = yield self._simple_select_one_onecol(
+ table="account_validity",
+ keyvalues={"user_id": user_id},
+ retcol="renewal_token",
+ desc="get_renewal_token_for_user",
+ )
+
+ defer.returnValue(res)
+
+ @defer.inlineCallbacks
+ def get_users_expiring_soon(self):
+ """Selects users whose account will expire in the [now, now + renew_at] time
+ window (see configuration for account_validity for information on what renew_at
+ refers to).
+
+ Returns:
+ Deferred: Resolves to a list[dict[user_id (str), expiration_ts_ms (int)]]
+ """
+ def select_users_txn(txn, now_ms, renew_at):
+ sql = (
+ "SELECT user_id, expiration_ts_ms FROM account_validity"
+ " WHERE email_sent = ? AND (expiration_ts_ms - ?) <= ?"
+ )
+ values = [False, now_ms, renew_at]
+ txn.execute(sql, values)
+ return self.cursor_to_dict(txn)
+
+ res = yield self.runInteraction(
+ "get_users_expiring_soon",
+ select_users_txn,
+ self.clock.time_msec(), self.config.account_validity.renew_at,
+ )
+
+ defer.returnValue(res)
+
+ @defer.inlineCallbacks
+ def set_renewal_mail_status(self, user_id, email_sent):
+ """Sets or unsets the flag that indicates whether a renewal email has been sent
+ to the user (and the user hasn't renewed their account yet).
+
+ Args:
+ user_id (str): ID of the user to set/unset the flag for.
+ email_sent (bool): Flag which indicates whether a renewal email has been sent
+ to this user.
+ """
+ yield self._simple_update_one(
+ table="account_validity",
+ keyvalues={"user_id": user_id},
+ updatevalues={"email_sent": email_sent},
+ desc="set_renewal_mail_status",
+ )
+
@defer.inlineCallbacks
def is_server_admin(self, user):
res = yield self._simple_select_one_onecol(
@@ -425,6 +582,8 @@ class RegistrationStore(
columns=["creation_ts"],
)
+ self._account_validity = hs.config.account_validity
+
# we no longer use refresh tokens, but it's possible that some people
# might have a background update queued to build this index. Just
# clear the background update.
@@ -561,9 +720,23 @@ class RegistrationStore(
"user_type": user_type,
},
)
+
except self.database_engine.module.IntegrityError:
raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
+ if self._account_validity.enabled:
+ now_ms = self.clock.time_msec()
+ expiration_ts = now_ms + self._account_validity.period
+ self._simple_insert_txn(
+ txn,
+ "account_validity",
+ values={
+ "user_id": user_id,
+ "expiration_ts_ms": expiration_ts,
+ "email_sent": False,
+ }
+ )
+
if token:
# it's possible for this to get a conflict, but only for a single user
# since tokens are namespaced based on their user ID
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index 5eb93b38b2..f8649e5d99 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -13,19 +13,7 @@
* limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS application_services(
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- url TEXT,
- token TEXT,
- hs_token TEXT,
- sender TEXT,
- UNIQUE(token)
-);
-
-CREATE TABLE IF NOT EXISTS application_services_regex(
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- as_id BIGINT UNSIGNED NOT NULL,
- namespace INTEGER, /* enum[room_id|room_alias|user_id] */
- regex TEXT,
- FOREIGN KEY(as_id) REFERENCES application_services(id)
-);
+/* We used to create a tables called application_services and
+ * application_services_regex, but these are no longer used and are removed in
+ * delta 54.
+ */
diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
deleted file mode 100644
index 4d725b92fe..0000000000
--- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-import simplejson as json
-
-logger = logging.getLogger(__name__)
-
-
-def run_create(cur, *args, **kwargs):
- cur.execute("SELECT id, regex FROM application_services_regex")
- for row in cur.fetchall():
- try:
- logger.debug("Checking %s..." % row[0])
- json.loads(row[1])
- except ValueError:
- # row isn't in json, make it so.
- string_regex = row[1]
- new_regex = json.dumps({
- "regex": string_regex,
- "exclusive": True
- })
- cur.execute(
- "UPDATE application_services_regex SET regex=? WHERE id=?",
- (new_regex, row[0])
- )
-
-
-def run_upgrade(*args, **kwargs):
- pass
diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql
index fecf11118c..5b8de52c33 100644
--- a/synapse/storage/schema/delta/16/unique_constraints.sql
+++ b/synapse/storage/schema/delta/16/unique_constraints.sql
@@ -18,14 +18,6 @@ DROP INDEX IF EXISTS room_memberships_event_id;
CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id);
--
-DELETE FROM feedback WHERE rowid not in (
- SELECT MIN(rowid) FROM feedback GROUP BY event_id
-);
-
-DROP INDEX IF EXISTS feedback_event_id;
-CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id);
-
---
DELETE FROM topics WHERE rowid not in (
SELECT MIN(rowid) FROM topics GROUP BY event_id
);
diff --git a/synapse/storage/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/delta/24/stats_reporting.sql
index 5f508af7a9..acea7483bd 100644
--- a/synapse/storage/schema/delta/24/stats_reporting.sql
+++ b/synapse/storage/schema/delta/24/stats_reporting.sql
@@ -1,4 +1,4 @@
-/* Copyright 2015, 2016 OpenMarket Ltd
+/* Copyright 2019 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,10 +13,6 @@
* limitations under the License.
*/
--- Should only ever contain one row
-CREATE TABLE IF NOT EXISTS stats_reporting(
- -- The stream ordering token which was most recently reported as stats
- reported_stream_token INTEGER,
- -- The time (seconds since epoch) stats were most recently reported
- reported_time BIGINT
-);
+ /* We used to create a table called stats_reporting, but this is no longer
+ * used and is removed in delta 54.
+ */
\ No newline at end of file
diff --git a/synapse/storage/schema/delta/30/state_stream.sql b/synapse/storage/schema/delta/30/state_stream.sql
index 706fe1dcf4..e85699e82e 100644
--- a/synapse/storage/schema/delta/30/state_stream.sql
+++ b/synapse/storage/schema/delta/30/state_stream.sql
@@ -14,15 +14,10 @@
*/
-/**
- * The positions in the event stream_ordering when the current_state was
- * replaced by the state at the event.
+/* We used to create a table called current_state_resets, but this is no
+ * longer used and is removed in delta 54.
*/
-CREATE TABLE IF NOT EXISTS current_state_resets(
- event_stream_ordering BIGINT PRIMARY KEY NOT NULL
-);
-
/* The outlier events that have aquired a state group typically through
* backfill. This is tracked separately to the events table, as assigning a
* state group change the position of the existing event in the stream
diff --git a/synapse/storage/schema/delta/32/remove_indices.sql b/synapse/storage/schema/delta/32/remove_indices.sql
index f859be46a6..4219cdd06a 100644
--- a/synapse/storage/schema/delta/32/remove_indices.sql
+++ b/synapse/storage/schema/delta/32/remove_indices.sql
@@ -24,13 +24,9 @@ DROP INDEX IF EXISTS state_groups_id; -- Duplicate of PRIMARY KEY
DROP INDEX IF EXISTS event_to_state_groups_id; -- Duplicate of PRIMARY KEY
DROP INDEX IF EXISTS event_push_actions_room_id_event_id_user_id_profile_tag; -- Duplicate of UNIQUE CONSTRAINT
-DROP INDEX IF EXISTS event_destinations_id; -- Prefix of UNIQUE CONSTRAINT
DROP INDEX IF EXISTS st_extrem_id; -- Prefix of UNIQUE CONSTRAINT
-DROP INDEX IF EXISTS event_content_hashes_id; -- Prefix of UNIQUE CONSTRAINT
DROP INDEX IF EXISTS event_signatures_id; -- Prefix of UNIQUE CONSTRAINT
-DROP INDEX IF EXISTS event_edge_hashes_id; -- Prefix of UNIQUE CONSTRAINT
DROP INDEX IF EXISTS redactions_event_id; -- Duplicate of UNIQUE CONSTRAINT
-DROP INDEX IF EXISTS room_hosts_room_id; -- Prefix of UNIQUE CONSTRAINT
-- The following indices were unused
DROP INDEX IF EXISTS remote_media_cache_thumbnails_media_id;
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/delta/54/account_validity.sql
index 71a91f8ec9..2357626000 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/delta/54/account_validity.sql
@@ -1,4 +1,4 @@
-/* Copyright 2014-2016 OpenMarket Ltd
+/* Copyright 2019 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,12 +13,15 @@
* limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS room_aliases(
- room_alias TEXT NOT NULL,
- room_id TEXT NOT NULL
-);
+DROP TABLE IF EXISTS account_validity;
-CREATE TABLE IF NOT EXISTS room_alias_servers(
- room_alias TEXT NOT NULL,
- server TEXT NOT NULL
+-- Track what users are in public rooms.
+CREATE TABLE IF NOT EXISTS account_validity (
+ user_id TEXT PRIMARY KEY,
+ expiration_ts_ms BIGINT NOT NULL,
+ email_sent BOOLEAN NOT NULL,
+ renewal_token TEXT
);
+
+CREATE INDEX account_validity_email_sent_idx ON account_validity(email_sent, expiration_ts_ms)
+CREATE UNIQUE INDEX account_validity_renewal_string_idx ON account_validity(renewal_token)
diff --git a/synapse/storage/schema/delta/54/drop_legacy_tables.sql b/synapse/storage/schema/delta/54/drop_legacy_tables.sql
new file mode 100644
index 0000000000..dbbe682697
--- /dev/null
+++ b/synapse/storage/schema/delta/54/drop_legacy_tables.sql
@@ -0,0 +1,30 @@
+/* Copyright 2019 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- we need to do this first due to foreign constraints
+DROP TABLE IF EXISTS application_services_regex;
+
+DROP TABLE IF EXISTS application_services;
+DROP TABLE IF EXISTS transaction_id_to_pdu;
+DROP TABLE IF EXISTS stats_reporting;
+DROP TABLE IF EXISTS current_state_resets;
+DROP TABLE IF EXISTS event_content_hashes;
+DROP TABLE IF EXISTS event_destinations;
+DROP TABLE IF EXISTS event_edge_hashes;
+DROP TABLE IF EXISTS event_signatures;
+DROP TABLE IF EXISTS feedback;
+DROP TABLE IF EXISTS room_hosts;
+DROP TABLE IF EXISTS server_tls_certificates;
+DROP TABLE IF EXISTS state_forward_extremities;
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
deleted file mode 100644
index bccd1c6f74..0000000000
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ /dev/null
@@ -1,91 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS event_forward_extremities(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- UNIQUE (event_id, room_id)
-);
-
-CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
-CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_backward_extremities(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- UNIQUE (event_id, room_id)
-);
-
-CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
-CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_edges(
- event_id TEXT NOT NULL,
- prev_event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- -- We no longer insert prev_state into this table, so all new rows will have
- -- is_state as false.
- is_state BOOL NOT NULL,
- UNIQUE (event_id, prev_event_id, room_id, is_state)
-);
-
-CREATE INDEX ev_edges_id ON event_edges(event_id);
-CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
-
-
-CREATE TABLE IF NOT EXISTS room_depth(
- room_id TEXT NOT NULL,
- min_depth INTEGER NOT NULL,
- UNIQUE (room_id)
-);
-
-CREATE INDEX room_depth_room ON room_depth(room_id);
-
-
-create TABLE IF NOT EXISTS event_destinations(
- event_id TEXT NOT NULL,
- destination TEXT NOT NULL,
- delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
- UNIQUE (event_id, destination)
-);
-
-CREATE INDEX event_destinations_id ON event_destinations(event_id);
-
-
-CREATE TABLE IF NOT EXISTS state_forward_extremities(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- type TEXT NOT NULL,
- state_key TEXT NOT NULL,
- UNIQUE (event_id, room_id)
-);
-
-CREATE INDEX st_extrem_keys ON state_forward_extremities(
- room_id, type, state_key
-);
-CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_auth(
- event_id TEXT NOT NULL,
- auth_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- UNIQUE (event_id, auth_id, room_id)
-);
-
-CREATE INDEX evauth_edges_id ON event_auth(event_id);
-CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
deleted file mode 100644
index 00ce85980e..0000000000
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ /dev/null
@@ -1,55 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS event_content_hashes (
- event_id TEXT,
- algorithm TEXT,
- hash bytea,
- UNIQUE (event_id, algorithm)
-);
-
-CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_reference_hashes (
- event_id TEXT,
- algorithm TEXT,
- hash bytea,
- UNIQUE (event_id, algorithm)
-);
-
-CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_signatures (
- event_id TEXT,
- signature_name TEXT,
- key_id TEXT,
- signature bytea,
- UNIQUE (event_id, signature_name, key_id)
-);
-
-CREATE INDEX event_signatures_id ON event_signatures(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_edge_hashes(
- event_id TEXT,
- prev_event_id TEXT,
- algorithm TEXT,
- hash bytea,
- UNIQUE (event_id, prev_event_id, algorithm)
-);
-
-CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
deleted file mode 100644
index dfbbf9fd54..0000000000
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ /dev/null
@@ -1,123 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS events(
- stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
- topological_ordering BIGINT NOT NULL,
- event_id TEXT NOT NULL,
- type TEXT NOT NULL,
- room_id TEXT NOT NULL,
- content TEXT NOT NULL,
- unrecognized_keys TEXT,
- processed BOOL NOT NULL,
- outlier BOOL NOT NULL,
- depth BIGINT DEFAULT 0 NOT NULL,
- UNIQUE (event_id)
-);
-
-CREATE INDEX events_stream_ordering ON events (stream_ordering);
-CREATE INDEX events_topological_ordering ON events (topological_ordering);
-CREATE INDEX events_room_id ON events (room_id);
-
-
-CREATE TABLE IF NOT EXISTS event_json(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- internal_metadata TEXT NOT NULL,
- json TEXT NOT NULL,
- UNIQUE (event_id)
-);
-
-CREATE INDEX event_json_room_id ON event_json(room_id);
-
-
-CREATE TABLE IF NOT EXISTS state_events(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- type TEXT NOT NULL,
- state_key TEXT NOT NULL,
- prev_state TEXT,
- UNIQUE (event_id)
-);
-
-CREATE INDEX state_events_room_id ON state_events (room_id);
-CREATE INDEX state_events_type ON state_events (type);
-CREATE INDEX state_events_state_key ON state_events (state_key);
-
-
-CREATE TABLE IF NOT EXISTS current_state_events(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- type TEXT NOT NULL,
- state_key TEXT NOT NULL,
- UNIQUE (room_id, type, state_key)
-);
-
-CREATE INDEX curr_events_event_id ON current_state_events (event_id);
-CREATE INDEX current_state_events_room_id ON current_state_events (room_id);
-CREATE INDEX current_state_events_type ON current_state_events (type);
-CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
-
-CREATE TABLE IF NOT EXISTS room_memberships(
- event_id TEXT NOT NULL,
- user_id TEXT NOT NULL,
- sender TEXT NOT NULL,
- room_id TEXT NOT NULL,
- membership TEXT NOT NULL
-);
-
-CREATE INDEX room_memberships_event_id ON room_memberships (event_id);
-CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
-CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
-
-CREATE TABLE IF NOT EXISTS feedback(
- event_id TEXT NOT NULL,
- feedback_type TEXT,
- target_event_id TEXT,
- sender TEXT,
- room_id TEXT
-);
-
-CREATE TABLE IF NOT EXISTS topics(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- topic TEXT NOT NULL
-);
-
-CREATE INDEX topics_event_id ON topics(event_id);
-CREATE INDEX topics_room_id ON topics(room_id);
-
-CREATE TABLE IF NOT EXISTS room_names(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- name TEXT NOT NULL
-);
-
-CREATE INDEX room_names_event_id ON room_names(event_id);
-CREATE INDEX room_names_room_id ON room_names(room_id);
-
-CREATE TABLE IF NOT EXISTS rooms(
- room_id TEXT PRIMARY KEY NOT NULL,
- is_public BOOL,
- creator TEXT
-);
-
-CREATE TABLE IF NOT EXISTS room_hosts(
- room_id TEXT NOT NULL,
- host TEXT NOT NULL,
- UNIQUE (room_id, host)
-);
-
-CREATE INDEX room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
deleted file mode 100644
index ca0ca1b694..0000000000
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS server_tls_certificates(
- server_name TEXT, -- Server name.
- fingerprint TEXT, -- Certificate fingerprint.
- from_server TEXT, -- Which key server the certificate was fetched from.
- ts_added_ms BIGINT, -- When the certifcate was added.
- tls_certificate bytea, -- DER encoded x509 certificate.
- UNIQUE (server_name, fingerprint)
-);
-
-CREATE TABLE IF NOT EXISTS server_signature_keys(
- server_name TEXT, -- Server name.
- key_id TEXT, -- Key version.
- from_server TEXT, -- Which key server the key was fetched form.
- ts_added_ms BIGINT, -- When the key was added.
- verify_key bytea, -- NACL verification key.
- UNIQUE (server_name, key_id)
-);
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
deleted file mode 100644
index 9c264d6ece..0000000000
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ /dev/null
@@ -1,65 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS local_media_repository (
- media_id TEXT, -- The id used to refer to the media.
- media_type TEXT, -- The MIME-type of the media.
- media_length INTEGER, -- Length of the media in bytes.
- created_ts BIGINT, -- When the content was uploaded in ms.
- upload_name TEXT, -- The name the media was uploaded with.
- user_id TEXT, -- The user who uploaded the file.
- UNIQUE (media_id)
-);
-
-CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
- media_id TEXT, -- The id used to refer to the media.
- thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
- thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
- thumbnail_type TEXT, -- The MIME-type of the thumbnail.
- thumbnail_method TEXT, -- The method used to make the thumbnail.
- thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
- UNIQUE (
- media_id, thumbnail_width, thumbnail_height, thumbnail_type
- )
-);
-
-CREATE INDEX local_media_repository_thumbnails_media_id
- ON local_media_repository_thumbnails (media_id);
-
-CREATE TABLE IF NOT EXISTS remote_media_cache (
- media_origin TEXT, -- The remote HS the media came from.
- media_id TEXT, -- The id used to refer to the media on that server.
- media_type TEXT, -- The MIME-type of the media.
- created_ts BIGINT, -- When the content was uploaded in ms.
- upload_name TEXT, -- The name the media was uploaded with.
- media_length INTEGER, -- Length of the media in bytes.
- filesystem_id TEXT, -- The name used to store the media on disk.
- UNIQUE (media_origin, media_id)
-);
-
-CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
- media_origin TEXT, -- The remote HS the media came from.
- media_id TEXT, -- The id used to refer to the media.
- thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
- thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
- thumbnail_method TEXT, -- The method used to make the thumbnail
- thumbnail_type TEXT, -- The MIME-type of the thumbnail.
- thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
- filesystem_id TEXT, -- The name used to store the media on disk.
- UNIQUE (
- media_origin, media_id, thumbnail_width, thumbnail_height,
- thumbnail_type
- )
-);
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
deleted file mode 100644
index 492725994c..0000000000
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ /dev/null
@@ -1,35 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS presence(
- user_id TEXT NOT NULL,
- state VARCHAR(20),
- status_msg TEXT,
- mtime BIGINT -- miliseconds since last state change
-);
-
--- For each of /my/ users which possibly-remote users are allowed to see their
--- presence state
-CREATE TABLE IF NOT EXISTS presence_allow_inbound(
- observed_user_id TEXT NOT NULL,
- observer_user_id TEXT NOT NULL -- a UserID,
-);
-
--- For each of /my/ users (watcher), which possibly-remote users are they
--- watching?
-CREATE TABLE IF NOT EXISTS presence_list(
- user_id TEXT NOT NULL,
- observed_user_id TEXT NOT NULL, -- a UserID,
- accepted BOOLEAN NOT NULL
-);
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
deleted file mode 100644
index b314e6df75..0000000000
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ /dev/null
@@ -1,19 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS profiles(
- user_id TEXT NOT NULL,
- displayname TEXT,
- avatar_url TEXT
-);
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
deleted file mode 100644
index 318f0d9aa5..0000000000
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS redactions (
- event_id TEXT NOT NULL,
- redacts TEXT NOT NULL,
- UNIQUE (event_id)
-);
-
-CREATE INDEX redactions_event_id ON redactions (event_id);
-CREATE INDEX redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
deleted file mode 100644
index b901e0f017..0000000000
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ /dev/null
@@ -1,40 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS state_groups(
- id INTEGER PRIMARY KEY,
- room_id TEXT NOT NULL,
- event_id TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS state_groups_state(
- state_group INTEGER NOT NULL,
- room_id TEXT NOT NULL,
- type TEXT NOT NULL,
- state_key TEXT NOT NULL,
- event_id TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS event_to_state_groups(
- event_id TEXT NOT NULL,
- state_group INTEGER NOT NULL,
- UNIQUE (event_id)
-);
-
-CREATE INDEX state_groups_id ON state_groups(id);
-
-CREATE INDEX state_groups_state_id ON state_groups_state(state_group);
-CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
-CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
deleted file mode 100644
index f6a058832e..0000000000
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ /dev/null
@@ -1,44 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--- Stores what transaction ids we have received and what our response was
-CREATE TABLE IF NOT EXISTS received_transactions(
- transaction_id TEXT,
- origin TEXT,
- ts BIGINT,
- response_code INTEGER,
- response_json bytea,
- has_been_referenced SMALLINT DEFAULT 0, -- Whether thishas been referenced by a prev_tx
- UNIQUE (transaction_id, origin)
-);
-
-CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
-
--- For sent transactions only.
-CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
- transaction_id INTEGER,
- destination TEXT,
- pdu_id TEXT,
- pdu_origin TEXT
-);
-
-CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
-CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
-
--- To track destination health
-CREATE TABLE IF NOT EXISTS destinations(
- destination TEXT PRIMARY KEY,
- retry_last_ts BIGINT,
- retry_interval INTEGER
-);
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
deleted file mode 100644
index 6c1d4c34a1..0000000000
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ /dev/null
@@ -1,43 +0,0 @@
-/* Copyright 2014-2016 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS users(
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- name TEXT,
- password_hash TEXT,
- creation_ts BIGINT,
- admin SMALLINT DEFAULT 0 NOT NULL,
- UNIQUE(name)
-);
-
-CREATE TABLE IF NOT EXISTS access_tokens(
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- user_id TEXT NOT NULL,
- device_id TEXT,
- token TEXT NOT NULL,
- last_used BIGINT,
- UNIQUE(token)
-);
-
-CREATE TABLE IF NOT EXISTS user_ips (
- user TEXT NOT NULL,
- access_token TEXT NOT NULL,
- device_id TEXT,
- ip TEXT NOT NULL,
- user_agent TEXT NOT NULL,
- last_seen BIGINT NOT NULL,
- UNIQUE (user, access_token, ip, user_agent)
-);
-
-CREATE INDEX user_ips_user ON user_ips(user);
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index aee0e68473..883fcd10b2 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -13,22 +13,11 @@
* limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS application_services(
- id BIGINT PRIMARY KEY,
- url TEXT,
- token TEXT,
- hs_token TEXT,
- sender TEXT,
- UNIQUE(token)
-);
+/* We used to create tables called application_services and
+ * application_services_regex, but these are no longer used and are removed in
+ * delta 54.
+ */
-CREATE TABLE IF NOT EXISTS application_services_regex(
- id BIGINT PRIMARY KEY,
- as_id BIGINT NOT NULL,
- namespace INTEGER, /* enum[room_id|room_alias|user_id] */
- regex TEXT,
- FOREIGN KEY(as_id) REFERENCES application_services(id)
-);
CREATE TABLE IF NOT EXISTS application_services_state(
as_id TEXT PRIMARY KEY,
diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql
index 6b5a5a88fa..10ce2aa7a0 100644
--- a/synapse/storage/schema/full_schemas/16/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/16/event_edges.sql
@@ -13,6 +13,11 @@
* limitations under the License.
*/
+/* We used to create tables called event_destinations and
+ * state_forward_extremities, but these are no longer used and are removed in
+ * delta 54.
+ */
+
CREATE TABLE IF NOT EXISTS event_forward_extremities(
event_id TEXT NOT NULL,
room_id TEXT NOT NULL,
@@ -54,31 +59,6 @@ CREATE TABLE IF NOT EXISTS room_depth(
CREATE INDEX room_depth_room ON room_depth(room_id);
-
-create TABLE IF NOT EXISTS event_destinations(
- event_id TEXT NOT NULL,
- destination TEXT NOT NULL,
- delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
- UNIQUE (event_id, destination)
-);
-
-CREATE INDEX event_destinations_id ON event_destinations(event_id);
-
-
-CREATE TABLE IF NOT EXISTS state_forward_extremities(
- event_id TEXT NOT NULL,
- room_id TEXT NOT NULL,
- type TEXT NOT NULL,
- state_key TEXT NOT NULL,
- UNIQUE (event_id, room_id)
-);
-
-CREATE INDEX st_extrem_keys ON state_forward_extremities(
- room_id, type, state_key
-);
-CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
-
-
CREATE TABLE IF NOT EXISTS event_auth(
event_id TEXT NOT NULL,
auth_id TEXT NOT NULL,
diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql
index 00ce85980e..95826da431 100644
--- a/synapse/storage/schema/full_schemas/16/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql
@@ -13,15 +13,9 @@
* limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS event_content_hashes (
- event_id TEXT,
- algorithm TEXT,
- hash bytea,
- UNIQUE (event_id, algorithm)
-);
-
-CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
-
+ /* We used to create tables called event_content_hashes and event_edge_hashes,
+ * but these are no longer used and are removed in delta 54.
+ */
CREATE TABLE IF NOT EXISTS event_reference_hashes (
event_id TEXT,
@@ -42,14 +36,3 @@ CREATE TABLE IF NOT EXISTS event_signatures (
);
CREATE INDEX event_signatures_id ON event_signatures(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_edge_hashes(
- event_id TEXT,
- prev_event_id TEXT,
- algorithm TEXT,
- hash bytea,
- UNIQUE (event_id, prev_event_id, algorithm)
-);
-
-CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 5f5cb8d01d..a1a2aa8e5b 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -13,6 +13,10 @@
* limitations under the License.
*/
+/* We used to create tables called room_hosts and feedback,
+ * but these are no longer used and are removed in delta 54.
+ */
+
CREATE TABLE IF NOT EXISTS events(
stream_ordering INTEGER PRIMARY KEY,
topological_ordering BIGINT NOT NULL,
@@ -91,15 +95,6 @@ CREATE TABLE IF NOT EXISTS room_memberships(
CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
-CREATE TABLE IF NOT EXISTS feedback(
- event_id TEXT NOT NULL,
- feedback_type TEXT,
- target_event_id TEXT,
- sender TEXT,
- room_id TEXT,
- UNIQUE (event_id)
-);
-
CREATE TABLE IF NOT EXISTS topics(
event_id TEXT NOT NULL,
room_id TEXT NOT NULL,
@@ -123,11 +118,3 @@ CREATE TABLE IF NOT EXISTS rooms(
is_public BOOL,
creator TEXT
);
-
-CREATE TABLE IF NOT EXISTS room_hosts(
- room_id TEXT NOT NULL,
- host TEXT NOT NULL,
- UNIQUE (room_id, host)
-);
-
-CREATE INDEX room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
index ca0ca1b694..11cdffdbb3 100644
--- a/synapse/storage/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -12,14 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS server_tls_certificates(
- server_name TEXT, -- Server name.
- fingerprint TEXT, -- Certificate fingerprint.
- from_server TEXT, -- Which key server the certificate was fetched from.
- ts_added_ms BIGINT, -- When the certifcate was added.
- tls_certificate bytea, -- DER encoded x509 certificate.
- UNIQUE (server_name, fingerprint)
-);
+
+-- we used to create a table called server_tls_certificates, but this is no
+-- longer used, and is removed in delta 54.
CREATE TABLE IF NOT EXISTS server_signature_keys(
server_name TEXT, -- Server name.
diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql
index 0892c4cf96..01d2d8f833 100644
--- a/synapse/storage/schema/full_schemas/16/presence.sql
+++ b/synapse/storage/schema/full_schemas/16/presence.sql
@@ -28,5 +28,5 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound(
UNIQUE (observed_user_id, observer_user_id)
);
--- We used to create a table called presence_list, but this is no longer used
+-- We used to create a table called presence_list, but this is no longer used
-- and is removed in delta 54.
\ No newline at end of file
diff --git a/synapse/storage/state_deltas.py b/synapse/storage/state_deltas.py
index 56e42f583d..31a0279b18 100644
--- a/synapse/storage/state_deltas.py
+++ b/synapse/storage/state_deltas.py
@@ -22,6 +22,24 @@ logger = logging.getLogger(__name__)
class StateDeltasStore(SQLBaseStore):
def get_current_state_deltas(self, prev_stream_id):
+ """Fetch a list of room state changes since the given stream id
+
+ Each entry in the result contains the following fields:
+ - stream_id (int)
+ - room_id (str)
+ - type (str): event type
+ - state_key (str):
+ - event_id (str|None): new event_id for this state key. None if the
+ state has been deleted.
+ - prev_event_id (str|None): previous event_id for this state key. None
+ if it's new state.
+
+ Args:
+ prev_stream_id (int): point to get changes since (exclusive)
+
+ Returns:
+ Deferred[list[dict]]: results
+ """
prev_stream_id = int(prev_stream_id)
if not self._curr_state_delta_stream_cache.has_any_entity_changed(
prev_stream_id
diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py
index f0e4a0e10c..2f16f23d91 100644
--- a/synapse/util/async_helpers.py
+++ b/synapse/util/async_helpers.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd.
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/util/manhole.py b/synapse/util/manhole.py
index 9cb7e9c9ab..628a2962d9 100644
--- a/synapse/util/manhole.py
+++ b/synapse/util/manhole.py
@@ -1,4 +1,5 @@
# Copyright 2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,10 +12,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import sys
+import traceback
from twisted.conch import manhole_ssh
from twisted.conch.insults import insults
-from twisted.conch.manhole import ColoredManhole
+from twisted.conch.manhole import ColoredManhole, ManholeInterpreter
from twisted.conch.ssh.keys import Key
from twisted.cred import checkers, portal
@@ -79,7 +82,7 @@ def manhole(username, password, globals):
rlm = manhole_ssh.TerminalRealm()
rlm.chainedProtocolFactory = lambda: insults.ServerProtocol(
- ColoredManhole,
+ SynapseManhole,
dict(globals, __name__="__console__")
)
@@ -88,3 +91,55 @@ def manhole(username, password, globals):
factory.privateKeys[b'ssh-rsa'] = Key.fromString(PRIVATE_KEY)
return factory
+
+
+class SynapseManhole(ColoredManhole):
+ """Overrides connectionMade to create our own ManholeInterpreter"""
+ def connectionMade(self):
+ super(SynapseManhole, self).connectionMade()
+
+ # replace the manhole interpreter with our own impl
+ self.interpreter = SynapseManholeInterpreter(self, self.namespace)
+
+ # this would also be a good place to add more keyHandlers.
+
+
+class SynapseManholeInterpreter(ManholeInterpreter):
+ def showsyntaxerror(self, filename=None):
+ """Display the syntax error that just occurred.
+
+ Overrides the base implementation, ignoring sys.excepthook. We always want
+ any syntax errors to be sent to the terminal, rather than sentry.
+ """
+ type, value, tb = sys.exc_info()
+ sys.last_type = type
+ sys.last_value = value
+ sys.last_traceback = tb
+ if filename and type is SyntaxError:
+ # Work hard to stuff the correct filename in the exception
+ try:
+ msg, (dummy_filename, lineno, offset, line) = value.args
+ except ValueError:
+ # Not the format we expect; leave it alone
+ pass
+ else:
+ # Stuff in the right filename
+ value = SyntaxError(msg, (filename, lineno, offset, line))
+ sys.last_value = value
+ lines = traceback.format_exception_only(type, value)
+ self.write(''.join(lines))
+
+ def showtraceback(self):
+ """Display the exception that just occurred.
+
+ Overrides the base implementation, ignoring sys.excepthook. We always want
+ any syntax errors to be sent to the terminal, rather than sentry.
+ """
+ sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
+ sys.last_traceback = last_tb
+ try:
+ # We remove the first stack item because it is our own code.
+ lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
+ self.write(''.join(lines))
+ finally:
+ last_tb = ei = None
|
