diff --git a/synapse/__init__.py b/synapse/__init__.py
index e5f680bb31..048d6e572f 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -27,4 +27,4 @@ try:
except ImportError:
pass
-__version__ = "0.99.0rc3"
+__version__ = "0.99.0"
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 5b97a54d45..62c633146f 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -15,19 +15,36 @@
import gc
import logging
+import signal
import sys
+import traceback
import psutil
from daemonize import Daemonize
from twisted.internet import error, reactor
+from twisted.protocols.tls import TLSMemoryBIOFactory
from synapse.app import check_bind_error
+from synapse.crypto import context_factory
from synapse.util import PreserveLoggingContext
from synapse.util.rlimit import change_resource_limit
logger = logging.getLogger(__name__)
+_sighup_callbacks = []
+
+
+def register_sighup(func):
+ """
+ Register a function to be called when a SIGHUP occurs.
+
+ Args:
+ func (function): Function to be called when sent a SIGHUP signal.
+ Will be called with a single argument, the homeserver.
+ """
+ _sighup_callbacks.append(func)
+
def start_worker_reactor(appname, config):
""" Run the reactor in the main process
@@ -189,3 +206,68 @@ def listen_ssl(
logger.info("Synapse now listening on port %d (TLS)", port)
return r
+
+
+def refresh_certificate(hs):
+ """
+ Refresh the TLS certificates that Synapse is using by re-reading them from
+ disk and updating the TLS context factories to use them.
+ """
+ logging.info("Loading certificate from disk...")
+ hs.config.read_certificate_from_disk()
+ hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
+ hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+ hs.config
+ )
+ logging.info("Certificate loaded.")
+
+ if hs._listening_services:
+ logging.info("Updating context factories...")
+ for i in hs._listening_services:
+ # When you listenSSL, it doesn't make an SSL port but a TCP one with
+ # a TLS wrapping factory around the factory you actually want to get
+ # requests. This factory attribute is public but missing from
+ # Twisted's documentation.
+ if isinstance(i.factory, TLSMemoryBIOFactory):
+ # We want to replace TLS factories with a new one, with the new
+ # TLS configuration. We do this by reaching in and pulling out
+ # the wrappedFactory, and then re-wrapping it.
+ i.factory = TLSMemoryBIOFactory(
+ hs.tls_server_context_factory,
+ False,
+ i.factory.wrappedFactory
+ )
+ logging.info("Context factories updated.")
+
+
+def start(hs, listeners=None):
+ """
+ Start a Synapse server or worker.
+
+ Args:
+ hs (synapse.server.HomeServer)
+ listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml)
+ """
+ try:
+ # Set up the SIGHUP machinery.
+ if hasattr(signal, "SIGHUP"):
+ def handle_sighup(*args, **kwargs):
+ for i in _sighup_callbacks:
+ i(hs)
+
+ signal.signal(signal.SIGHUP, handle_sighup)
+
+ register_sighup(refresh_certificate)
+
+ # Load the certificate from disk.
+ refresh_certificate(hs)
+
+ # It is now safe to start your Synapse.
+ hs.start_listening(listeners)
+ hs.get_datastore().start_profiling()
+ except Exception:
+ traceback.print_exc(file=sys.stderr)
+ reactor = hs.get_reactor()
+ if reactor.running:
+ reactor.stop()
+ sys.exit(1)
diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 8559e141af..33107f56d1 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -168,12 +168,7 @@ def start(config_options):
)
ps.setup()
- ps.start_listening(config.worker_listeners)
-
- def start():
- ps.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ps, config.worker_listeners)
_base.start_worker_reactor("synapse-appservice", config)
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index f8a417cb60..a9d2147022 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -25,7 +25,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.http.server import JsonResource
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
@@ -173,17 +172,7 @@ def start(config_options):
)
ss.setup()
-
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-client-reader", config)
diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py
index 656e0edc0f..b8e5196152 100644
--- a/synapse/app/event_creator.py
+++ b/synapse/app/event_creator.py
@@ -25,7 +25,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.http.server import JsonResource
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
@@ -194,17 +193,7 @@ def start(config_options):
)
ss.setup()
-
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-event-creator", config)
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index 3de2715132..6ee2b76dcd 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -26,7 +26,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.federation.transport.server import TransportLayerServer
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
@@ -87,6 +86,16 @@ class FederationReaderServer(HomeServer):
resources.update({
FEDERATION_PREFIX: TransportLayerServer(self),
})
+ if name == "openid" and "federation" not in res["names"]:
+ # Only load the openid resource separately if federation resource
+ # is not specified since federation resource includes openid
+ # resource.
+ resources.update({
+ FEDERATION_PREFIX: TransportLayerServer(
+ self,
+ servlet_groups=["openid"],
+ ),
+ })
root_resource = create_resource_tree(resources, NoResource())
@@ -99,7 +108,8 @@ class FederationReaderServer(HomeServer):
listener_config,
root_resource,
self.version_string,
- )
+ ),
+ reactor=self.get_reactor()
)
logger.info("Synapse federation reader now listening on port %d", port)
@@ -160,17 +170,7 @@ def start(config_options):
)
ss.setup()
-
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-federation-reader", config)
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index d944e0517f..a461442fdc 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -25,7 +25,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.federation import send_queue
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
@@ -192,17 +191,8 @@ def start(config_options):
)
ss.setup()
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
_base.start_worker_reactor("synapse-federation-sender", config)
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index d9ef6edc3c..d5b954361d 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -26,7 +26,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.http.server import JsonResource
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.http.site import SynapseSite
@@ -250,17 +249,7 @@ def start(config_options):
)
ss.setup()
-
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-frontend-proxy", config)
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 250a17cef8..b4476bf16e 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -17,7 +17,6 @@
import gc
import logging
import os
-import signal
import sys
import traceback
@@ -28,7 +27,6 @@ from prometheus_client import Gauge
from twisted.application import service
from twisted.internet import defer, reactor
-from twisted.protocols.tls import TLSMemoryBIOFactory
from twisted.web.resource import EncodingResourceWrapper, NoResource
from twisted.web.server import GzipEncoderFactory
from twisted.web.static import File
@@ -49,7 +47,6 @@ from synapse.app import _base
from synapse.app._base import listen_ssl, listen_tcp, quit_with_error
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
-from synapse.crypto import context_factory
from synapse.federation.transport.server import TransportLayerServer
from synapse.http.additional_resource import AdditionalResource
from synapse.http.server import RootRedirect
@@ -86,7 +83,6 @@ def gz_wrap(r):
class SynapseHomeServer(HomeServer):
DATASTORE_CLASS = DataStore
- _listening_services = []
def _listener_http(self, config, listener_config):
port = listener_config["port"]
@@ -102,6 +98,10 @@ class SynapseHomeServer(HomeServer):
resources = {}
for res in listener_config["resources"]:
for name in res["names"]:
+ if name == "openid" and "federation" in res["names"]:
+ # Skip loading openid resource if federation is defined
+ # since federation resource will include openid
+ continue
resources.update(self._configure_named_resource(
name, res.get("compress", False),
))
@@ -137,6 +137,7 @@ class SynapseHomeServer(HomeServer):
self.version_string,
),
self.tls_server_context_factory,
+ reactor=self.get_reactor(),
)
else:
@@ -149,7 +150,8 @@ class SynapseHomeServer(HomeServer):
listener_config,
root_resource,
self.version_string,
- )
+ ),
+ reactor=self.get_reactor(),
)
def _configure_named_resource(self, name, compress=False):
@@ -196,6 +198,11 @@ class SynapseHomeServer(HomeServer):
FEDERATION_PREFIX: TransportLayerServer(self),
})
+ if name == "openid":
+ resources.update({
+ FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]),
+ })
+
if name in ["static", "client"]:
resources.update({
STATIC_PREFIX: File(
@@ -241,10 +248,10 @@ class SynapseHomeServer(HomeServer):
return resources
- def start_listening(self):
+ def start_listening(self, listeners):
config = self.get_config()
- for listener in config.listeners:
+ for listener in listeners:
if listener["type"] == "http":
self._listening_services.extend(
self._listener_http(config, listener)
@@ -328,20 +335,11 @@ def setup(config_options):
# generating config files and shouldn't try to continue.
sys.exit(0)
- sighup_callbacks = []
synapse.config.logger.setup_logging(
config,
- use_worker_options=False,
- register_sighup=sighup_callbacks.append
+ use_worker_options=False
)
- def handle_sighup(*args, **kwargs):
- for i in sighup_callbacks:
- i(*args, **kwargs)
-
- if hasattr(signal, "SIGHUP"):
- signal.signal(signal.SIGHUP, handle_sighup)
-
events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
@@ -377,76 +375,73 @@ def setup(config_options):
hs.setup()
- def refresh_certificate(*args):
+ @defer.inlineCallbacks
+ def do_acme():
"""
- Refresh the TLS certificates that Synapse is using by re-reading them
- from disk and updating the TLS context factories to use them.
+ Reprovision an ACME certificate, if it's required.
+
+ Returns:
+ Deferred[bool]: Whether the cert has been updated.
"""
- logging.info("Reloading certificate from disk...")
- hs.config.read_certificate_from_disk()
- hs.tls_server_context_factory = context_factory.ServerContextFactory(config)
- hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
+ acme = hs.get_acme_handler()
+
+ # Check how long the certificate is active for.
+ cert_days_remaining = hs.config.is_disk_cert_valid(
+ allow_self_signed=False
)
- logging.info("Certificate reloaded.")
-
- logging.info("Updating context factories...")
- for i in hs._listening_services:
- if isinstance(i.factory, TLSMemoryBIOFactory):
- i.factory = TLSMemoryBIOFactory(
- hs.tls_server_context_factory,
- False,
- i.factory.wrappedFactory
- )
- logging.info("Context factories updated.")
- sighup_callbacks.append(refresh_certificate)
+ # We want to reprovision if cert_days_remaining is None (meaning no
+ # certificate exists), or the days remaining number it returns
+ # is less than our re-registration threshold.
+ provision = False
+
+ if (cert_days_remaining is None):
+ provision = True
+
+ if cert_days_remaining > hs.config.acme_reprovision_threshold:
+ provision = True
+
+ if provision:
+ yield acme.provision_certificate()
+
+ defer.returnValue(provision)
+
+ @defer.inlineCallbacks
+ def reprovision_acme():
+ """
+ Provision a certificate from ACME, if required, and reload the TLS
+ certificate if it's renewed.
+ """
+ reprovisioned = yield do_acme()
+ if reprovisioned:
+ _base.refresh_certificate(hs)
@defer.inlineCallbacks
def start():
try:
- # Check if the certificate is still valid.
- cert_days_remaining = hs.config.is_disk_cert_valid()
-
+ # Run the ACME provisioning code, if it's enabled.
if hs.config.acme_enabled:
- # If ACME is enabled, we might need to provision a certificate
- # before starting.
acme = hs.get_acme_handler()
-
# Start up the webservices which we will respond to ACME
- # challenges with.
+ # challenges with, and then provision.
yield acme.start_listening()
+ yield do_acme()
- # We want to reprovision if cert_days_remaining is None (meaning no
- # certificate exists), or the days remaining number it returns
- # is less than our re-registration threshold.
- if (cert_days_remaining is None) or (
- not cert_days_remaining > hs.config.acme_reprovision_threshold
- ):
- yield acme.provision_certificate()
-
- # Read the certificate from disk and build the context factories for
- # TLS.
- hs.config.read_certificate_from_disk()
- hs.tls_server_context_factory = context_factory.ServerContextFactory(config)
- hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
+ # Check if it needs to be reprovisioned every day.
+ hs.get_clock().looping_call(
+ reprovision_acme,
+ 24 * 60 * 60 * 1000
+ )
+
+ _base.start(hs, config.listeners)
- # It is now safe to start your Synapse.
- hs.start_listening()
hs.get_pusherpool().start()
- hs.get_datastore().start_profiling()
hs.get_datastore().start_doing_background_updates()
- except Exception as e:
- # If a DeferredList failed (like in listening on the ACME listener),
- # we need to print the subfailure explicitly.
- if isinstance(e, defer.FirstError):
- e.subFailure.printTraceback(sys.stderr)
- sys.exit(1)
-
- # Something else went wrong when starting. Print it and bail out.
+ except Exception:
+ # Print the exception and bail out.
traceback.print_exc(file=sys.stderr)
+ if reactor.running:
+ reactor.stop()
sys.exit(1)
reactor.callWhenRunning(start)
@@ -455,7 +450,8 @@ def setup(config_options):
class SynapseService(service.Service):
- """A twisted Service class that will start synapse. Used to run synapse
+ """
+ A twisted Service class that will start synapse. Used to run synapse
via twistd and a .tac.
"""
def __init__(self, config):
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index 4ecf64031b..d4cc4e9443 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -26,7 +26,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
@@ -160,17 +159,7 @@ def start(config_options):
)
ss.setup()
-
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-media-repository", config)
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 83b0863f00..cbf0d67f51 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -224,11 +224,10 @@ def start(config_options):
)
ps.setup()
- ps.start_listening(config.worker_listeners)
def start():
+ _base.start(ps, config.worker_listeners)
ps.get_pusherpool().start()
- ps.get_datastore().start_profiling()
reactor.callWhenRunning(start)
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 0354e82bf8..9163b56d86 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -445,12 +445,7 @@ def start(config_options):
)
ss.setup()
- ss.start_listening(config.worker_listeners)
-
- def start():
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-synchrotron", config)
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index 176d55a783..d1ab9512cd 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -26,7 +26,6 @@ from synapse.app import _base
from synapse.config._base import ConfigError
from synapse.config.homeserver import HomeServerConfig
from synapse.config.logger import setup_logging
-from synapse.crypto import context_factory
from synapse.http.server import JsonResource
from synapse.http.site import SynapseSite
from synapse.metrics import RegistryProxy
@@ -220,17 +219,7 @@ def start(config_options):
)
ss.setup()
-
- def start():
- ss.config.read_certificate_from_disk()
- ss.tls_server_context_factory = context_factory.ServerContextFactory(config)
- ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
- config
- )
- ss.start_listening(config.worker_listeners)
- ss.get_datastore().start_profiling()
-
- reactor.callWhenRunning(start)
+ reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
_base.start_worker_reactor("synapse-user-dir", config)
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index a795e39b1a..4b938053fb 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -15,7 +15,6 @@
import logging
import logging.config
import os
-import signal
import sys
from string import Template
@@ -24,6 +23,7 @@ import yaml
from twisted.logger import STDLibLogObserver, globalLogBeginner
import synapse
+from synapse.app import _base as appbase
from synapse.util.logcontext import LoggingContextFilter
from synapse.util.versionstring import get_version_string
@@ -127,7 +127,7 @@ class LoggingConfig(Config):
)
-def setup_logging(config, use_worker_options=False, register_sighup=None):
+def setup_logging(config, use_worker_options=False):
""" Set up python logging
Args:
@@ -140,12 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
register_sighup (func | None): Function to call to register a
sighup handler.
"""
- if not register_sighup:
- if getattr(signal, "SIGHUP"):
- register_sighup = lambda x: signal.signal(signal.SIGHUP, x)
- else:
- register_sighup = lambda x: None
-
log_config = (config.worker_log_config if use_worker_options
else config.log_config)
log_file = (config.worker_log_file if use_worker_options
@@ -187,7 +181,7 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
else:
handler = logging.StreamHandler()
- def sighup(signum, stack):
+ def sighup(*args):
pass
handler.setFormatter(formatter)
@@ -200,14 +194,14 @@ def setup_logging(config, use_worker_options=False, register_sighup=None):
with open(log_config, 'r') as f:
logging.config.dictConfig(yaml.load(f))
- def sighup(signum, stack):
+ def sighup(*args):
# it might be better to use a file watcher or something for this.
load_log_config()
logging.info("Reloaded log config from %s due to SIGHUP", log_config)
load_log_config()
- register_sighup(sighup)
+ appbase.register_sighup(sighup)
# make sure that the first thing we log is a thing we can grep backwards
# for
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 268a43ff00..ce0458195c 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -24,6 +24,14 @@ from ._base import Config, ConfigError
logger = logging.Logger(__name__)
+# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes
+# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen
+# on IPv6 when '::' is set.
+#
+# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in
+# in the list.
+DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0']
+
class ServerConfig(Config):
@@ -124,10 +132,13 @@ class ServerConfig(Config):
bind_address = listener.pop("bind_address", None)
bind_addresses = listener.setdefault("bind_addresses", [])
+ # if bind_address was specified, add it to the list of addresses
if bind_address:
bind_addresses.append(bind_address)
- elif not bind_addresses:
- bind_addresses.append('')
+
+ # if we still have an empty list of addresses, use the default list
+ if not bind_addresses:
+ bind_addresses.extend(DEFAULT_BIND_ADDRESSES)
if not self.web_client_location:
_warn_if_webclient_configured(self.listeners)
@@ -295,61 +306,92 @@ class ServerConfig(Config):
# List of ports that Synapse should listen on, their purpose and their
# configuration.
+ #
+ # Options for each listener include:
+ #
+ # port: the TCP port to bind to
+ #
+ # bind_addresses: a list of local addresses to listen on. The default is
+ # 'all local interfaces'.
+ #
+ # type: the type of listener. Normally 'http', but other valid options are:
+ # 'manhole' (see docs/manhole.md),
+ # 'metrics' (see docs/metrics-howto.rst),
+ # 'replication' (see docs/workers.rst).
+ #
+ # tls: set to true to enable TLS for this listener. Will use the TLS
+ # key/cert specified in tls_private_key_path / tls_certificate_path.
+ #
+ # x_forwarded: Only valid for an 'http' listener. Set to true to use the
+ # X-Forwarded-For header as the client IP. Useful when Synapse is
+ # behind a reverse-proxy.
+ #
+ # resources: Only valid for an 'http' listener. A list of resources to host
+ # on this port. Options for each resource are:
+ #
+ # names: a list of names of HTTP resources. See below for a list of
+ # valid resource names.
+ #
+ # compress: set to true to enable HTTP comression for this resource.
+ #
+ # additional_resources: Only valid for an 'http' listener. A map of
+ # additional endpoints which should be loaded via dynamic modules.
+ #
+ # Valid resource names are:
+ #
+ # client: the client-server API (/_matrix/client). Also implies 'media' and
+ # 'static'.
+ #
+ # consent: user consent forms (/_matrix/consent). See
+ # docs/consent_tracking.md.
+ #
+ # federation: the server-server API (/_matrix/federation). Also implies
+ # 'media', 'keys', 'openid'
+ #
+ # keys: the key discovery API (/_matrix/keys).
+ #
+ # media: the media API (/_matrix/media).
+ #
+ # metrics: the metrics interface. See docs/metrics-howto.rst.
+ #
+ # openid: OpenID authentication.
+ #
+ # replication: the HTTP replication API (/_synapse/replication). See
+ # docs/workers.rst.
+ #
+ # static: static resources under synapse/static (/_matrix/static). (Mostly
+ # useful for 'fallback authentication'.)
+ #
+ # webclient: A web client. Requires web_client_location to be set.
+ #
listeners:
- # Main HTTPS listener
+ # Main HTTPS listener.
# For when matrix traffic is sent directly to synapse.
- -
- # The port to listen for HTTPS requests on.
- port: %(bind_port)s
-
- # Local addresses to listen on.
- # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6
- # addresses by default. For most other OSes, this will only listen
- # on IPv6.
- bind_addresses:
- - '::'
- - '0.0.0.0'
-
- # This is a 'http' listener, allows us to specify 'resources'.
+ - port: %(bind_port)s
type: http
-
tls: true
- # Use the X-Forwarded-For (XFF) header as the client IP and not the
- # actual client IP.
- x_forwarded: false
-
# List of HTTP resources to serve on this listener.
resources:
- -
- # List of resources to host on this listener.
- names:
- - client # The client-server APIs, both v1 and v2
- # - webclient # A web client. Requires web_client_location to be set.
-
- # Should synapse compress HTTP responses to clients that support it?
- # This should be disabled if running synapse behind a load balancer
- # that can do automatic compression.
+ - names: [client]
compress: true
-
- - names: [federation] # Federation APIs
+ - names: [federation]
compress: false
- # optional list of additional endpoints which can be loaded via
- # dynamic modules
+ # example addional_resources:
+ #
# additional_resources:
# "/_matrix/my/custom/endpoint":
# module: my_module.CustomRequestHandler
# config: {}
- # Unsecure HTTP listener,
- # For when matrix traffic passes through loadbalancer that unwraps TLS.
+ # Unsecure HTTP listener
+ # For when matrix traffic passes through a reverse-proxy that unwraps TLS.
- port: %(unsecure_port)s
tls: false
- bind_addresses: ['::', '0.0.0.0']
+ bind_addresses: ['::1', '127.0.0.1']
type: http
-
- x_forwarded: false
+ x_forwarded: true
resources:
- names: [client]
@@ -480,6 +522,7 @@ KNOWN_RESOURCES = (
'keys',
'media',
'metrics',
+ 'openid',
'replication',
'static',
'webclient',
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 5f63676d9c..9fcc79816d 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -37,7 +37,7 @@ class TlsConfig(Config):
self.acme_enabled = acme_config.get("enabled", False)
self.acme_url = acme_config.get(
- "url", "https://acme-v01.api.letsencrypt.org/directory"
+ "url", u"https://acme-v01.api.letsencrypt.org/directory"
)
self.acme_port = acme_config.get("port", 80)
self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0'])
@@ -45,7 +45,11 @@ class TlsConfig(Config):
self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
- self._original_tls_fingerprints = config["tls_fingerprints"]
+ self._original_tls_fingerprints = config.get("tls_fingerprints", [])
+
+ if self._original_tls_fingerprints is None:
+ self._original_tls_fingerprints = []
+
self.tls_fingerprints = list(self._original_tls_fingerprints)
self.no_tls = config.get("no_tls", False)
@@ -60,10 +64,14 @@ class TlsConfig(Config):
self.tls_certificate = None
self.tls_private_key = None
- def is_disk_cert_valid(self):
+ def is_disk_cert_valid(self, allow_self_signed=True):
"""
Is the certificate we have on disk valid, and if so, for how long?
+ Args:
+ allow_self_signed (bool): Should we allow the certificate we
+ read to be self signed?
+
Returns:
int: Days remaining of certificate validity.
None: No certificate exists.
@@ -84,6 +92,12 @@ class TlsConfig(Config):
logger.exception("Failed to parse existing certificate off disk!")
raise
+ if not allow_self_signed:
+ if tls_certificate.get_subject() == tls_certificate.get_issuer():
+ raise ValueError(
+ "TLS Certificate is self signed, and this is not permitted"
+ )
+
# YYYYMMDDhhmmssZ -- in UTC
expires_on = datetime.strptime(
tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ"
@@ -199,10 +213,10 @@ class TlsConfig(Config):
# If your server runs behind a reverse-proxy which terminates TLS connections
# (for both client and federation connections), it may be useful to disable
- # All TLS support for incoming connections. Setting no_tls to False will
+ # All TLS support for incoming connections. Setting no_tls to True will
# do so (and avoid the need to give synapse a TLS private key).
#
- # no_tls: False
+ # no_tls: True
# List of allowed TLS fingerprints for this server to publish along
# with the signing keys for this server. Other matrix servers that
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 67ae0212c3..a2396ab466 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -43,9 +43,20 @@ logger = logging.getLogger(__name__)
class TransportLayerServer(JsonResource):
"""Handles incoming federation HTTP requests"""
- def __init__(self, hs):
+ def __init__(self, hs, servlet_groups=None):
+ """Initialize the TransportLayerServer
+
+ Will by default register all servlets. For custom behaviour, pass in
+ a list of servlet_groups to register.
+
+ Args:
+ hs (synapse.server.HomeServer): homeserver
+ servlet_groups (list[str], optional): List of servlet groups to register.
+ Defaults to ``DEFAULT_SERVLET_GROUPS``.
+ """
self.hs = hs
self.clock = hs.get_clock()
+ self.servlet_groups = servlet_groups
super(TransportLayerServer, self).__init__(hs, canonical_json=False)
@@ -67,6 +78,7 @@ class TransportLayerServer(JsonResource):
resource=self,
ratelimiter=self.ratelimiter,
authenticator=self.authenticator,
+ servlet_groups=self.servlet_groups,
)
@@ -1308,10 +1320,12 @@ FEDERATION_SERVLET_CLASSES = (
FederationClientKeysClaimServlet,
FederationThirdPartyInviteExchangeServlet,
On3pidBindServlet,
- OpenIdUserInfo,
FederationVersionServlet,
)
+OPENID_SERVLET_CLASSES = (
+ OpenIdUserInfo,
+)
ROOM_LIST_CLASSES = (
PublicRoomList,
@@ -1350,44 +1364,83 @@ GROUP_ATTESTATION_SERVLET_CLASSES = (
FederationGroupsRenewAttestaionServlet,
)
+DEFAULT_SERVLET_GROUPS = (
+ "federation",
+ "room_list",
+ "group_server",
+ "group_local",
+ "group_attestation",
+ "openid",
+)
+
+
+def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None):
+ """Initialize and register servlet classes.
-def register_servlets(hs, resource, authenticator, ratelimiter):
- for servletclass in FEDERATION_SERVLET_CLASSES:
- servletclass(
- handler=hs.get_federation_server(),
- authenticator=authenticator,
- ratelimiter=ratelimiter,
- server_name=hs.hostname,
- ).register(resource)
-
- for servletclass in ROOM_LIST_CLASSES:
- servletclass(
- handler=hs.get_room_list_handler(),
- authenticator=authenticator,
- ratelimiter=ratelimiter,
- server_name=hs.hostname,
- ).register(resource)
-
- for servletclass in GROUP_SERVER_SERVLET_CLASSES:
- servletclass(
- handler=hs.get_groups_server_handler(),
- authenticator=authenticator,
- ratelimiter=ratelimiter,
- server_name=hs.hostname,
- ).register(resource)
-
- for servletclass in GROUP_LOCAL_SERVLET_CLASSES:
- servletclass(
- handler=hs.get_groups_local_handler(),
- authenticator=authenticator,
- ratelimiter=ratelimiter,
- server_name=hs.hostname,
- ).register(resource)
-
- for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES:
- servletclass(
- handler=hs.get_groups_attestation_renewer(),
- authenticator=authenticator,
- ratelimiter=ratelimiter,
- server_name=hs.hostname,
- ).register(resource)
+ Will by default register all servlets. For custom behaviour, pass in
+ a list of servlet_groups to register.
+
+ Args:
+ hs (synapse.server.HomeServer): homeserver
+ resource (TransportLayerServer): resource class to register to
+ authenticator (Authenticator): authenticator to use
+ ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use
+ servlet_groups (list[str], optional): List of servlet groups to register.
+ Defaults to ``DEFAULT_SERVLET_GROUPS``.
+ """
+ if not servlet_groups:
+ servlet_groups = DEFAULT_SERVLET_GROUPS
+
+ if "federation" in servlet_groups:
+ for servletclass in FEDERATION_SERVLET_CLASSES:
+ servletclass(
+ handler=hs.get_federation_server(),
+ authenticator=authenticator,
+ ratelimiter=ratelimiter,
+ server_name=hs.hostname,
+ ).register(resource)
+
+ if "openid" in servlet_groups:
+ for servletclass in OPENID_SERVLET_CLASSES:
+ servletclass(
+ handler=hs.get_federation_server(),
+ authenticator=authenticator,
+ ratelimiter=ratelimiter,
+ server_name=hs.hostname,
+ ).register(resource)
+
+ if "room_list" in servlet_groups:
+ for servletclass in ROOM_LIST_CLASSES:
+ servletclass(
+ handler=hs.get_room_list_handler(),
+ authenticator=authenticator,
+ ratelimiter=ratelimiter,
+ server_name=hs.hostname,
+ ).register(resource)
+
+ if "group_server" in servlet_groups:
+ for servletclass in GROUP_SERVER_SERVLET_CLASSES:
+ servletclass(
+ handler=hs.get_groups_server_handler(),
+ authenticator=authenticator,
+ ratelimiter=ratelimiter,
+ server_name=hs.hostname,
+ ).register(resource)
+
+ if "group_local" in servlet_groups:
+ for servletclass in GROUP_LOCAL_SERVLET_CLASSES:
+ servletclass(
+ handler=hs.get_groups_local_handler(),
+ authenticator=authenticator,
+ ratelimiter=ratelimiter,
+ server_name=hs.hostname,
+ ).register(resource)
+
+ if "group_attestation" in servlet_groups:
+ for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES:
+ servletclass(
+ handler=hs.get_groups_attestation_renewer(),
+ authenticator=authenticator,
+ ratelimiter=ratelimiter,
+ server_name=hs.hostname,
+ ).register(resource)
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 756721e304..5d087ee26b 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -85,7 +85,7 @@ CONDITIONAL_REQUIREMENTS = {
"saml2": ["pysaml2>=4.5.0"],
"url_preview": ["lxml>=3.5.0"],
- "test": ["mock>=2.0"],
+ "test": ["mock>=2.0", "parameterized"],
}
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 0251146722..39d157a44b 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -75,7 +75,7 @@ class SyncRestServlet(RestServlet):
"""
PATTERNS = client_v2_patterns("/sync$")
- ALLOWED_PRESENCE = set(["online", "offline"])
+ ALLOWED_PRESENCE = set(["online", "offline", "unavailable"])
def __init__(self, hs):
super(SyncRestServlet, self).__init__()
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index 29e62bfcdd..27e7cbf3cc 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -38,6 +38,7 @@ class VersionsRestServlet(RestServlet):
"r0.1.0",
"r0.2.0",
"r0.3.0",
+ "r0.4.0",
],
# as per MSC1497:
"unstable_features": {
diff --git a/synapse/server.py b/synapse/server.py
index 6c52101616..a2cf8a91cd 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -112,6 +112,8 @@ class HomeServer(object):
Attributes:
config (synapse.config.homeserver.HomeserverConfig):
+ _listening_services (list[twisted.internet.tcp.Port]): TCP ports that
+ we are listening on to provide HTTP services.
"""
__metaclass__ = abc.ABCMeta
@@ -196,6 +198,7 @@ class HomeServer(object):
self._reactor = reactor
self.hostname = hostname
self._building = {}
+ self._listening_services = []
self.clock = Clock(reactor)
self.distributor = Distributor()
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 4872ff55b6..e124161845 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -50,6 +50,21 @@ sql_query_timer = Histogram("synapse_storage_query_time", "sec", ["verb"])
sql_txn_timer = Histogram("synapse_storage_transaction_time", "sec", ["desc"])
+# Unique indexes which have been added in background updates. Maps from table name
+# to the name of the background update which added the unique index to that table.
+#
+# This is used by the upsert logic to figure out which tables are safe to do a proper
+# UPSERT on: until the relevant background update has completed, we
+# have to emulate an upsert by locking the table.
+#
+UNIQUE_INDEX_BACKGROUND_UPDATES = {
+ "user_ips": "user_ips_device_unique_index",
+ "device_lists_remote_extremeties": "device_lists_remote_extremeties_unique_idx",
+ "device_lists_remote_cache": "device_lists_remote_cache_unique_idx",
+ "event_search": "event_search_event_id_idx",
+}
+
+
class LoggingTransaction(object):
"""An object that almost-transparently proxies for the 'txn' object
passed to the constructor. Adds logging and metrics to the .execute()
@@ -194,7 +209,7 @@ class SQLBaseStore(object):
self.database_engine = hs.database_engine
# A set of tables that are not safe to use native upserts in.
- self._unsafe_to_upsert_tables = {"user_ips"}
+ self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys())
# We add the user_directory_search table to the blacklist on SQLite
# because the existing search table does not have an index, making it
@@ -230,12 +245,12 @@ class SQLBaseStore(object):
)
updates = [x["update_name"] for x in updates]
- # The User IPs table in schema #53 was missing a unique index, which we
- # run as a background update.
- if "user_ips_device_unique_index" not in updates:
- self._unsafe_to_upsert_tables.discard("user_ips")
+ for table, update_name in UNIQUE_INDEX_BACKGROUND_UPDATES.items():
+ if update_name not in updates:
+ logger.debug("Now safe to upsert in %s", table)
+ self._unsafe_to_upsert_tables.discard(table)
- # If there's any tables left to check, reschedule to run.
+ # If there's any updates still running, reschedule to run.
if updates:
self._clock.call_later(
15.0,
diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py
index ce48212265..e8b574ee5e 100644
--- a/synapse/storage/user_directory.py
+++ b/synapse/storage/user_directory.py
@@ -22,6 +22,7 @@ from twisted.internet import defer
from synapse.api.constants import EventTypes, JoinRules
from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+from synapse.storage.state import StateFilter
from synapse.types import get_domain_from_id, get_localpart_from_id
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
@@ -31,12 +32,19 @@ logger = logging.getLogger(__name__)
class UserDirectoryStore(SQLBaseStore):
- @cachedInlineCallbacks(cache_context=True)
- def is_room_world_readable_or_publicly_joinable(self, room_id, cache_context):
+ @defer.inlineCallbacks
+ def is_room_world_readable_or_publicly_joinable(self, room_id):
"""Check if the room is either world_readable or publically joinable
"""
- current_state_ids = yield self.get_current_state_ids(
- room_id, on_invalidate=cache_context.invalidate
+
+ # Create a state filter that only queries join and history state event
+ types_to_filter = (
+ (EventTypes.JoinRules, ""),
+ (EventTypes.RoomHistoryVisibility, ""),
+ )
+
+ current_state_ids = yield self.get_filtered_current_state_ids(
+ room_id, StateFilter.from_types(types_to_filter),
)
join_rules_id = current_state_ids.get((EventTypes.JoinRules, ""))
|
