From 82be4457deeab9cc4e21ffebfec78bb2bac2e5b1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jan 2015 13:46:37 +0000 Subject: Add twisted Service interface --- synapse/app/homeserver.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5fec8da7ca..e99153988f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -19,6 +19,7 @@ from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer from twisted.internet import reactor +from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource from twisted.web.static import File @@ -189,10 +190,10 @@ class SynapseHomeServer(HomeServer): logger.info("Synapse now listening on port %d", unsecure_port) -def setup(): +def setup(config_options, should_run=True): config = HomeServerConfig.load_config( "Synapse Homeserver", - sys.argv[1:], + config_options, generate_section="Homeserver" ) @@ -254,6 +255,9 @@ def setup(): bind_port = None hs.start_listening(bind_port, config.unsecure_port) + if not should_run: + return + if config.daemonize: print config.pid_file daemon = Daemonize( @@ -270,6 +274,17 @@ def setup(): reactor.run() +class SynapseService(service.Service): + def __init__(self, config): + self.config = config + + def startService(self): + setup(self.config, should_run=False) + + def stopService(self): + return self._port.stopListening() + + def run(): with LoggingContext("run"): reactor.run() @@ -277,7 +292,7 @@ def run(): def main(): with LoggingContext("main"): - setup() + setup(sys.argv[1:]) if __name__ == '__main__': -- cgit 1.5.1 From b8a6692657d350389f1c3dc8bd903110c2ae1d72 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Mar 2015 09:39:42 +0000 Subject: Add documentation. When starting via twistd respect soft_file_limit config option. --- synapse/app/homeserver.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index dff08c8bc5..26ef02e3de 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -297,6 +297,15 @@ def change_resource_limit(soft_file_no): def setup(config_options, should_run=True): + """ + Args: + config_options_options: The options passed to Synapse. Usually + `sys.argv[1:]`. + should_run (bool): Whether to start the reactor. + + Returns: + HomeServer + """ config = HomeServerConfig.load_config( "Synapse Homeserver", config_options, @@ -372,7 +381,7 @@ def setup(config_options, should_run=True): hs.get_replication_layer().start_get_pdu_cache() if not should_run: - return + return hs if config.daemonize: print config.pid_file @@ -390,13 +399,19 @@ def setup(config_options, should_run=True): else: run(config) + return hs + class SynapseService(service.Service): + """A twisted Service class that will start synapse. Used to run synapse + via twistd and a .tac. + """ def __init__(self, config): self.config = config def startService(self): - setup(self.config, should_run=False) + hs = setup(self.config, should_run=False) + change_resource_limit(hs.config.soft_file_limit) def stopService(self): return self._port.stopListening() -- cgit 1.5.1 From c37eceeb9ef2514237468a7f5e3e6e07d293de16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Mar 2015 09:58:33 +0000 Subject: Split out the 'run' from 'setup' --- synapse/app/homeserver.py | 53 ++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 26ef02e3de..e84b9ef5b3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -296,7 +296,7 @@ def change_resource_limit(soft_file_no): logger.warn("Failed to set file limit: %s", e) -def setup(config_options, should_run=True): +def setup(config_options): """ Args: config_options_options: The options passed to Synapse. Usually @@ -380,25 +380,6 @@ def setup(config_options, should_run=True): hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() - if not should_run: - return hs - - if config.daemonize: - print config.pid_file - - daemon = Daemonize( - app="synapse-homeserver", - pid=config.pid_file, - action=lambda: run(config), - auto_close_fds=False, - verbose=True, - logger=logger, - ) - - daemon.start() - else: - run(config) - return hs @@ -410,24 +391,44 @@ class SynapseService(service.Service): self.config = config def startService(self): - hs = setup(self.config, should_run=False) + hs = setup(self.config) change_resource_limit(hs.config.soft_file_limit) def stopService(self): return self._port.stopListening() -def run(config): - with LoggingContext("run"): - change_resource_limit(config.soft_file_limit) +def run(hs): + + def in_thread(): + with LoggingContext("run"): + change_resource_limit(hs.config.soft_file_limit) - reactor.run() + reactor.run() + + if hs.config.daemonize: + + print hs.config.pid_file + + daemon = Daemonize( + app="synapse-homeserver", + pid=hs.config.pid_file, + action=lambda: in_thread(), + auto_close_fds=False, + verbose=True, + logger=logger, + ) + + daemon.start() + else: + in_thread(hs.config) def main(): with LoggingContext("main"): check_requirements() - setup(sys.argv[1:]) + hs = setup(sys.argv[1:]) + run(hs) if __name__ == '__main__': -- cgit 1.5.1 From 045afd6b6154ffb76664a16bdcb4d6bca791573d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Mar 2015 10:19:03 +0000 Subject: in_thread takes no arguments --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e84b9ef5b3..3649406efb 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -421,7 +421,7 @@ def run(hs): daemon.start() else: - in_thread(hs.config) + in_thread() def main(): -- cgit 1.5.1 From 211c14c3910ca0cd86a55ac27ef575127e73e721 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 24 Feb 2015 14:23:50 +0000 Subject: No need to explicitly pass 'web_client' in to create_resource_tree as it can be found via config --- synapse/app/homeserver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3649406efb..5df4ba23ab 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -110,7 +110,7 @@ class SynapseHomeServer(HomeServer): # so that :memory: sqlite works ) - def create_resource_tree(self, web_client, redirect_root_to_web_client): + def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. This in unduly complicated because Twisted does not support putting @@ -122,6 +122,8 @@ class SynapseHomeServer(HomeServer): location of the web client. This does nothing if web_client is not True. """ + web_client = self.get_config().webclient + # list containing (path_str, Resource) e.g: # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] desired_tree = [ @@ -340,7 +342,6 @@ def setup(config_options): ) hs.create_resource_tree( - web_client=config.webclient, redirect_root_to_web_client=True, ) -- cgit 1.5.1 From e07fc62833dbbfc9dda7c344f194fc070cfc587f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 24 Feb 2015 15:49:14 +0000 Subject: A trivial 'hello world'-style resource on /_synapse/metrics, with optional commandline flag --- synapse/app/homeserver.py | 7 +++++++ synapse/metrics/resource.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 synapse/metrics/resource.py (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5df4ba23ab..9747f7a015 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -147,6 +147,13 @@ class SynapseHomeServer(HomeServer): else: self.root_resource = Resource() + if self.get_config().enable_metrics: + from synapse.metrics.resource import ( + MetricsResource, METRICS_PREFIX + ) + + desired_tree.append((METRICS_PREFIX, MetricsResource(self))) + # ideally we'd just use getChild and putChild but getChild doesn't work # unless you give it a Request object IN ADDITION to the name :/ So # instead, we'll store a copy of this mapping so we can actually add diff --git a/synapse/metrics/resource.py b/synapse/metrics/resource.py new file mode 100644 index 0000000000..b480484624 --- /dev/null +++ b/synapse/metrics/resource.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + + +METRICS_PREFIX = "/_synapse/metrics" + + +class MetricsResource(Resource): + isLeaf = True + + def __init__(self, hs): + Resource.__init__(self) # Resource is old-style, so no super() + + self.hs = hs + + def render_GET(self, request): + response = "Hello, world!" + + request.setHeader("Content-Type", "text/plain") + request.setHeader("Content-Length", str(len(response))) + + return response -- cgit 1.5.1 From e0214a263b610f21d59d16cf84b5a3eee8485b28 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Mar 2015 15:33:53 +0000 Subject: Build MetricsResource as a specific HomeServer dependency --- synapse/app/homeserver.py | 16 ++++++++++------ synapse/server.py | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 9747f7a015..3801302c60 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -47,6 +47,7 @@ from synapse.crypto import context_factory from synapse.util.logcontext import LoggingContext from synapse.rest.client.v1 import ClientV1RestResource from synapse.rest.client.v2_alpha import ClientV2AlphaRestResource +from synapse.metrics.resource import MetricsResource, METRICS_PREFIX from daemonize import Daemonize import twisted.manhole.telnet @@ -100,6 +101,12 @@ class SynapseHomeServer(HomeServer): def build_resource_for_server_key(self): return LocalKey(self) + def build_resource_for_metrics(self): + if self.get_config().enable_metrics: + return MetricsResource(self) + else: + return None + def build_db_pool(self): return adbapi.ConnectionPool( "sqlite3", self.get_db_name(), @@ -147,12 +154,9 @@ class SynapseHomeServer(HomeServer): else: self.root_resource = Resource() - if self.get_config().enable_metrics: - from synapse.metrics.resource import ( - MetricsResource, METRICS_PREFIX - ) - - desired_tree.append((METRICS_PREFIX, MetricsResource(self))) + metrics_resource = self.get_resource_for_metrics() + if metrics_resource is not None: + desired_tree.append((METRICS_PREFIX, metrics_resource)) # ideally we'd just use getChild and putChild but getChild doesn't work # unless you give it a Request object IN ADDITION to the name :/ So diff --git a/synapse/server.py b/synapse/server.py index e3eefda4fc..c7772244ba 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -80,6 +80,7 @@ class BaseHomeServer(object): 'resource_for_server_key', 'resource_for_media_repository', 'resource_for_app_services', + 'resource_for_metrics', 'event_sources', 'ratelimiter', 'keyring', -- cgit 1.5.1 From a2cdd11d4a05f76b0be96d7e2de9294a9ff2cd57 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Mar 2015 15:51:33 +0000 Subject: Fold the slightly-odd bind_port/secure_port/etc.. logic into SynapseHomeServer.start_listening() --- synapse/app/homeserver.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3801302c60..2ba701f53f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -219,17 +219,20 @@ class SynapseHomeServer(HomeServer): """ return "%s-%s" % (resource, path_seg) - def start_listening(self, secure_port, unsecure_port): - if secure_port is not None: + def start_listening(self): + config = self.get_config() + + if not config.no_tls and config.bind_port is not None: reactor.listenSSL( - secure_port, Site(self.root_resource), self.tls_context_factory + config.bind_port, Site(self.root_resource), self.tls_context_factory ) - logger.info("Synapse now listening on port %d", secure_port) - if unsecure_port is not None: + logger.info("Synapse now listening on port %d", config.bind_port) + + if config.unsecure_port is not None: reactor.listenTCP( - unsecure_port, Site(self.root_resource) + config.unsecure_port, Site(self.root_resource) ) - logger.info("Synapse now listening on port %d", unsecure_port) + logger.info("Synapse now listening on port %d", config.unsecure_port) def get_version_string(): @@ -381,11 +384,7 @@ def setup(config_options): f.namespace['hs'] = hs reactor.listenTCP(config.manhole, f, interface='127.0.0.1') - bind_port = config.bind_port - if config.no_tls: - bind_port = None - - hs.start_listening(bind_port, config.unsecure_port) + hs.start_listening() hs.get_pusherpool().start() hs.get_state_handler().start_caching() -- cgit 1.5.1 From b98b4c135d9738d5cf701712fc244209651cddf7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 12 Mar 2015 16:05:46 +0000 Subject: Option to serve metrics from their own localhost-only TCP port instead of muxed on the main listener --- synapse/app/homeserver.py | 12 ++++++++++-- synapse/config/metrics.py | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2ba701f53f..aa7c722efc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -129,7 +129,8 @@ class SynapseHomeServer(HomeServer): location of the web client. This does nothing if web_client is not True. """ - web_client = self.get_config().webclient + config = self.get_config() + web_client = config.webclient # list containing (path_str, Resource) e.g: # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] @@ -155,7 +156,7 @@ class SynapseHomeServer(HomeServer): self.root_resource = Resource() metrics_resource = self.get_resource_for_metrics() - if metrics_resource is not None: + if config.metrics_port is None and metrics_resource is not None: desired_tree.append((METRICS_PREFIX, metrics_resource)) # ideally we'd just use getChild and putChild but getChild doesn't work @@ -234,6 +235,13 @@ class SynapseHomeServer(HomeServer): ) logger.info("Synapse now listening on port %d", config.unsecure_port) + metrics_resource = self.get_resource_for_metrics() + if metrics_resource and config.metrics_port is not None: + reactor.listenTCP( + config.metrics_port, Site(metrics_resource), interface="127.0.0.1", + ) + logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) + def get_version_string(): try: diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 90aba10bba..901a429c76 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -20,6 +20,7 @@ class MetricsConfig(Config): def __init__(self, args): super(MetricsConfig, self).__init__(args) self.enable_metrics = args.enable_metrics + self.metrics_port = args.metrics_port @classmethod def add_arguments(cls, parser): @@ -29,3 +30,7 @@ class MetricsConfig(Config): '--enable-metrics', dest="enable_metrics", action="store_true", help="Enable collection and rendering of performance metrics" ) + metrics_group.add_argument( + '--metrics-port', metavar="PORT", type=int, + help="Separate port to accept metrics requests on (on localhost)" + ) -- cgit 1.5.1 From c8032aec1794cb3ad79b4b61892917768b372bd6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 14 Mar 2015 00:12:20 +0000 Subject: actually uphold the bind_host parameter. in theory should make ipv6 binds work like bind_host: 'fe80::1%lo0' --- synapse/app/homeserver.py | 9 +++++++-- synapse/http/server.py | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index aa7c722efc..15c454af76 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -225,13 +225,18 @@ class SynapseHomeServer(HomeServer): if not config.no_tls and config.bind_port is not None: reactor.listenSSL( - config.bind_port, Site(self.root_resource), self.tls_context_factory + config.bind_port, + Site(self.root_resource), + self.tls_context_factory, + interface=config.bind_host ) logger.info("Synapse now listening on port %d", config.bind_port) if config.unsecure_port is not None: reactor.listenTCP( - config.unsecure_port, Site(self.root_resource) + config.unsecure_port, + Site(self.root_resource), + interface=config.bind_host ) logger.info("Synapse now listening on port %d", config.unsecure_port) diff --git a/synapse/http/server.py b/synapse/http/server.py index d77cb77799..f1376ee243 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -86,6 +86,7 @@ class JsonResource(HttpServer, resource.Resource): self.clock = hs.get_clock() self.path_regexs = {} self.version_string = hs.version_string + self.hs = hs def register_path(self, method, path_pattern, callback): self.path_regexs.setdefault(method, []).append( @@ -99,7 +100,11 @@ class JsonResource(HttpServer, resource.Resource): port (int): The port to listen on. """ - reactor.listenTCP(port, server.Site(self)) + reactor.listenTCP( + port, + server.Site(self), + interface=self.hs.config.bind_host + ) # Gets called by twisted def render(self, request): -- cgit 1.5.1 From 7564dac8cbb245581c4cba19717f1c30b431059e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 17 Mar 2015 12:45:37 +0100 Subject: Wire up the webclient option It existed but was hardcoded to True. Give it an underscore for consistency. Also don't pull in syweb unless we're actually using the web client. --- synapse/app/homeserver.py | 8 +++++--- synapse/config/server.py | 4 +++- synapse/python_dependencies.py | 20 ++++++++++++++++---- 3 files changed, 24 insertions(+), 8 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c454af76..500cae05fb 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -60,7 +60,6 @@ import re import resource import subprocess import sqlite3 -import syweb logger = logging.getLogger(__name__) @@ -83,6 +82,7 @@ class SynapseHomeServer(HomeServer): return AppServiceRestResource(self) def build_resource_for_web_client(self): + import syweb syweb_path = os.path.dirname(syweb.__file__) webclient_path = os.path.join(syweb_path, "webclient") return File(webclient_path) # TODO configurable? @@ -130,7 +130,7 @@ class SynapseHomeServer(HomeServer): True. """ config = self.get_config() - web_client = config.webclient + web_client = config.web_client # list containing (path_str, Resource) e.g: # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] @@ -343,7 +343,8 @@ def setup(config_options): config.setup_logging() - check_requirements() + # check any extra requirements we have now we have a config + check_requirements(config) version_string = get_version_string() @@ -450,6 +451,7 @@ def run(hs): def main(): with LoggingContext("main"): + # check base requirements check_requirements() hs = setup(sys.argv[1:]) run(hs) diff --git a/synapse/config/server.py b/synapse/config/server.py index b042d4eed9..58a828cc4c 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -28,7 +28,7 @@ class ServerConfig(Config): self.unsecure_port = args.unsecure_port self.daemonize = args.daemonize self.pid_file = self.abspath(args.pid_file) - self.webclient = True + self.web_client = args.web_client self.manhole = args.manhole self.soft_file_limit = args.soft_file_limit @@ -68,6 +68,8 @@ class ServerConfig(Config): server_group.add_argument('--pid-file', default="homeserver.pid", help="When running as a daemon, the file to" " store the pid in") + server_group.add_argument('--web_client', default=True, type=bool, + help="Whether or not to serve a web client") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, help="Turn on the twisted telnet manhole" diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 8a5849d960..e27ecbed28 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,6 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil>=0.0.3": ["syutil"], - "matrix_angular_sdk>=0.6.5": ["syweb>=0.6.5"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -18,6 +17,19 @@ REQUIREMENTS = { "pillow": ["PIL"], "pydenticon": ["pydenticon"], } +CONDITIONAL_REQUIREMENTS = { + "web_client": { + "matrix_angular_sdk>=0.6.5": ["syweb>=0.6.5"], + } +} + + +def requirements(config=None, include_conditional=False): + reqs = REQUIREMENTS.copy() + for key,req in CONDITIONAL_REQUIREMENTS.items(): + if (config and getattr(config, key)) or include_conditional: + reqs.update(req) + return reqs def github_link(project, version, egg): @@ -46,10 +58,10 @@ class MissingRequirementError(Exception): pass -def check_requirements(): +def check_requirements(config=None): """Checks that all the modules needed by synapse have been correctly installed and are at the correct version""" - for dependency, module_requirements in REQUIREMENTS.items(): + for dependency, module_requirements in requirements(config, include_conditional=False).items(): for module_requirement in module_requirements: if ">=" in module_requirement: module_name, required_version = module_requirement.split(">=") @@ -110,7 +122,7 @@ def list_requirements(): egg = link.split("#egg=")[1] linked.append(egg.split('-')[0]) result.append(link) - for requirement in REQUIREMENTS: + for requirement in requirements(include_conditional=True): is_linked = False for link in linked: if requirement.replace('-', '_').startswith(link): -- cgit 1.5.1