From c42ed47660e99026d50de833b0b06c75a3b25d37 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 11:52:52 +0100 Subject: Fix up create_resource_tree --- synapse/app/homeserver.py | 204 +++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 104 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 65a5dfa84e..79c7a8558d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -145,109 +145,6 @@ class SynapseHomeServer(HomeServer): **self.db_config.get("args", {}) ) - def create_resource_tree(self, redirect_root_to_web_client): - """Create the resource tree for this Home Server. - - This in unduly complicated because Twisted does not support putting - child resources more than 1 level deep at a time. - - Args: - web_client (bool): True to enable the web client. - redirect_root_to_web_client (bool): True to redirect '/' to the - location of the web client. This does nothing if web_client is not - True. - """ - config = self.get_config() - web_client = config.web_client - - # list containing (path_str, Resource) e.g: - # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] - desired_tree = [ - (CLIENT_PREFIX, self.get_resource_for_client()), - (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), - (FEDERATION_PREFIX, self.get_resource_for_federation()), - (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), - (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), - (SERVER_KEY_V2_PREFIX, self.get_resource_for_server_key_v2()), - (MEDIA_PREFIX, self.get_resource_for_media_repository()), - (STATIC_PREFIX, self.get_resource_for_static_content()), - ] - - if web_client: - logger.info("Adding the web client.") - desired_tree.append((WEB_CLIENT_PREFIX, - self.get_resource_for_web_client())) - - if web_client and redirect_root_to_web_client: - self.root_resource = RootRedirect(WEB_CLIENT_PREFIX) - else: - self.root_resource = Resource() - - metrics_resource = self.get_resource_for_metrics() - if config.metrics_port is None and metrics_resource is not None: - desired_tree.append((METRICS_PREFIX, metrics_resource)) - - # ideally we'd just use getChild and putChild but getChild doesn't work - # unless you give it a Request object IN ADDITION to the name :/ So - # instead, we'll store a copy of this mapping so we can actually add - # extra resources to existing nodes. See self._resource_id for the key. - resource_mappings = {} - for full_path, res in desired_tree: - logger.info("Attaching %s to path %s", res, full_path) - last_resource = self.root_resource - for path_seg in full_path.split('/')[1:-1]: - if path_seg not in last_resource.listNames(): - # resource doesn't exist, so make a "dummy resource" - child_resource = Resource() - last_resource.putChild(path_seg, child_resource) - res_id = self._resource_id(last_resource, path_seg) - resource_mappings[res_id] = child_resource - last_resource = child_resource - else: - # we have an existing Resource, use that instead. - res_id = self._resource_id(last_resource, path_seg) - last_resource = resource_mappings[res_id] - - # =========================== - # now attach the actual desired resource - last_path_seg = full_path.split('/')[-1] - - # if there is already a resource here, thieve its children and - # replace it - res_id = self._resource_id(last_resource, last_path_seg) - if res_id in resource_mappings: - # there is a dummy resource at this path already, which needs - # to be replaced with the desired resource. - existing_dummy_resource = resource_mappings[res_id] - for child_name in existing_dummy_resource.listNames(): - child_res_id = self._resource_id(existing_dummy_resource, - child_name) - child_resource = resource_mappings[child_res_id] - # steal the children - res.putChild(child_name, child_resource) - - # finally, insert the desired resource in the right place - last_resource.putChild(last_path_seg, res) - res_id = self._resource_id(last_resource, last_path_seg) - resource_mappings[res_id] = res - - return self.root_resource - - def _resource_id(self, resource, path_seg): - """Construct an arbitrary resource ID so you can retrieve the mapping - later. - - If you want to represent resource A putChild resource B with path C, - the mapping should looks like _resource_id(A,C) = B. - - Args: - resource (Resource): The *parent* Resource - path_seg (str): The name of the child Resource to be attached. - Returns: - str: A unique string which can be a key to the child Resource. - """ - return "%s-%s" % (resource, path_seg) - def start_listening(self): config = self.get_config() @@ -447,7 +344,26 @@ def setup(config_options): database_engine=database_engine, ) - hs.create_resource_tree( + resources = { + CLIENT_PREFIX: hs.get_resource_for_client(), + CLIENT_V2_ALPHA_PREFIX: hs.get_resource_for_client_v2_alpha(), + FEDERATION_PREFIX: hs.get_resource_for_federation(), + CONTENT_REPO_PREFIX: hs.get_resource_for_content_repo(), + SERVER_KEY_PREFIX: hs.get_resource_for_server_key(), + SERVER_KEY_V2_PREFIX: hs.get_resource_for_server_key_v2(), + MEDIA_PREFIX: hs.get_resource_for_media_repository(), + STATIC_PREFIX: hs.get_resource_for_static_content(), + } + + if config.web_client: + resources[WEB_CLIENT_PREFIX] = hs.get_resource_for_web_client() + + metrics_resource = hs.get_resource_for_metrics() + if config.metrics_port is None and metrics_resource is not None: + resources[METRICS_PREFIX] = metrics_resource + + hs.root_resource = create_resource_tree( + resources, redirect_root_to_web_client=True, ) @@ -525,6 +441,86 @@ class SynapseSite(Site): self.access_logger.info(line) +def create_resource_tree(desired_tree, redirect_root_to_web_client=True): + """Create the resource tree for this Home Server. + + This in unduly complicated because Twisted does not support putting + child resources more than 1 level deep at a time. + + Args: + web_client (bool): True to enable the web client. + redirect_root_to_web_client (bool): True to redirect '/' to the + location of the web client. This does nothing if web_client is not + True. + """ + if redirect_root_to_web_client and WEB_CLIENT_PREFIX in desired_tree: + root_resource = RootRedirect(WEB_CLIENT_PREFIX) + else: + root_resource = Resource() + + # ideally we'd just use getChild and putChild but getChild doesn't work + # unless you give it a Request object IN ADDITION to the name :/ So + # instead, we'll store a copy of this mapping so we can actually add + # extra resources to existing nodes. See self._resource_id for the key. + resource_mappings = {} + for full_path, res in desired_tree.items(): + logger.info("Attaching %s to path %s", res, full_path) + last_resource = root_resource + for path_seg in full_path.split('/')[1:-1]: + if path_seg not in last_resource.listNames(): + # resource doesn't exist, so make a "dummy resource" + child_resource = Resource() + last_resource.putChild(path_seg, child_resource) + res_id = _resource_id(last_resource, path_seg) + resource_mappings[res_id] = child_resource + last_resource = child_resource + else: + # we have an existing Resource, use that instead. + res_id = _resource_id(last_resource, path_seg) + last_resource = resource_mappings[res_id] + + # =========================== + # now attach the actual desired resource + last_path_seg = full_path.split('/')[-1] + + # if there is already a resource here, thieve its children and + # replace it + res_id = _resource_id(last_resource, last_path_seg) + if res_id in resource_mappings: + # there is a dummy resource at this path already, which needs + # to be replaced with the desired resource. + existing_dummy_resource = resource_mappings[res_id] + for child_name in existing_dummy_resource.listNames(): + child_res_id = _resource_id(existing_dummy_resource, + child_name) + child_resource = resource_mappings[child_res_id] + # steal the children + res.putChild(child_name, child_resource) + + # finally, insert the desired resource in the right place + last_resource.putChild(last_path_seg, res) + res_id = _resource_id(last_resource, last_path_seg) + resource_mappings[res_id] = res + + return root_resource + + +def _resource_id(resource, path_seg): + """Construct an arbitrary resource ID so you can retrieve the mapping + later. + + If you want to represent resource A putChild resource B with path C, + the mapping should looks like _resource_id(A,C) = B. + + Args: + resource (Resource): The *parent* Resource + path_seg (str): The name of the child Resource to be attached. + Returns: + str: A unique string which can be a key to the child Resource. + """ + return "%s-%s" % (resource, path_seg) + + def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: -- cgit 1.4.1 From fd2c07bfed9182f4fc3ccf50b4390645fe038da4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 15:33:07 +0100 Subject: Use config.listeners --- synapse/app/homeserver.py | 150 ++++++++++++++++++++++++++-------------------- synapse/config/server.py | 114 +++++++++++++++++++++++++---------- 2 files changed, 168 insertions(+), 96 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 79c7a8558d..4228bac673 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -87,16 +87,10 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - res = ClientV1RestResource(self) - if self.config.gzip_responses: - res = gz_wrap(res) - return res + return ClientV1RestResource(self) def build_resource_for_client_v2_alpha(self): - res = ClientV2AlphaRestResource(self) - if self.config.gzip_responses: - res = gz_wrap(res) - return res + return ClientV2AlphaRestResource(self) def build_resource_for_federation(self): return JsonResource(self) @@ -145,49 +139,102 @@ class SynapseHomeServer(HomeServer): **self.db_config.get("args", {}) ) - def start_listening(self): - config = self.get_config() + def _listener_http(self, config, listener_config): + port = listener_config["port"] + bind_address = listener_config.get("bind_address", "") + tls = listener_config.get("tls", False) + + if tls and config.no_tls: + return - if not config.no_tls and config.bind_port is not None: + metrics_resource = self.get_resource_for_metrics() + + resources = {} + for res in listener_config["resources"]: + for name in res["names"]: + if name == "client": + if res["compress"]: + client_v1 = gz_wrap(self.get_resource_for_client()) + client_v2 = gz_wrap(self.get_resource_for_client_v2_alpha()) + else: + client_v1 = self.get_resource_for_client() + client_v2 = self.get_resource_for_client_v2_alpha() + + resources.update({ + CLIENT_PREFIX: client_v1, + CLIENT_V2_ALPHA_PREFIX: client_v2, + }) + + if name == "federation": + resources.update({ + FEDERATION_PREFIX: self.get_resource_for_federation(), + }) + + if name in ["static", "client"]: + resources.update({ + STATIC_PREFIX: self.get_resource_for_static_content(), + }) + + if name in ["media", "federation", "client"]: + resources.update({ + MEDIA_PREFIX: self.get_resource_for_media_repository(), + CONTENT_REPO_PREFIX: self.get_resource_for_content_repo(), + }) + + if name in ["keys", "federation"]: + resources.update({ + SERVER_KEY_PREFIX: self.get_resource_for_server_key(), + SERVER_KEY_V2_PREFIX: self.get_resource_for_server_key_v2(), + }) + + if name == "webclient": + resources[WEB_CLIENT_PREFIX] = self.get_resource_for_web_client() + + if name == "metrics" and metrics_resource: + resources[METRICS_PREFIX] = metrics_resource + + root_resource = create_resource_tree(resources) + if tls: reactor.listenSSL( - config.bind_port, + port, SynapseSite( "synapse.access.https", config, - self.root_resource, + root_resource, ), self.tls_context_factory, - interface=config.bind_host + interface=bind_address ) - logger.info("Synapse now listening on port %d", config.bind_port) - - if config.unsecure_port is not None: + else: reactor.listenTCP( - config.unsecure_port, + port, SynapseSite( - "synapse.access.http", + "synapse.access.https", config, - self.root_resource, + root_resource, ), - interface=config.bind_host + interface=bind_address ) - logger.info("Synapse now listening on port %d", config.unsecure_port) + logger.info("Synapse now listening on port %d", port) - metrics_resource = self.get_resource_for_metrics() - if metrics_resource and config.metrics_port is not None: - reactor.listenTCP( - config.metrics_port, - SynapseSite( - "synapse.access.metrics", - config, - metrics_resource, - ), - interface=config.metrics_bind_host, - ) - logger.info( - "Metrics now running on %s port %d", - config.metrics_bind_host, config.metrics_port, - ) + def start_listening(self): + config = self.get_config() + + for listener in config.listeners: + if listener["type"] == "http": + self._listener_http(config, listener) + elif listener["type"] == "manhole": + f = twisted.manhole.telnet.ShellFactory() + f.username = "matrix" + f.password = "rabbithole" + f.namespace['hs'] = self + reactor.listenTCP( + listener["port"], + f, + interface=listener.get("bind_address", '127.0.0.1') + ) + else: + logger.warn("Unrecognized listener type: %s", listener["type"]) def run_startup_checks(self, db_conn, database_engine): all_users_native = are_all_users_on_domain( @@ -322,11 +369,6 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - if re.search(":[0-9]+$", config.server_name): - domain_with_port = config.server_name - else: - domain_with_port = "%s:%s" % (config.server_name, config.bind_port) - tls_context_factory = context_factory.ServerContextFactory(config) database_engine = create_engine(config.database_config["name"]) @@ -334,7 +376,6 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, - domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_config=config.database_config, tls_context_factory=tls_context_factory, @@ -344,29 +385,6 @@ def setup(config_options): database_engine=database_engine, ) - resources = { - CLIENT_PREFIX: hs.get_resource_for_client(), - CLIENT_V2_ALPHA_PREFIX: hs.get_resource_for_client_v2_alpha(), - FEDERATION_PREFIX: hs.get_resource_for_federation(), - CONTENT_REPO_PREFIX: hs.get_resource_for_content_repo(), - SERVER_KEY_PREFIX: hs.get_resource_for_server_key(), - SERVER_KEY_V2_PREFIX: hs.get_resource_for_server_key_v2(), - MEDIA_PREFIX: hs.get_resource_for_media_repository(), - STATIC_PREFIX: hs.get_resource_for_static_content(), - } - - if config.web_client: - resources[WEB_CLIENT_PREFIX] = hs.get_resource_for_web_client() - - metrics_resource = hs.get_resource_for_metrics() - if config.metrics_port is None and metrics_resource is not None: - resources[METRICS_PREFIX] = metrics_resource - - hs.root_resource = create_resource_tree( - resources, - redirect_root_to_web_client=True, - ) - logger.info("Preparing database: %r...", config.database_config) try: diff --git a/synapse/config/server.py b/synapse/config/server.py index 022ebcea94..26017c7efa 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -20,26 +20,73 @@ class ServerConfig(Config): def read_config(self, config): self.server_name = config["server_name"] - self.bind_port = config["bind_port"] - self.bind_host = config["bind_host"] - self.unsecure_port = config["unsecure_port"] self.manhole = config.get("manhole") self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.use_frozen_dicts = config.get("use_frozen_dicts", True) - self.gzip_responses = config["gzip_responses"] + + self.listeners = config.get("listeners", []) + + bind_port = config.get("bind_port") + if bind_port: + self.listeners = [] + bind_host = config.get("bind_host", "") + gzip_responses = config.get("gzip_responses", True) + + self.listeners.append({ + "port": bind_port, + "bind_address": bind_host, + "tls": True, + "type": "http", + "resources": [ + { + "names": ["client", "webclient"], + "compress": gzip_responses, + }, + { + "names": ["federation"], + "compress": False, + } + ] + }) + + unsecure_port = config.get("unsecure_port", bind_port - 400) + if unsecure_port: + self.listeners.append({ + "port": unsecure_port, + "bind_address": bind_host, + "tls": False, + "type": "http", + "resources": [ + { + "names": ["client", "webclient"], + "compress": gzip_responses, + }, + { + "names": ["federation"], + "compress": False, + } + ] + }) # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") if not content_addr: + for listener in self.listeners: + if listener["type"] == "http" and not listener.get("tls", False): + unsecure_port = listener["port"] + break + else: + raise RuntimeError("Could not determine 'content_addr'") + host = self.server_name if ':' not in host: - host = "%s:%d" % (host, self.unsecure_port) + host = "%s:%d" % (host, unsecure_port) else: host = host.split(':')[0] - host = "%s:%d" % (host, self.unsecure_port) + host = "%s:%d" % (host, unsecure_port) content_addr = "http://%s" % (host,) self.content_addr = content_addr @@ -61,9 +108,17 @@ class ServerConfig(Config): # e.g. matrix.org, localhost:8080, etc. server_name: "%(server_name)s" + # The port to listen for HTTPS requests on. + # For when matrix traffic is sent directly to synapse. + # bind_port: %(bind_port)s + + # The port to listen for HTTP requests on. + # For when matrix traffic passes through loadbalancer that unwraps TLS. + # unsecure_port: %(unsecure_port)s + # Local interface to listen on. # The empty string will cause synapse to listen on all interfaces. - bind_host: "" + # bind_host: "" # When running as a daemon, the file to store the pid in pid_file: %(pid_file)s @@ -83,31 +138,30 @@ class ServerConfig(Config): # Should synapse compress HTTP responses to clients that support it? # This should be disabled if running synapse behind a load balancer # that can do automatic compression. - gzip_responses: True + # gzip_responses: True listeners: - # For when matrix traffic is sent directly to synapse. - secure: - # The type of - type: http_resource - - # The port to listen for HTTPS requests on. - port: %(bind_port)s - - # Is this a TLS socket? - tls: true - - # Local interface to listen on. - # The empty string will cause synapse to listen on all interfaces. - bind_address: "" - - # For when matrix traffic passes through loadbalancer that unwraps TLS. - unsecure: - port: %(unsecure_port)s - tls: false - bind_address: "" - - + - port: %(unsecure_port)s + tls: false + bind_address: '' + type: http + + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false + + - port: %(bind_port)s + tls: true + bind_address: '' + type: http + + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false """ % locals() def read_arguments(self, args): -- cgit 1.4.1 From 9c5fc81c2ddd29eac62d368e7f8d24972f8894a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:13:23 +0100 Subject: Correctly handle x_forwaded listener option --- synapse/app/homeserver.py | 31 +++++++++++++++++++++++++++---- synapse/config/server.py | 2 ++ synapse/server.py | 12 ++---------- 3 files changed, 31 insertions(+), 14 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 4228bac673..12da0bc4b5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -34,7 +34,7 @@ from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.web.static import File -from twisted.web.server import Site, GzipEncoderFactory +from twisted.web.server import Site, GzipEncoderFactory, Request from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -199,7 +199,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", - config, + listener_config, root_resource, ), self.tls_context_factory, @@ -210,7 +210,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", - config, + listener_config, root_resource, ), interface=bind_address @@ -441,6 +441,28 @@ class SynapseService(service.Service): return self._port.stopListening() +class XForwardedForRequest(Request): + def __init__(self, *args, **kw): + Request.__init__(self, *args, **kw) + + """ + Add a layer on top of another request that only uses the value of an + X-Forwarded-For header as the result of C{getClientIP}. + """ + def getClientIP(self): + """ + @return: The client address (the first address) in the value of the + I{X-Forwarded-For header}. If the header is not present, return + C{b"-"}. + """ + return self.requestHeaders.getRawHeaders( + b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip() + + +def XForwardedFactory(*args, **kwargs): + return XForwardedForRequest(*args, **kwargs) + + class SynapseSite(Site): """ Subclass of a twisted http Site that does access logging with python's @@ -448,7 +470,8 @@ class SynapseSite(Site): """ def __init__(self, logger_name, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) - if config.captcha_ip_origin_is_x_forwarded: + if config.get("x_forwarded", False): + self.requestFactory = XForwardedFactory self._log_formatter = proxiedLogFormatter else: self._log_formatter = combinedLogFormatter diff --git a/synapse/config/server.py b/synapse/config/server.py index 26017c7efa..9dab167b21 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -157,6 +157,8 @@ class ServerConfig(Config): bind_address: '' type: http + x_forwarded: False + resources: - names: [client, webclient] compress: true diff --git a/synapse/server.py b/synapse/server.py index 8b3dc675cc..4d1fb1cbf6 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -132,16 +132,8 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) def get_ip_from_request(self, request): - # May be an X-Forwarding-For header depending on config - ip_addr = request.getClientIP() - if self.config.captcha_ip_origin_is_x_forwarded: - # use the header - if request.requestHeaders.hasHeader("X-Forwarded-For"): - ip_addr = request.requestHeaders.getRawHeaders( - "X-Forwarded-For" - )[0] - - return ip_addr + # X-Forwarded-For is handled by our custom request type. + return request.getClientIP() def is_mine(self, domain_specific_string): return domain_specific_string.domain == self.hostname -- cgit 1.4.1 From 942e39e87c735a6ead5375681ceea035a945fd7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:13:54 +0100 Subject: PEP8 --- synapse/app/homeserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 12da0bc4b5..91cc06a49e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -63,7 +63,6 @@ import synapse import logging import os -import re import resource import subprocess @@ -532,8 +531,9 @@ def create_resource_tree(desired_tree, redirect_root_to_web_client=True): # to be replaced with the desired resource. existing_dummy_resource = resource_mappings[res_id] for child_name in existing_dummy_resource.listNames(): - child_res_id = _resource_id(existing_dummy_resource, - child_name) + child_res_id = _resource_id( + existing_dummy_resource, child_name + ) child_resource = resource_mappings[child_res_id] # steal the children res.putChild(child_name, child_resource) -- cgit 1.4.1 From a005b7269a9c1df517f6a01e72c1eea5f4cb0354 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:41:36 +0100 Subject: Add backwards compat support for metrics, manhole and webclient config options --- synapse/app/homeserver.py | 7 ------- synapse/config/metrics.py | 6 ------ synapse/config/server.py | 30 +++++++++++++++++++++++++++--- 3 files changed, 27 insertions(+), 16 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 91cc06a49e..95e9122d3e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -408,13 +408,6 @@ def setup(config_options): logger.info("Database prepared in %r.", config.database_config) - if config.manhole: - f = twisted.manhole.telnet.ShellFactory() - f.username = "matrix" - f.password = "rabbithole" - f.namespace['hs'] = hs - reactor.listenTCP(config.manhole, f, interface='127.0.0.1') - hs.start_listening() hs.get_pusherpool().start() diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 0cfb30ce7f..ae5a691527 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -28,10 +28,4 @@ class MetricsConfig(Config): # Enable collection and rendering of performance metrics enable_metrics: False - - # Separate port to accept metrics requests on - # metrics_port: 8081 - - # Which host to bind the metric listener to - # metrics_bind_host: 127.0.0.1 """ diff --git a/synapse/config/server.py b/synapse/config/server.py index 9dab167b21..95bc967d0e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -20,7 +20,6 @@ class ServerConfig(Config): def read_config(self, config): self.server_name = config["server_name"] - self.manhole = config.get("manhole") self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] @@ -35,6 +34,8 @@ class ServerConfig(Config): bind_host = config.get("bind_host", "") gzip_responses = config.get("gzip_responses", True) + names = ["client", "webclient"] if self.web_client else ["client"] + self.listeners.append({ "port": bind_port, "bind_address": bind_host, @@ -42,7 +43,7 @@ class ServerConfig(Config): "type": "http", "resources": [ { - "names": ["client", "webclient"], + "names": names, "compress": gzip_responses, }, { @@ -61,7 +62,7 @@ class ServerConfig(Config): "type": "http", "resources": [ { - "names": ["client", "webclient"], + "names": names, "compress": gzip_responses, }, { @@ -71,6 +72,29 @@ class ServerConfig(Config): ] }) + manhole = config.get("manhole") + if manhole: + self.listeners.append({ + "port": manhole, + "bind_address": "127.0.0.1", + "type": "manhole", + }) + + metrics_port = config.get("metrics_port") + if metrics_port: + self.listeners.append({ + "port": metrics_port, + "bind_address": config.get("metrics_bind_host", "127.0.0.1"), + "tls": False, + "type": "http", + "resources": [ + { + "names": ["metrics"], + "compress": False, + }, + ] + }) + # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") if not content_addr: -- cgit 1.4.1 From b5209c57441d9e7bace28a03774d2605a6572514 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 16:36:49 +0100 Subject: Create SynapseRequest that overrides __repr__ to not print access_token --- synapse/app/homeserver.py | 49 +++++++++++++++++++++++++++++++++++++++++------ synapse/http/server.py | 14 +++----------- 2 files changed, 46 insertions(+), 17 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95e9122d3e..7c1ad6bc13 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -63,6 +63,7 @@ import synapse import logging import os +import re import resource import subprocess @@ -433,9 +434,34 @@ class SynapseService(service.Service): return self._port.stopListening() -class XForwardedForRequest(Request): - def __init__(self, *args, **kw): +class SynapseRequest(Request): + def __init__(self, site_tag, *args, **kw): Request.__init__(self, *args, **kw) + self.site_tag = site_tag + self.authenticated_entity = None + + def __repr__(self): + # We overwrite this so that we don't log ``access_token`` + return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % ( + self.__class__.__name__, + id(self), + self.method, + self.get_redacted_uri(), + self.clientproto, + self.site_tag, + ) + + def get_redacted_uri(self): + return re.sub( + r'(\?.*access_token=)[^&]*(.*)$', + r'\1\2', + self.uri + ) + + +class XForwardedForRequest(SynapseRequest): + def __init__(self, *args, **kw): + SynapseRequest.__init__(self, *args, **kw) """ Add a layer on top of another request that only uses the value of an @@ -451,8 +477,16 @@ class XForwardedForRequest(Request): b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip() -def XForwardedFactory(*args, **kwargs): - return XForwardedForRequest(*args, **kwargs) +class SynapseRequestFactory(object): + def __init__(self, site_tag, x_forwarded_for): + self.site_tag = site_tag + self.x_forwarded_for = x_forwarded_for + + def __call__(self, *args, **kwargs): + if self.x_forwarded_for: + return XForwardedForRequest(self.site_tag, *args, **kwargs) + else: + return SynapseRequest(self.site_tag, *args, **kwargs) class SynapseSite(Site): @@ -462,8 +496,11 @@ class SynapseSite(Site): """ def __init__(self, logger_name, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) - if config.get("x_forwarded", False): - self.requestFactory = XForwardedFactory + + proxied = config.get("x_forwarded", False) + self.requestFactory = SynapseRequestFactory(None, proxied) + + if proxied: self._log_formatter = proxiedLogFormatter else: self._log_formatter = combinedLogFormatter diff --git a/synapse/http/server.py b/synapse/http/server.py index e6e8a59f6c..7f8b9dbb29 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -32,7 +32,6 @@ from twisted.web.util import redirectTo import collections import logging -import re import urllib logger = logging.getLogger(__name__) @@ -83,18 +82,11 @@ def request_handler(request_handler): code = None start = self.clock.time_msec() try: - request_uri = request.uri - - # Don't log access_tokens - request_uri = re.sub( - r'(\?.*access_token=)[^&]*(.*)$', - r'\1\2', - request_uri - ) - logger.info( "%s - Received request: %s %s", - request.getClientIP(), request.method, request_uri + request.getClientIP(), + request.method, + request.get_redacted_uri() ) d = request_handler(self, request) with PreserveLoggingContext(): -- cgit 1.4.1 From cee69441d3d3b4d966b6ec69c7dbf4eb3b876bb3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 17:11:44 +0100 Subject: Log more when we have processed the request --- synapse/api/auth.py | 2 ++ synapse/app/homeserver.py | 10 ++++++++-- synapse/federation/transport/server.py | 1 + synapse/http/server.py | 14 ++++++++++++-- 4 files changed, 23 insertions(+), 4 deletions(-) (limited to 'synapse/app') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d5bf0be85c..4da62e5d8d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -370,6 +370,8 @@ class Auth(object): user_agent=user_agent ) + request.authenticated_entity = user.to_string() + defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError( diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7c1ad6bc13..fca6f06e3b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -143,6 +143,7 @@ class SynapseHomeServer(HomeServer): port = listener_config["port"] bind_address = listener_config.get("bind_address", "") tls = listener_config.get("tls", False) + site_tag = listener_config.get("tag", port) if tls and config.no_tls: return @@ -199,6 +200,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", + site_tag, listener_config, root_resource, ), @@ -210,6 +212,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", + site_tag, listener_config, root_resource, ), @@ -458,6 +461,9 @@ class SynapseRequest(Request): self.uri ) + def get_user_agent(self): + return self.requestHeaders.getRawHeaders("User-Agent", [None])[-1] + class XForwardedForRequest(SynapseRequest): def __init__(self, *args, **kw): @@ -494,11 +500,11 @@ class SynapseSite(Site): Subclass of a twisted http Site that does access logging with python's standard logging """ - def __init__(self, logger_name, config, resource, *args, **kwargs): + def __init__(self, logger_name, tag, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) proxied = config.get("x_forwarded", False) - self.requestFactory = SynapseRequestFactory(None, proxied) + self.requestFactory = SynapseRequestFactory(tag, proxied) if proxied: self._log_formatter = proxiedLogFormatter diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 31190e700a..bad93c6b2f 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -94,6 +94,7 @@ class TransportLayerServer(object): yield self.keyring.verify_json_for_server(origin, json_request) logger.info("Request from %s", origin) + request.authenticated_entity = origin defer.returnValue((origin, content)) diff --git a/synapse/http/server.py b/synapse/http/server.py index 7f8b9dbb29..34645a371a 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -125,8 +125,18 @@ def request_handler(request_handler): code = str(code) if code else "-" end = self.clock.time_msec() logger.info( - "Processed request: %dms %s %s %s", - end-start, code, request.method, request.path + "%s - %s - {%s}" + " Processed request: %dms %sB %s \"%s %s %s\" \"%s\"", + request.getClientIP(), + request.site_tag, + request.authenticated_entity, + end-start, + request.sentLength, + code, + request.method, + request.get_redacted_uri(), + request.clientproto, + request.get_user_agent(), ) return wrapped_request_handler -- cgit 1.4.1 From aaa749d366f768dd164f899c1d8e5eedd44ee5f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 18:18:05 +0100 Subject: Disable twisted access logging. Move access logging to SynapseRequest object --- synapse/app/homeserver.py | 64 ++++++++++++++++++++++++--------- synapse/http/server.py | 90 +++++++++++++++++------------------------------ 2 files changed, 79 insertions(+), 75 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fca6f06e3b..7effedf7dc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,7 +35,6 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.web.static import File from twisted.web.server import Site, GzipEncoderFactory, Request -from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource @@ -61,11 +60,13 @@ import twisted.manhole.telnet import synapse +import contextlib import logging import os import re import resource import subprocess +import time logger = logging.getLogger("synapse.app.homeserver") @@ -438,10 +439,11 @@ class SynapseService(service.Service): class SynapseRequest(Request): - def __init__(self, site_tag, *args, **kw): + def __init__(self, site, *args, **kw): Request.__init__(self, *args, **kw) - self.site_tag = site_tag + self.site = site self.authenticated_entity = None + self.start_time = 0 def __repr__(self): # We overwrite this so that we don't log ``access_token`` @@ -451,7 +453,7 @@ class SynapseRequest(Request): self.method, self.get_redacted_uri(), self.clientproto, - self.site_tag, + self.site.site_tag, ) def get_redacted_uri(self): @@ -464,6 +466,38 @@ class SynapseRequest(Request): def get_user_agent(self): return self.requestHeaders.getRawHeaders("User-Agent", [None])[-1] + def started_processing(self): + self.site.access_logger.info( + "%s - %s - Received request: %s %s", + self.getClientIP(), + self.site.site_tag, + self.method, + self.get_redacted_uri() + ) + self.start_time = int(time.time() * 1000) + + def finished_processing(self): + self.site.access_logger.info( + "%s - %s - {%s}" + " Processed request: %dms %sB %s \"%s %s %s\" \"%s\"", + self.getClientIP(), + self.site.site_tag, + self.authenticated_entity, + int(time.time() * 1000) - self.start_time, + self.sentLength, + self.code, + self.method, + self.get_redacted_uri(), + self.clientproto, + self.get_user_agent(), + ) + + @contextlib.contextmanager + def processing(self): + self.started_processing() + yield + self.finished_processing() + class XForwardedForRequest(SynapseRequest): def __init__(self, *args, **kw): @@ -484,15 +518,15 @@ class XForwardedForRequest(SynapseRequest): class SynapseRequestFactory(object): - def __init__(self, site_tag, x_forwarded_for): - self.site_tag = site_tag + def __init__(self, site, x_forwarded_for): + self.site = site self.x_forwarded_for = x_forwarded_for def __call__(self, *args, **kwargs): if self.x_forwarded_for: - return XForwardedForRequest(self.site_tag, *args, **kwargs) + return XForwardedForRequest(self.site, *args, **kwargs) else: - return SynapseRequest(self.site_tag, *args, **kwargs) + return SynapseRequest(self.site, *args, **kwargs) class SynapseSite(Site): @@ -500,21 +534,17 @@ class SynapseSite(Site): Subclass of a twisted http Site that does access logging with python's standard logging """ - def __init__(self, logger_name, tag, config, resource, *args, **kwargs): + def __init__(self, logger_name, site_tag, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) - proxied = config.get("x_forwarded", False) - self.requestFactory = SynapseRequestFactory(tag, proxied) + self.site_tag = site_tag - if proxied: - self._log_formatter = proxiedLogFormatter - else: - self._log_formatter = combinedLogFormatter + proxied = config.get("x_forwarded", False) + self.requestFactory = SynapseRequestFactory(self, proxied) self.access_logger = logging.getLogger(logger_name) def log(self, request): - line = self._log_formatter(self._logDateTime, request) - self.access_logger.info(line) + pass def create_resource_tree(desired_tree, redirect_root_to_web_client=True): diff --git a/synapse/http/server.py b/synapse/http/server.py index 34645a371a..807ff95c65 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -79,65 +79,39 @@ def request_handler(request_handler): _next_request_id += 1 with LoggingContext(request_id) as request_context: request_context.request = request_id - code = None - start = self.clock.time_msec() - try: - logger.info( - "%s - Received request: %s %s", - request.getClientIP(), - request.method, - request.get_redacted_uri() - ) - d = request_handler(self, request) - with PreserveLoggingContext(): - yield d - code = request.code - except CodeMessageException as e: - code = e.code - if isinstance(e, SynapseError): - logger.info( - "%s SynapseError: %s - %s", request, code, e.msg + with request.processing(): + try: + d = request_handler(self, request) + with PreserveLoggingContext(): + yield d + except CodeMessageException as e: + code = e.code + if isinstance(e, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, e.msg + ) + else: + logger.exception(e) + outgoing_responses_counter.inc(request.method, str(code)) + respond_with_json( + request, code, cs_exception(e), send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + except: + logger.exception( + "Failed handle request %s.%s on %r: %r", + request_handler.__module__, + request_handler.__name__, + self, + request + ) + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True ) - else: - logger.exception(e) - outgoing_responses_counter.inc(request.method, str(code)) - respond_with_json( - request, code, cs_exception(e), send_cors=True, - pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, - ) - except: - code = 500 - logger.exception( - "Failed handle request %s.%s on %r: %r", - request_handler.__module__, - request_handler.__name__, - self, - request - ) - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) - finally: - code = str(code) if code else "-" - end = self.clock.time_msec() - logger.info( - "%s - %s - {%s}" - " Processed request: %dms %sB %s \"%s %s %s\" \"%s\"", - request.getClientIP(), - request.site_tag, - request.authenticated_entity, - end-start, - request.sentLength, - code, - request.method, - request.get_redacted_uri(), - request.clientproto, - request.get_user_agent(), - ) return wrapped_request_handler -- cgit 1.4.1 From 9e5a353663c111cfe34b727f75eed3616201ef72 Mon Sep 17 00:00:00 2001 From: Eric Myhre Date: Thu, 18 Jun 2015 23:38:20 -0500 Subject: Make upload dir a configurable path. Fixes SYN-425. Signed-off-by: Eric Myhre --- synapse/app/homeserver.py | 3 +-- synapse/config/repository.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95e9122d3e..9aec23d065 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -112,7 +112,7 @@ class SynapseHomeServer(HomeServer): def build_resource_for_content_repo(self): return ContentRepoResource( - self, self.upload_dir, self.auth, self.content_addr + self, self.config.uploads_path, self.auth, self.content_addr ) def build_resource_for_media_repository(self): @@ -375,7 +375,6 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, - upload_dir=os.path.abspath("uploads"), db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, diff --git a/synapse/config/repository.py b/synapse/config/repository.py index adaf4e4bb2..6891abd71d 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -21,13 +21,18 @@ class ContentRepositoryConfig(Config): self.max_upload_size = self.parse_size(config["max_upload_size"]) self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) + self.uploads_path = self.ensure_directory(config["uploads_path"]) def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") + uploads_path = self.default_path("uploads") return """ # Directory where uploaded images and attachments are stored. media_store_path: "%(media_store)s" + # Directory where in-progress uploads are stored. + uploads_path: "%(uploads_path)s" + # The largest allowed upload size in bytes max_upload_size: "10M" -- cgit 1.4.1 From eb928c9f52697bef99b982a1aa5229a20e65c447 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 10:16:48 +0100 Subject: Add site_tag to logger --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7effedf7dc..c02fc889c8 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -200,7 +200,7 @@ class SynapseHomeServer(HomeServer): reactor.listenSSL( port, SynapseSite( - "synapse.access.https", + "synapse.access.https.%s" % (site_tag,), site_tag, listener_config, root_resource, @@ -212,7 +212,7 @@ class SynapseHomeServer(HomeServer): reactor.listenTCP( port, SynapseSite( - "synapse.access.https", + "synapse.access.http.%s" % (site_tag,), site_tag, listener_config, root_resource, -- cgit 1.4.1