From 455579ca90dd5479dae785b5a1b9bdd201654ea6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:55 +0000 Subject: Make database selection configurable --- synapse/config/database.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'synapse/config') diff --git a/synapse/config/database.py b/synapse/config/database.py index 87efe54645..8dc9873f8c 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -26,6 +26,11 @@ class DatabaseConfig(Config): self.database_path = self.abspath(args.database_path) self.event_cache_size = self.parse_size(args.event_cache_size) + if args.database_config: + self.database_config = self.abspath(args.database_config) + else: + self.database_config = None + @classmethod def add_arguments(cls, parser): super(DatabaseConfig, cls).add_arguments(parser) @@ -38,6 +43,10 @@ class DatabaseConfig(Config): "--event-cache-size", default="100K", help="Number of events to cache in memory." ) + db_group.add_argument( + "--database-config", default=None, + help="Location of the database configuration file." + ) @classmethod def generate_config(cls, args, config_dir_path): -- cgit 1.4.1 From 8366fde82f609448bd96a882c72cea7d2baa52f0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 30 Mar 2015 12:01:09 -0400 Subject: turn --disable-registration into --enable-registration, given the default is for registration to be disabled by default now. this is backwards incompatible by removing the old --disable-registration arg, but makes for a much more intuitive arg --- synapse/config/registration.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 4401e774d1..a6a2d2c5e1 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -25,11 +25,11 @@ class RegistrationConfig(Config): def __init__(self, args): super(RegistrationConfig, self).__init__(args) - # `args.disable_registration` may either be a bool or a string depending - # on if the option was given a value (e.g. --disable-registration=false - # would set `args.disable_registration` to "false" not False.) - self.disable_registration = bool( - distutils.util.strtobool(str(args.disable_registration)) + # `args.enable_registration` may either be a bool or a string depending + # on if the option was given a value (e.g. --enable-registration=true + # would set `args.enable_registration` to "true" not True.) + self.disable_registration = not bool( + distutils.util.strtobool(str(args.enable_registration)) ) self.registration_shared_secret = args.registration_shared_secret @@ -39,11 +39,11 @@ class RegistrationConfig(Config): reg_group = parser.add_argument_group("registration") reg_group.add_argument( - "--disable-registration", - const=True, - default=True, + "--enable-registration", + const=False, + default=False, nargs='?', - help="Disable registration of new users.", + help="Enable registration for new users.", ) reg_group.add_argument( "--registration-shared-secret", type=str, @@ -53,8 +53,8 @@ class RegistrationConfig(Config): @classmethod def generate_config(cls, args, config_dir_path): - if args.disable_registration is None: - args.disable_registration = True + if args.enable_registration is None: + args.enable_registration = False if args.registration_shared_secret is None: args.registration_shared_secret = random_string_with_symbols(50) -- cgit 1.4.1 From 59bf16eddcb793705ee6bc243a2158824f7e05c8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:13:10 +0100 Subject: New registration for C/S API v2. Only ReCAPTCHA working currently. --- synapse/api/constants.py | 2 + synapse/config/captcha.py | 7 ++- synapse/handlers/auth.py | 90 +++++++++++++++++++++++++++----- synapse/handlers/register.py | 11 +++- synapse/http/client.py | 2 + synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/_base.py | 6 +++ synapse/rest/client/v2_alpha/register.py | 86 ++++++++++++++++++++++++++++++ 8 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/register.py (limited to 'synapse/config') diff --git a/synapse/api/constants.py b/synapse/api/constants.py index b16bf4247d..3e0ce170a4 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -62,6 +62,8 @@ class LoginType(object): APPLICATION_SERVICE = u"m.login.application_service" SHARED_SECRET = u"org.matrix.login.shared_secret" + HIDDEN_TYPES = [APPLICATION_SERVICE, SHARED_SECRET] + class EventTypes(object): Member = "m.room.member" diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 7e21c7414d..07fbfadc0f 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -20,6 +20,7 @@ class CaptchaConfig(Config): def __init__(self, args): super(CaptchaConfig, self).__init__(args) self.recaptcha_private_key = args.recaptcha_private_key + self.recaptcha_public_key = args.recaptcha_public_key self.enable_registration_captcha = args.enable_registration_captcha self.captcha_ip_origin_is_x_forwarded = ( args.captcha_ip_origin_is_x_forwarded @@ -30,9 +31,13 @@ class CaptchaConfig(Config): def add_arguments(cls, parser): super(CaptchaConfig, cls).add_arguments(parser) group = parser.add_argument_group("recaptcha") + group.add_argument( + "--recaptcha-public-key", type=str, default="YOUR_PUBLIC_KEY", + help="This Home Server's ReCAPTCHA public key." + ) group.add_argument( "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY", - help="The matching private key for the web client's public key." + help="This Home Server's ReCAPTCHA private key." ) group.add_argument( "--enable-registration-captcha", type=bool, default=False, diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e4a73da9a7..ec625f4ea8 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,9 +19,12 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes +from synapse.http.client import SimpleHttpClient +from twisted.web.client import PartialDownloadError import logging import bcrypt +import simplejson logger = logging.getLogger(__name__) @@ -33,7 +36,7 @@ class AuthHandler(BaseHandler): super(AuthHandler, self).__init__(hs) @defer.inlineCallbacks - def check_auth(self, flows, clientdict): + def check_auth(self, flows, clientdict, clientip=None): """ Takes a dictionary sent by the client in the login / registration protocol and handles the login flow. @@ -50,11 +53,12 @@ class AuthHandler(BaseHandler): login request and should be passed back to the client. """ types = { - LoginType.PASSWORD: self.check_password_auth + LoginType.PASSWORD: self.check_password_auth, + LoginType.RECAPTCHA: self.check_recaptcha, } - if 'auth' not in clientdict: - defer.returnValue((False, auth_dict_for_flows(flows))) + if not clientdict or 'auth' not in clientdict: + defer.returnValue((False, self.auth_dict_for_flows(flows))) authdict = clientdict['auth'] @@ -67,7 +71,7 @@ class AuthHandler(BaseHandler): raise LoginError(400, "", Codes.MISSING_PARAM) if authdict['type'] not in types: raise LoginError(400, "", Codes.UNRECOGNIZED) - result = yield types[authdict['type']](authdict) + result = yield types[authdict['type']](authdict, clientip) if result: creds[authdict['type']] = result @@ -76,12 +80,12 @@ class AuthHandler(BaseHandler): logger.info("Auth completed with creds: %r", creds) defer.returnValue((True, creds)) - ret = auth_dict_for_flows(flows) + ret = self.auth_dict_for_flows(flows) ret['completed'] = creds.keys() defer.returnValue((False, ret)) @defer.inlineCallbacks - def check_password_auth(self, authdict): + def check_password_auth(self, authdict, _): if "user" not in authdict or "password" not in authdict: raise LoginError(400, "", Codes.MISSING_PARAM) @@ -93,17 +97,77 @@ class AuthHandler(BaseHandler): user_info = yield self.store.get_user_by_id(user_id=user) if not user_info: logger.warn("Attempted to login as %s but they do not exist", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) stored_hash = user_info[0]["password_hash"] if bcrypt.checkpw(password, stored_hash): defer.returnValue(user) else: logger.warn("Failed password login for user %s", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + @defer.inlineCallbacks + def check_recaptcha(self, authdict, clientip): + try: + user_response = authdict["response"] + except KeyError: + # Client tried to provide captcha but didn't give the parameter: + # bad request. + raise LoginError( + 400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED + ) + + logger.info( + "Submitting recaptcha response %s with remoteip %s", + user_response, clientip + ) + + # TODO: get this from the homeserver rather than creating a new one for + # each request + try: + client = SimpleHttpClient(self.hs) + data = yield client.post_urlencoded_get_json( + "https://www.google.com/recaptcha/api/siteverify", + args={ + 'secret': self.hs.config.recaptcha_private_key, + 'response': user_response, + 'remoteip': clientip, + } + ) + except PartialDownloadError as pde: + # Twisted is silly + data = pde.response + resp_body = simplejson.loads(data) + if 'success' in resp_body and resp_body['success']: + defer.returnValue(True) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + + def get_params_recaptcha(self): + return {"public_key": self.hs.config.recaptcha_public_key} + + def auth_dict_for_flows(self, flows): + public_flows = [] + for f in flows: + hidden = False + for stagetype in f: + if stagetype in LoginType.HIDDEN_TYPES: + hidden = True + if not hidden: + public_flows.append(f) + + get_params = { + LoginType.RECAPTCHA: self.get_params_recaptcha, + } + + params = {} + + for f in public_flows: + for stage in f: + if stage in get_params and stage not in params: + params[stage] = get_params[stage]() -def auth_dict_for_flows(flows): - return { - "flows": {"stages": f for f in flows} - } + return { + "flows": [{"stages": f} for f in public_flows], + "params": params + } \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c25e321099..542759a827 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -157,7 +157,11 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def check_recaptcha(self, ip, private_key, challenge, response): - """Checks a recaptcha is correct.""" + """ + Checks a recaptcha is correct. + + Used only by c/s api v1 + """ captcha_response = yield self._validate_captcha( ip, @@ -282,6 +286,8 @@ class RegistrationHandler(BaseHandler): def _validate_captcha(self, ip_addr, private_key, challenge, response): """Validates the captcha provided. + Used only by c/s api v1 + Returns: dict: Containing 'valid'(bool) and 'error_url'(str) if invalid. @@ -299,6 +305,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): + """ + Used only by c/s api v1 + """ # TODO: get this from the homeserver rather than creating a new one for # each request client = CaptchaServerHttpClient(self.hs) diff --git a/synapse/http/client.py b/synapse/http/client.py index 2ae1c4d3a4..e8a5dedab4 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -200,6 +200,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): """ Separate HTTP client for talking to google's captcha servers Only slightly special because accepts partial download responses + + used only by c/s api v1 """ @defer.inlineCallbacks diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 041f538e20..98189ead26 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -16,7 +16,8 @@ from . import ( sync, filter, - password + password, + register ) from synapse.http.server import JsonResource @@ -34,3 +35,4 @@ class ClientV2AlphaRestResource(JsonResource): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) password.register_servlets(hs, client_resource) + register.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index c772cc986f..db2c9b244a 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -40,6 +40,12 @@ def client_v2_pattern(path_regex): return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) +def parse_request_allow_empty(request): + content = request.content.read() + if content == None or content == '': + return None + return simplejson.loads(content) + def parse_json_dict_from_request(request): try: content = simplejson.loads(request.content.read()) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py new file mode 100644 index 0000000000..84da010c29 --- /dev/null +++ b/synapse/rest/client/v2_alpha/register.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern, parse_request_allow_empty + +import logging + + +logger = logging.getLogger(__name__) + + +class RegisterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/register") + + def __init__(self, hs): + super(RegisterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.registration_handler = hs.get_handlers().registration_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_request_allow_empty(request) + + authed, result = yield self.auth_handler.check_auth([ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], + [LoginType.APPLICATION_SERVICE] + ], body) + + if not authed: + defer.returnValue((401, result)) + + is_application_server = LoginType.APPLICATION_SERVICE in result + is_using_shared_secret = LoginType.SHARED_SECRET in result + + can_register = ( + not self.hs.config.disable_registration + or is_application_server + or is_using_shared_secret + ) + if not can_register: + raise SynapseError(403, "Registration has been disabled") + + if 'username' not in body or 'password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + desired_username = body['username'] + new_password = body['password'] + + (user_id, token) = yield self.registration_handler.register( + localpart=desired_username, + password=new_password + ) + result = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + def on_OPTIONS(self, _): + return 200, {} + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) \ No newline at end of file -- cgit 1.4.1 From af853a4cdb4d6a15a9a249da8bf1aa5be1998aae Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 09:22:31 +0100 Subject: Add AppServiceConfig --- synapse/config/appservice.py | 31 +++++++++++++++++++++++++++++++ synapse/config/homeserver.py | 3 ++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 synapse/config/appservice.py (limited to 'synapse/config') diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py new file mode 100644 index 0000000000..399a716d80 --- /dev/null +++ b/synapse/config/appservice.py @@ -0,0 +1,31 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class AppServiceConfig(Config): + + def __init__(self, args): + super(AppServiceConfig, self).__init__(args) + self.app_service_config_files = args.app_service_config_files + + @classmethod + def add_arguments(cls, parser): + super(AppServiceConfig, cls).add_arguments(parser) + group = parser.add_argument_group("appservice") + group.add_argument( + "--app-service-config-files", type=str, nargs='+', + help="A list of application service config files to use." + ) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 241afdf872..3edfadb98b 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -24,12 +24,13 @@ from .email import EmailConfig from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig +from .appservice import AppServiceConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, EmailConfig, VoipConfig, RegistrationConfig, - MetricsConfig,): + MetricsConfig, AppServiceConfig,): pass -- cgit 1.4.1 From 0775c624698dbe5a837280e729ec488fd0dda28e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 18:16:23 +0100 Subject: Fix --enable-registration flag to work if you don't give a value --- demo/start.sh | 4 ++-- synapse/config/registration.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/config') diff --git a/demo/start.sh b/demo/start.sh index d647400d39..0485be8053 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -33,8 +33,8 @@ for port in 8080 8081 8082; do --manhole $((port + 1000)) \ --tls-dh-params-path "demo/demo.tls.dh" \ --media-store-path "demo/media_store.$port" \ - $PARAMS $SYNAPSE_PARAMS \ - --enable-registration + $PARAMS $SYNAPSE_PARAMS \ + --enable-registration python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ diff --git a/synapse/config/registration.py b/synapse/config/registration.py index a6a2d2c5e1..d5c8f4bf7b 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -40,7 +40,7 @@ class RegistrationConfig(Config): reg_group.add_argument( "--enable-registration", - const=False, + const=True, default=False, nargs='?', help="Enable registration for new users.", -- cgit 1.4.1 From d488463fa38ac91d30c008fb9c595140f9785b42 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Apr 2015 16:04:52 +0100 Subject: Add a version 2 of the key server api --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 8 +- synapse/config/server.py | 34 ++++++++- synapse/rest/key/v2/__init__.py | 19 +++++ synapse/rest/key/v2/local_key_resource.py | 118 ++++++++++++++++++++++++++++++ synapse/server.py | 1 + 6 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 synapse/rest/key/v2/__init__.py create mode 100644 synapse/rest/key/v2/local_key_resource.py (limited to 'synapse/config') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 3d43674625..15c8558ea7 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -22,5 +22,6 @@ STATIC_PREFIX = "/_matrix/static" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" +SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" MEDIA_PREFIX = "/_matrix/media/v1" APP_SERVICE_PREFIX = "/_matrix/appservice/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 27e53a9e56..e681941612 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,10 +35,12 @@ from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.rest.key.v1.server_key_resource import LocalKey +from synapse.rest.key.v2 import KeyApiV2Resource from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, STATIC_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, STATIC_PREFIX, + SERVER_KEY_V2_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -96,6 +98,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_server_key(self): return LocalKey(self) + def build_resource_for_server_key_v2(self): + return KeyApiV2Resource(self) + def build_resource_for_metrics(self): if self.get_config().enable_metrics: return MetricsResource(self) @@ -135,6 +140,7 @@ class SynapseHomeServer(HomeServer): (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), + (SERVER_KEY_V2_PREFIX, self.get_resource_for_server_key_v2()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), (STATIC_PREFIX, self.get_resource_for_static_content()), ] diff --git a/synapse/config/server.py b/synapse/config/server.py index 58a828cc4c..050ab90403 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -23,6 +23,9 @@ class ServerConfig(Config): super(ServerConfig, self).__init__(args) self.server_name = args.server_name self.signing_key = self.read_signing_key(args.signing_key_path) + self.old_signing_keys = self.read_old_signing_keys( + args.old_signing_key_path + ) self.bind_port = args.bind_port self.bind_host = args.bind_host self.unsecure_port = args.unsecure_port @@ -31,6 +34,7 @@ class ServerConfig(Config): self.web_client = args.web_client self.manhole = args.manhole self.soft_file_limit = args.soft_file_limit + self.key_refresh_interval = args.key_refresh_interval if not args.content_addr: host = args.server_name @@ -55,6 +59,14 @@ class ServerConfig(Config): ) server_group.add_argument("--signing-key-path", help="The signing key to sign messages with") + server_group.add_argument("--old-signing-key-path", + help="The old signing keys") + server_group.add_argument("--key-refresh-interval", + default=24 * 60 * 60 * 1000, # 1 Day + help="How long a key response is valid for." + " Used to set the exipiry in /key/v2/." + " Controls how frequently servers will" + " query what keys are still valid") server_group.add_argument("-p", "--bind-port", metavar="PORT", type=int, help="https port to listen on", default=8448) @@ -96,6 +108,19 @@ class ServerConfig(Config): " Try running again with --generate-config" ) + def read_old_signing_keys(self, old_signing_key_path): + old_signing_keys = self.read_file( + old_signing_key_path, "old_signing_key" + ) + try: + return syutil.crypto.signing_key.read_old_signing_keys( + old_signing_keys.splitlines(True) + ) + except Exception: + raise ConfigError( + "Error reading old signing keys." + ) + @classmethod def generate_config(cls, args, config_dir_path): super(ServerConfig, cls).generate_config(args, config_dir_path) @@ -110,7 +135,7 @@ class ServerConfig(Config): with open(args.signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.signing_key.generate_singing_key("auto"),), + (syutil.crypto.signing_key.generate_signing_key("auto"),), ) else: signing_keys = cls.read_file(args.signing_key_path, "signing_key") @@ -126,3 +151,10 @@ class ServerConfig(Config): signing_key_file, (key,), ) + + if not args.old_signing_key_path: + args.old_signing_key_path = base_key_name + ".old.signing.keys" + + if not os.path.exists(args.old_signing_key_path): + with open(args.old_signing_key_path, "w") as old_signing_key_file: + pass diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py new file mode 100644 index 0000000000..b79ed02590 --- /dev/null +++ b/synapse/rest/key/v2/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .local_key_resource import LocalKey + +class KeyApiV2Resource(LocalKey): + pass diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py new file mode 100644 index 0000000000..5c77f308df --- /dev/null +++ b/synapse/rest/key/v2/local_key_resource.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.web.resource import Resource +from synapse.http.server import respond_with_json_bytes +from syutil.crypto.jsonsign import sign_json +from syutil.base64util import encode_base64 +from syutil.jsonutil import encode_canonical_json +from OpenSSL import crypto +import logging + + +logger = logging.getLogger(__name__) + + +class LocalKey(Resource): + """HTTP resource containing encoding the TLS X.509 certificate and NACL + signature verification keys for this server:: + + GET /_matrix/key/v2/ HTTP/1.1 + + HTTP/1.1 200 OK + Content-Type: application/json + { + "expires": # integer posix timestamp when this result expires. + "server_name": "this.server.example.com" + "verify_keys": { + "algorithm:version": # base64 encoded NACL verification key. + }, + "old_verify_keys": { + "algorithm:version": { + "expired": # integer posix timestamp when the key expired. + "key": # base64 encoded NACL verification key. + } + } + "tls_certificate": # base64 ASN.1 DER encoded X.509 tls cert. + "signatures": { + "this.server.example.com": { + "algorithm:version": # NACL signature for this server + } + } + } + """ + + def __init__(self, hs): + self.version_string = hs.version_string + self.config = hs.config + self.clock = hs.clock + self.update_response_body(self.clock.time_msec()) + Resource.__init__(self) + + def update_response_body(self, time_now_msec): + refresh_interval = self.config.key_refresh_interval + self.expires = int(time_now_msec + refresh_interval) + self.response_body = encode_canonical_json(self.response_json_object()) + + + def response_json_object(self): + verify_keys = {} + for key in self.config.signing_key: + verify_key_bytes = key.verify_key.encode() + key_id = "%s:%s" % (key.alg, key.version) + verify_keys[key_id] = encode_base64(verify_key_bytes) + + old_verify_keys = {} + for key in self.config.old_signing_keys: + key_id = "%s:%s" % (key.alg, key.version) + verify_key_bytes = key.encode() + old_verify_keys[key_id] = { + u"key": encode_base64(verify_key_bytes), + u"expired": key.expired, + } + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, + self.config.tls_certificate + ) + json_object = { + u"expires": self.expires, + u"server_name": self.config.server_name, + u"verify_keys": verify_keys, + u"old_verify_keys": old_verify_keys, + u"tls_certificate": encode_base64(x509_certificate_bytes) + } + for key in self.config.signing_key: + json_object = sign_json( + json_object, + self.config.server_name, + key, + ) + return json_object + + def render_GET(self, request): + time_now = self.clock.time_msec() + # Update the expiry time if less than half the interval remains. + if time_now + self.config.key_refresh_interval / 2 > self.expires: + self.update_response_body() + return respond_with_json_bytes( + request, 200, self.response_body, + version_string=self.version_string + ) + + def getChild(self, name, request): + if name == '': + return self diff --git a/synapse/server.py b/synapse/server.py index 0bd87bdd77..a602b425e3 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -78,6 +78,7 @@ class BaseHomeServer(object): 'resource_for_web_client', 'resource_for_content_repo', 'resource_for_server_key', + 'resource_for_server_key_v2', 'resource_for_media_repository', 'resource_for_metrics', 'event_sources', -- cgit 1.4.1 From 88cb06e996349d9a2e69d5f29dafe764d35b7966 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Apr 2015 16:18:17 +0100 Subject: Update syutil version to 0.0.4 --- synapse/config/server.py | 2 +- synapse/python_dependencies.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index 58a828cc4c..d4c223f348 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -110,7 +110,7 @@ class ServerConfig(Config): with open(args.signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.signing_key.generate_singing_key("auto"),), + (syutil.crypto.signing_key.generate_signing_key("auto"),), ) else: signing_keys = cls.read_file(args.signing_key_path, "signing_key") diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 6b6d5508b8..dac927d0a7 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.3": ["syutil"], + "syutil>=0.0.4": ["syutil"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -43,8 +43,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/syutil", - version="v0.0.3", - egg="syutil-0.0.3", + version="v0.0.4", + egg="syutil-0.0.4", ), github_link( project="matrix-org/matrix-angular-sdk", -- cgit 1.4.1 From 5e7a90316d43f0dc42c5b18e505ce9b432e15461 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:08:18 +0100 Subject: Update --database-path metavar to SQLITE_DATABASE_PATH --- synapse/config/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/database.py b/synapse/config/database.py index 8dc9873f8c..f3d0898c09 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -37,7 +37,7 @@ class DatabaseConfig(Config): db_group = parser.add_argument_group("database") db_group.add_argument( "-d", "--database-path", default="homeserver.db", - help="The database name." + metavar="SQLITE_DATABASE_PATH", help="The database name." ) db_group.add_argument( "--event-cache-size", default="100K", -- cgit 1.4.1 From f30d47c87651f92b69be224e016bda2cd7285f04 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 22 Apr 2015 14:21:08 +0100 Subject: Implement remote key lookup api --- synapse/config/server.py | 4 +- synapse/crypto/keyclient.py | 6 +- synapse/crypto/keyring.py | 75 +++++++------ synapse/rest/key/v2/__init__.py | 10 +- synapse/rest/key/v2/local_key_resource.py | 9 +- synapse/rest/key/v2/remote_key_resource.py | 174 +++++++++++++++++++++++++++++ synapse/storage/keys.py | 35 +++--- 7 files changed, 252 insertions(+), 61 deletions(-) create mode 100644 synapse/rest/key/v2/remote_key_resource.py (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index 050ab90403..a26fb115f2 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -62,7 +62,7 @@ class ServerConfig(Config): server_group.add_argument("--old-signing-key-path", help="The old signing keys") server_group.add_argument("--key-refresh-interval", - default=24 * 60 * 60 * 1000, # 1 Day + default=24 * 60 * 60 * 1000, # 1 Day help="How long a key response is valid for." " Used to set the exipiry in /key/v2/." " Controls how frequently servers will" @@ -156,5 +156,5 @@ class ServerConfig(Config): args.old_signing_key_path = base_key_name + ".old.signing.keys" if not os.path.exists(args.old_signing_key_path): - with open(args.old_signing_key_path, "w") as old_signing_key_file: + with open(args.old_signing_key_path, "w"): pass diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 2452c7a26e..4911f0896b 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -26,7 +26,7 @@ import logging logger = logging.getLogger(__name__) KEY_API_V1 = b"/_matrix/key/v1/" -KEY_API_V2 = b"/_matrix/key/v2/local" + @defer.inlineCallbacks def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1): @@ -94,8 +94,8 @@ class SynapseKeyClientProtocol(HTTPClient): if status != b"200": # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) - error = SynapseKeyClientError("Non-200 response %r from %r" % - (status, self.host) + error = SynapseKeyClientError( + "Non-200 response %r from %r" % (status, self.host) ) error.status = status self.errback(error) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 5528d0a280..17ac66731c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -15,7 +15,9 @@ from synapse.crypto.keyclient import fetch_server_key from twisted.internet import defer -from syutil.crypto.jsonsign import verify_signed_json, signature_ids +from syutil.crypto.jsonsign import ( + verify_signed_json, signature_ids, sign_json, encode_canonical_json +) from syutil.crypto.signing_key import ( is_signing_algorithm_supported, decode_verify_key_bytes ) @@ -26,6 +28,8 @@ from synapse.util.retryutils import get_retry_limiter from OpenSSL import crypto +import urllib +import hashlib import logging @@ -37,6 +41,7 @@ class Keyring(object): self.store = hs.get_datastore() self.clock = hs.get_clock() self.client = hs.get_http_client() + self.config = hs.get_config() self.perspective_servers = {} self.hs = hs @@ -127,7 +132,6 @@ class Keyring(object): server_name, key_ids ) - for key_id in key_ids: if key_id in keys: defer.returnValue(keys[key_id]) @@ -142,17 +146,18 @@ class Keyring(object): perspective_name, self.clock, self.store ) - responses = yield self.client.post_json( - destination=perspective_name, - path=b"/_matrix/key/v2/query", - data={u"server_keys": {server_name: list(key_ids)}}, - ) + with limiter: + responses = yield self.client.post_json( + destination=perspective_name, + path=b"/_matrix/key/v2/query", + data={u"server_keys": {server_name: list(key_ids)}}, + ) - keys = dict() + keys = {} for response in responses: if (u"signatures" not in response - or perspective_name not in response[u"signatures"]): + or perspective_name not in response[u"signatures"]): raise ValueError( "Key response not signed by perspective server" " %r" % (perspective_name,) @@ -181,7 +186,9 @@ class Keyring(object): " server %r" % (perspective_name,) ) - response_keys = process_v2_response(self, server_name, key_ids) + response_keys = yield self.process_v2_response( + server_name, perspective_name, response + ) keys.update(response_keys) @@ -202,15 +209,15 @@ class Keyring(object): if requested_key_id in keys: continue - (response_json, tls_certificate) = yield fetch_server_key( + (response, tls_certificate) = yield fetch_server_key( server_name, self.hs.tls_context_factory, - path="/_matrix/key/v2/server/%s" % ( + path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), - ), + )).encode("ascii"), ) if (u"signatures" not in response - or server_name not in response[u"signatures"]): + or server_name not in response[u"signatures"]): raise ValueError("Key response not signed by remote server") if "tls_fingerprints" not in response: @@ -223,17 +230,18 @@ class Keyring(object): sha256_fingerprint_b64 = encode_base64(sha256_fingerprint) response_sha256_fingerprints = set() - for fingerprint in response_json[u"tls_fingerprints"]: + for fingerprint in response[u"tls_fingerprints"]: if u"sha256" in fingerprint: response_sha256_fingerprints.add(fingerprint[u"sha256"]) - if sha256_fingerprint not in response_sha256_fingerprints: + if sha256_fingerprint_b64 not in response_sha256_fingerprints: raise ValueError("TLS certificate not allowed by fingerprints") response_keys = yield self.process_v2_response( server_name=server_name, from_server=server_name, - response_json=response_json, + requested_id=requested_key_id, + response_json=response, ) keys.update(response_keys) @@ -244,19 +252,15 @@ class Keyring(object): verify_keys=keys, ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return - - raise ValueError("No verification key found for given key ids") + defer.returnValue(keys) @defer.inlineCallbacks - def process_v2_response(self, server_name, from_server, json_response): - time_now_ms = clock.time_msec() + def process_v2_response(self, server_name, from_server, response_json, + requested_id=None): + time_now_ms = self.clock.time_msec() response_keys = {} verify_keys = {} - for key_id, key_data in response["verify_keys"].items(): + for key_id, key_data in response_json["verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) @@ -264,7 +268,7 @@ class Keyring(object): verify_keys[key_id] = verify_key old_verify_keys = {} - for key_id, key_data in response["verify_keys"].items(): + for key_id, key_data in response_json["old_verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) @@ -273,21 +277,21 @@ class Keyring(object): verify_key.time_added = time_now_ms old_verify_keys[key_id] = verify_key - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: + for key_id in response_json["signatures"][server_name]: + if key_id not in response_json["verify_keys"]: raise ValueError( "Key response must include verification keys for all" " signatures" ) if key_id in verify_keys: verify_signed_json( - response, + response_json, server_name, verify_keys[key_id] ) signed_key_json = sign_json( - response, + response_json, self.config.server_name, self.config.signing_key[0], ) @@ -295,7 +299,9 @@ class Keyring(object): signed_key_json_bytes = encode_canonical_json(signed_key_json) ts_valid_until_ms = signed_key_json[u"valid_until"] - updated_key_ids = set([requested_key_id]) + updated_key_ids = set() + if requested_id is not None: + updated_key_ids.add(requested_id) updated_key_ids.update(verify_keys) updated_key_ids.update(old_verify_keys) @@ -307,8 +313,8 @@ class Keyring(object): server_name=server_name, key_id=key_id, from_server=server_name, - ts_now_ms=ts_now_ms, - ts_valid_until_ms=valid_until, + ts_now_ms=time_now_ms, + ts_expires_ms=ts_valid_until_ms, key_json_bytes=signed_key_json_bytes, ) @@ -373,7 +379,6 @@ class Keyring(object): verify_keys[key_id] ) - yield self.store.store_server_certificate( server_name, server_name, diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py index b79ed02590..1c14791b09 100644 --- a/synapse/rest/key/v2/__init__.py +++ b/synapse/rest/key/v2/__init__.py @@ -13,7 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.web.resource import Resource from .local_key_resource import LocalKey +from .remote_key_resource import RemoteKey -class KeyApiV2Resource(LocalKey): - pass + +class KeyApiV2Resource(Resource): + def __init__(self, hs): + Resource.__init__(self) + self.putChild("server", LocalKey(hs)) + self.putChild("query", RemoteKey(hs)) diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index 1c0e0717c1..982a460962 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -31,7 +31,7 @@ class LocalKey(Resource): """HTTP resource containing encoding the TLS X.509 certificate and NACL signature verification keys for this server:: - GET /_matrix/key/v2/ HTTP/1.1 + GET /_matrix/key/v2/server/a.key.id HTTP/1.1 HTTP/1.1 200 OK Content-Type: application/json @@ -56,6 +56,8 @@ class LocalKey(Resource): } """ + isLeaf = True + def __init__(self, hs): self.version_string = hs.version_string self.config = hs.config @@ -68,7 +70,6 @@ class LocalKey(Resource): self.expires = int(time_now_msec + refresh_interval) self.response_body = encode_canonical_json(self.response_json_object()) - def response_json_object(self): verify_keys = {} for key in self.config.signing_key: @@ -120,7 +121,3 @@ class LocalKey(Resource): request, 200, self.response_body, version_string=self.version_string ) - - def getChild(self, name, request): - if name == '': - return self diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py new file mode 100644 index 0000000000..cf6f2c2e73 --- /dev/null +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -0,0 +1,174 @@ +from synapse.http.server import request_handler, respond_with_json_bytes +from synapse.api.errors import SynapseError, Codes + +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + + +from io import BytesIO +import json +import logging +logger = logging.getLogger(__name__) + + +class RemoteKey(Resource): + """HTTP resource for retreiving the TLS certificate and NACL signature + verification keys for a collection of servers. Checks that the reported + X.509 TLS certificate matches the one used in the HTTPS connection. Checks + that the NACL signature for the remote server is valid. Returns a dict of + JSON signed by both the remote server and by this server. + + Supports individual GET APIs and a bulk query POST API. + + Requsts: + + GET /_matrix/key/v2/query/remote.server.example.com HTTP/1.1 + + GET /_matrix/key/v2/query/remote.server.example.com/a.key.id HTTP/1.1 + + POST /_matrix/v2/query HTTP/1.1 + Content-Type: application/json + { + "server_keys": { "remote.server.example.com": ["a.key.id"] } + } + + Response: + + HTTP/1.1 200 OK + Content-Type: application/json + { + "server_keys": [ + { + "server_name": "remote.server.example.com" + "valid_until": # posix timestamp + "verify_keys": { + "a.key.id": { # The identifier for a key. + key: "" # base64 encoded verification key. + } + } + "old_verify_keys": { + "an.old.key.id": { # The identifier for an old key. + key: "", # base64 encoded key + expired: 0, # when th e + } + } + "tls_fingerprints": [ + { "sha256": # fingerprint } + ] + "signatures": { + "remote.server.example.com": {...} + "this.server.example.com": {...} + } + } + ] + } + """ + + isLeaf = True + + def __init__(self, hs): + self.keyring = hs.get_keyring() + self.store = hs.get_datastore() + self.version_string = hs.version_string + self.clock = hs.get_clock() + + def render_GET(self, request): + self.async_render_GET(request) + return NOT_DONE_YET + + @request_handler + @defer.inlineCallbacks + def async_render_GET(self, request): + if len(request.postpath) == 1: + server, = request.postpath + query = {server: [None]} + elif len(request.postpath) == 2: + server, key_id = request.postpath + query = {server: [key_id]} + else: + raise SynapseError( + 404, "Not found %r" % request.postpath, Codes.NOT_FOUND + ) + yield self.query_keys(request, query, query_remote_on_cache_miss=True) + + def render_POST(self, request): + self.async_render_POST(request) + return NOT_DONE_YET + + @request_handler + @defer.inlineCallbacks + def async_render_POST(self, request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise ValueError() + except ValueError: + raise SynapseError( + 400, "Content must be JSON object.", errcode=Codes.NOT_JSON + ) + + query = content["server_keys"] + + yield self.query_keys(request, query, query_remote_on_cache_miss=True) + + @defer.inlineCallbacks + def query_keys(self, request, query, query_remote_on_cache_miss=False): + store_queries = [] + for server_name, key_ids in query.items(): + for key_id in key_ids: + store_queries.append((server_name, key_id, None)) + + cached = yield self.store.get_server_keys_json(store_queries) + + json_results = [] + + time_now_ms = self.clock.time_msec() + + cache_misses = dict() + for (server_name, key_id, from_server), results in cached.items(): + results = [ + (result["ts_added_ms"], result) for result in results + if result["ts_valid_until_ms"] > time_now_ms + ] + + if not results: + if key_id is not None: + cache_misses.setdefault(server_name, set()).add(key_id) + continue + + if key_id is not None: + most_recent_result = max(results) + json_results.append(most_recent_result[-1]["key_json"]) + else: + for result in results: + json_results.append(result[-1]["key_json"]) + + if cache_misses and query_remote_on_cache_miss: + for server_name, key_ids in cache_misses.items(): + try: + yield self.keyring.get_server_verify_key_v2_direct( + server_name, key_ids + ) + except: + logger.exception("Failed to get key for %r", server_name) + pass + yield self.query_keys( + request, query, query_remote_on_cache_miss=False + ) + else: + result_io = BytesIO() + result_io.write(b"{\"server_keys\":") + sep = b"[" + for json_bytes in json_results: + result_io.write(sep) + result_io.write(json_bytes) + sep = b"," + if sep == b"[": + result_io.write(sep) + result_io.write(b"]}") + + respond_with_json_bytes( + request, 200, result_io.getvalue(), + version_string=self.version_string + ) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 8b08d42859..22b158d71e 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -140,8 +140,8 @@ class KeyStore(SQLBaseStore): "key_id": key_id, "from_server": from_server, "ts_added_ms": ts_now_ms, - "ts_valid_until_ms": ts_valid_until_ms, - "key_json": key_json_bytes, + "ts_valid_until_ms": ts_expires_ms, + "key_json": buffer(key_json_bytes), }, or_replace=True, ) @@ -149,9 +149,9 @@ class KeyStore(SQLBaseStore): def get_server_keys_json(self, server_keys): """Retrive the key json for a list of server_keys and key ids. If no keys are found for a given server, key_id and source then - that server, key_id, and source triplet will be missing from the - returned dictionary. The JSON is returned as a byte array so that it - can be efficiently used in an HTTP response. + that server, key_id, and source triplet entry will be an empty list. + The JSON is returned as a byte array so that it can be efficiently + used in an HTTP response. Args: server_keys (list): List of (server_name, key_id, source) triplets. Returns: @@ -161,16 +161,25 @@ class KeyStore(SQLBaseStore): def _get_server_keys_json_txn(txn): results = {} for server_name, key_id, from_server in server_keys: - rows = _simple_select_list_txn( - keyvalues={ - "server_name": server_name, - "key_id": key_id, - "from_server": from_server, - }, - retcols=("ts_valid_until_ms", "key_json"), + keyvalues = {"server_name": server_name} + if key_id is not None: + keyvalues["key_id"] = key_id + if from_server is not None: + keyvalues["from_server"] = from_server + rows = self._simple_select_list_txn( + txn, + "server_keys_json", + keyvalues=keyvalues, + retcols=( + "key_id", + "from_server", + "ts_added_ms", + "ts_valid_until_ms", + "key_json", + ), ) results[(server_name, key_id, from_server)] = rows return results - return runInteraction( + return self.runInteraction( "get_server_keys_json", _get_server_keys_json_txn ) -- cgit 1.4.1 From 2e0d9219b9585801ac8a8b5f3911107643ea3519 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 11:45:29 +0100 Subject: Remove now-redundant email config --- synapse/config/email.py | 42 ------------------------------------------ synapse/config/homeserver.py | 3 +-- 2 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 synapse/config/email.py (limited to 'synapse/config') diff --git a/synapse/config/email.py b/synapse/config/email.py deleted file mode 100644 index f0854f8c37..0000000000 --- a/synapse/config/email.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._base import Config - - -class EmailConfig(Config): - - def __init__(self, args): - super(EmailConfig, self).__init__(args) - self.email_from_address = args.email_from_address - self.email_smtp_server = args.email_smtp_server - - @classmethod - def add_arguments(cls, parser): - super(EmailConfig, cls).add_arguments(parser) - email_group = parser.add_argument_group("email") - email_group.add_argument( - "--email-from-address", - default="FROM@EXAMPLE.COM", - help="The address to send emails from (e.g. for password resets)." - ) - email_group.add_argument( - "--email-smtp-server", - default="", - help=( - "The SMTP server to send emails from (e.g. for password" - " resets)." - ) - ) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3edfadb98b..efbdd93c25 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -20,7 +20,6 @@ from .database import DatabaseConfig from .ratelimiting import RatelimitConfig from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig -from .email import EmailConfig from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig @@ -29,7 +28,7 @@ from .appservice import AppServiceConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - EmailConfig, VoipConfig, RegistrationConfig, + VoipConfig, RegistrationConfig, MetricsConfig, AppServiceConfig,): pass -- cgit 1.4.1 From 149ed9f151770def0e4c130c2dcc1c64bcf65b19 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 10:07:55 +0100 Subject: Better help for the old-signing-key option --- synapse/config/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index a26fb115f2..3ce3ed584f 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -60,7 +60,10 @@ class ServerConfig(Config): server_group.add_argument("--signing-key-path", help="The signing key to sign messages with") server_group.add_argument("--old-signing-key-path", - help="The old signing keys") + help="The keys that the server used to sign" + " sign messages with but won't use" + " to sign new messages. E.g. it has" + " lost its private key") server_group.add_argument("--key-refresh-interval", default=24 * 60 * 60 * 1000, # 1 Day help="How long a key response is valid for." -- cgit 1.4.1 From c8c710eca73093d56e9c298065faf938d0a9ca5b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 10:22:22 +0100 Subject: Move the key related config parser into a separate file --- synapse/config/homeserver.py | 3 +- synapse/config/key.py | 110 +++++++++++++++++++++++++++++++++++++++++++ synapse/config/server.py | 85 +-------------------------------- 3 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 synapse/config/key.py (limited to 'synapse/config') diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3edfadb98b..967a0f45d6 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -25,12 +25,13 @@ from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig from .appservice import AppServiceConfig +from .key import KeyConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, EmailConfig, VoipConfig, RegistrationConfig, - MetricsConfig, AppServiceConfig,): + MetricsConfig, AppServiceConfig, KeyConfig,): pass diff --git a/synapse/config/key.py b/synapse/config/key.py new file mode 100644 index 0000000000..327105732a --- /dev/null +++ b/synapse/config/key.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from ._base import Config, ConfigError +import syutil.crypto.signing_key + + +class KeyConfig(Config): + + def __init__(self, args): + super(KeyConfig, self).__init__(args) + self.signing_key = self.read_signing_key(args.signing_key_path) + self.old_signing_keys = self.read_old_signing_keys( + args.old_signing_key_path + ) + self.key_refresh_interval = args.key_refresh_interval + + @classmethod + def add_arguments(cls, parser): + super(KeyConfig, cls).add_arguments(parser) + key_group = parser.add_argument_group("keys") + key_group.add_argument("--signing-key-path", + help="The signing key to sign messages with") + key_group.add_argument("--old-signing-key-path", + help="The keys that the server used to sign" + " sign messages with but won't use" + " to sign new messages. E.g. it has" + " lost its private key") + key_group.add_argument("--key-refresh-interval", + default=24 * 60 * 60 * 1000, # 1 Day + help="How long a key response is valid for." + " Used to set the exipiry in /key/v2/." + " Controls how frequently servers will" + " query what keys are still valid") + + def read_signing_key(self, signing_key_path): + signing_keys = self.read_file(signing_key_path, "signing_key") + try: + return syutil.crypto.signing_key.read_signing_keys( + signing_keys.splitlines(True) + ) + except Exception: + raise ConfigError( + "Error reading signing_key." + " Try running again with --generate-config" + ) + + def read_old_signing_keys(self, old_signing_key_path): + old_signing_keys = self.read_file( + old_signing_key_path, "old_signing_key" + ) + try: + return syutil.crypto.signing_key.read_old_signing_keys( + old_signing_keys.splitlines(True) + ) + except Exception: + raise ConfigError( + "Error reading old signing keys." + ) + + @classmethod + def generate_config(cls, args, config_dir_path): + super(KeyConfig, cls).generate_config(args, config_dir_path) + base_key_name = os.path.join(config_dir_path, args.server_name) + + args.pid_file = os.path.abspath(args.pid_file) + + if not args.signing_key_path: + args.signing_key_path = base_key_name + ".signing.key" + + if not os.path.exists(args.signing_key_path): + with open(args.signing_key_path, "w") as signing_key_file: + syutil.crypto.signing_key.write_signing_keys( + signing_key_file, + (syutil.crypto.signing_key.generate_signing_key("auto"),), + ) + else: + signing_keys = cls.read_file(args.signing_key_path, "signing_key") + if len(signing_keys.split("\n")[0].split()) == 1: + # handle keys in the old format. + key = syutil.crypto.signing_key.decode_signing_key_base64( + syutil.crypto.signing_key.NACL_ED25519, + "auto", + signing_keys.split("\n")[0] + ) + with open(args.signing_key_path, "w") as signing_key_file: + syutil.crypto.signing_key.write_signing_keys( + signing_key_file, + (key,), + ) + + if not args.old_signing_key_path: + args.old_signing_key_path = base_key_name + ".old.signing.keys" + + if not os.path.exists(args.old_signing_key_path): + with open(args.old_signing_key_path, "w"): + pass diff --git a/synapse/config/server.py b/synapse/config/server.py index 3ce3ed584f..c25feb4c58 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -13,19 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -from ._base import Config, ConfigError -import syutil.crypto.signing_key +from ._base import Config class ServerConfig(Config): def __init__(self, args): super(ServerConfig, self).__init__(args) self.server_name = args.server_name - self.signing_key = self.read_signing_key(args.signing_key_path) - self.old_signing_keys = self.read_old_signing_keys( - args.old_signing_key_path - ) self.bind_port = args.bind_port self.bind_host = args.bind_host self.unsecure_port = args.unsecure_port @@ -34,7 +28,6 @@ class ServerConfig(Config): self.web_client = args.web_client self.manhole = args.manhole self.soft_file_limit = args.soft_file_limit - self.key_refresh_interval = args.key_refresh_interval if not args.content_addr: host = args.server_name @@ -57,19 +50,6 @@ class ServerConfig(Config): "This is used by remote servers to connect to this server, " "e.g. matrix.org, localhost:8080, etc." ) - server_group.add_argument("--signing-key-path", - help="The signing key to sign messages with") - server_group.add_argument("--old-signing-key-path", - help="The keys that the server used to sign" - " sign messages with but won't use" - " to sign new messages. E.g. it has" - " lost its private key") - server_group.add_argument("--key-refresh-interval", - default=24 * 60 * 60 * 1000, # 1 Day - help="How long a key response is valid for." - " Used to set the exipiry in /key/v2/." - " Controls how frequently servers will" - " query what keys are still valid") server_group.add_argument("-p", "--bind-port", metavar="PORT", type=int, help="https port to listen on", default=8448) @@ -98,66 +78,3 @@ class ServerConfig(Config): "Zero is used to indicate synapse " "should set the soft limit to the hard" "limit.") - - def read_signing_key(self, signing_key_path): - signing_keys = self.read_file(signing_key_path, "signing_key") - try: - return syutil.crypto.signing_key.read_signing_keys( - signing_keys.splitlines(True) - ) - except Exception: - raise ConfigError( - "Error reading signing_key." - " Try running again with --generate-config" - ) - - def read_old_signing_keys(self, old_signing_key_path): - old_signing_keys = self.read_file( - old_signing_key_path, "old_signing_key" - ) - try: - return syutil.crypto.signing_key.read_old_signing_keys( - old_signing_keys.splitlines(True) - ) - except Exception: - raise ConfigError( - "Error reading old signing keys." - ) - - @classmethod - def generate_config(cls, args, config_dir_path): - super(ServerConfig, cls).generate_config(args, config_dir_path) - base_key_name = os.path.join(config_dir_path, args.server_name) - - args.pid_file = os.path.abspath(args.pid_file) - - if not args.signing_key_path: - args.signing_key_path = base_key_name + ".signing.key" - - if not os.path.exists(args.signing_key_path): - with open(args.signing_key_path, "w") as signing_key_file: - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (syutil.crypto.signing_key.generate_signing_key("auto"),), - ) - else: - signing_keys = cls.read_file(args.signing_key_path, "signing_key") - if len(signing_keys.split("\n")[0].split()) == 1: - # handle keys in the old format. - key = syutil.crypto.signing_key.decode_signing_key_base64( - syutil.crypto.signing_key.NACL_ED25519, - "auto", - signing_keys.split("\n")[0] - ) - with open(args.signing_key_path, "w") as signing_key_file: - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (key,), - ) - - if not args.old_signing_key_path: - args.old_signing_key_path = base_key_name + ".old.signing.keys" - - if not os.path.exists(args.old_signing_key_path): - with open(args.old_signing_key_path, "w"): - pass -- cgit 1.4.1 From b1e68add1992d0072aa37f3bdf07dc226200fe5d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 11:26:19 +0100 Subject: Add a config file for perspective servers --- synapse/config/_base.py | 11 +++++++++++ synapse/config/key.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 87cdbf1d30..f07ea4cc46 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -77,6 +77,17 @@ class Config(object): with open(file_path) as file_stream: return file_stream.read() + @classmethod + def read_yaml_file(cls, file_path, config_name): + cls.check_file(file_path, config_name) + with open(file_path) as file_stream: + try: + return yaml.load(file_stream) + except Exception as e: + raise ConfigError( + "Error parsing yaml in file %r: " % (file_path,), e + ) + @staticmethod def default_path(name): return os.path.abspath(os.path.join(os.path.curdir, name)) diff --git a/synapse/config/key.py b/synapse/config/key.py index 327105732a..de4e33a7f3 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -27,6 +27,9 @@ class KeyConfig(Config): args.old_signing_key_path ) self.key_refresh_interval = args.key_refresh_interval + self.perspectives = self.read_perspectives( + args.perspectives_config_path + ) @classmethod def add_arguments(cls, parser): @@ -45,6 +48,15 @@ class KeyConfig(Config): " Used to set the exipiry in /key/v2/." " Controls how frequently servers will" " query what keys are still valid") + key_group.add_argument("--perspectives-config-path", + help="The trusted servers to download signing" + " keys from") + + def read_perspectives(self, perspectives_config_path): + servers = self.read_yaml_file( + perspectives_config_path, "perspectives_config_path" + ) + return servers def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") @@ -108,3 +120,10 @@ class KeyConfig(Config): if not os.path.exists(args.old_signing_key_path): with open(args.old_signing_key_path, "w"): pass + + if not args.perspectives_config_path: + args.perspectives_config_path = base_key_name + ".perspectives" + + if not os.path.exists(args.perspectives_config_path): + with open(args.perspectives_config_path, "w") as perspectives_file: + perspectives_file.write("@@@") -- cgit 1.4.1 From 869dc94cbb5810e50efb7b6dd8320817aca01554 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 11:27:56 +0100 Subject: Call the super classes when generating config --- synapse/config/registration.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index d5c8f4bf7b..ad81cc4f45 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -53,6 +53,7 @@ class RegistrationConfig(Config): @classmethod def generate_config(cls, args, config_dir_path): + super(RegistrationConfig, cls).genenerate_config(args, config_dir_path) if args.enable_registration is None: args.enable_registration = False -- cgit 1.4.1 From bdcb23ca25bc2ea72ab3bc28d76c6b72d68206b3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 11:29:19 +0100 Subject: Fix spelling --- synapse/config/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index ad81cc4f45..f412a72f59 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -53,7 +53,7 @@ class RegistrationConfig(Config): @classmethod def generate_config(cls, args, config_dir_path): - super(RegistrationConfig, cls).genenerate_config(args, config_dir_path) + super(RegistrationConfig, cls).generate_config(args, config_dir_path) if args.enable_registration is None: args.enable_registration = False -- cgit 1.4.1 From 288702170d6fc8b44926856b37e4a0e1bb5b2ac4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 17:01:34 +0100 Subject: Add config for setting the perspective servers --- synapse/config/_base.py | 4 ++-- synapse/config/key.py | 22 ++++++++++++++++++++-- synapse/crypto/keyring.py | 6 +++++- 3 files changed, 27 insertions(+), 5 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index f07ea4cc46..6fd086a471 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -83,9 +83,9 @@ class Config(object): with open(file_path) as file_stream: try: return yaml.load(file_stream) - except Exception as e: + except: raise ConfigError( - "Error parsing yaml in file %r: " % (file_path,), e + "Error parsing yaml in file %r" % (file_path,) ) @staticmethod diff --git a/synapse/config/key.py b/synapse/config/key.py index de4e33a7f3..a2de6d5c17 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -16,6 +16,10 @@ import os from ._base import Config, ConfigError import syutil.crypto.signing_key +from syutil.crypto.signing_key import ( + is_signing_algorithm_supported, decode_verify_key_bytes +) +from syutil.base64util import decode_base64 class KeyConfig(Config): @@ -53,9 +57,17 @@ class KeyConfig(Config): " keys from") def read_perspectives(self, perspectives_config_path): - servers = self.read_yaml_file( + config = self.read_yaml_file( perspectives_config_path, "perspectives_config_path" ) + servers = {} + for server_name, server_config in config["servers"].items(): + for key_id, key_data in server_config["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + servers.setdefault(server_name, {})[key_id] = verify_key return servers def read_signing_key(self, signing_key_path): @@ -126,4 +138,10 @@ class KeyConfig(Config): if not os.path.exists(args.perspectives_config_path): with open(args.perspectives_config_path, "w") as perspectives_file: - perspectives_file.write("@@@") + perspectives_file.write( + 'servers:\n' + ' matrix.org:\n' + ' verify_keys:\n' + ' "ed25519:auto":\n' + ' key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"\n' + ) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d248776bc1..f7ae227916 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -42,7 +42,7 @@ class Keyring(object): self.clock = hs.get_clock() self.client = hs.get_http_client() self.config = hs.get_config() - self.perspective_servers = {} + self.perspective_servers = self.config.perspectives self.hs = hs @defer.inlineCallbacks @@ -111,6 +111,10 @@ class Keyring(object): ) break except: + logging.info( + "Unable to getting key %r for %r from %r", + key_ids, server_name, perspective_name, + ) pass limiter = yield get_retry_limiter( -- cgit 1.4.1 From 1ef66cc3bd541ee1e4a017cfdd008eacaec5bcf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:57:43 +0100 Subject: Move database configuration into config module --- synapse/app/homeserver.py | 35 ++++------------------------------- synapse/config/database.py | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 33 deletions(-) (limited to 'synapse/config') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3709cd7bf9..f29f9d702e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -350,42 +350,15 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) - if config.database_config: - with open(config.database_config, 'r') as f: - db_config = yaml.safe_load(f) - else: - db_config = { - "name": "sqlite3", - "args": { - "database": config.database_path, - }, - } - - db_config = { - k: v for k, v in db_config.items() - } - - name = db_config.get("name", None) - if name == "psycopg2": - pass - elif name == "sqlite3": - db_config.setdefault("args", {}).update({ - "cp_min": 1, - "cp_max": 1, - "check_same_thread": False, - }) - else: - raise RuntimeError("Unsupported database type '%s'" % (name,)) - - database_engine = create_engine(name) - db_config["args"]["cp_openfun"] = database_engine.on_new_connection + database_engine = create_engine(config.database_config["name"]) + config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, - db_config=db_config, + db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -404,7 +377,7 @@ def setup(config_options): try: db_conn = database_engine.module.connect( **{ - k: v for k, v in db_config.get("args", {}).items() + k: v for k, v in config.database_config.get("args", {}).items() if not k.startswith("cp_") } ) diff --git a/synapse/config/database.py b/synapse/config/database.py index f3d0898c09..190d119df4 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -15,6 +15,7 @@ from ._base import Config import os +import yaml class DatabaseConfig(Config): @@ -27,9 +28,27 @@ class DatabaseConfig(Config): self.event_cache_size = self.parse_size(args.event_cache_size) if args.database_config: - self.database_config = self.abspath(args.database_config) + with open(args.database_config) as f: + self.database_config = yaml.safe_load(f) else: - self.database_config = None + self.database_config = { + "name": "sqlite3", + "args": { + "database": self.database_path, + }, + } + + name = self.database_config.get("name", None) + if name == "psycopg2": + pass + elif name == "sqlite3": + self.database_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) @classmethod def add_arguments(cls, parser): -- cgit 1.4.1 From 0bc71103e16a774b53383365d461c7705e313a5b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 28 Apr 2015 10:17:10 +0100 Subject: Output vim style mode markers into the yaml config file --- synapse/config/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 87cdbf1d30..6017cb6334 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -147,9 +147,10 @@ class Config(object): and value is not None): config[key] = value with open(config_args.config_path, "w") as config_file: - # TODO(paul) it would be lovely if we wrote out vim- and emacs- - # style mode markers into the file, to hint to people that - # this is a YAML file. + # TODO(mark/paul) We might want to output emacs-style mode + # markers as well as vim-style mode markers into the file, + # to further hint to people this is a YAML file. + config_file.write("# vim:ft=yaml\n") yaml.dump(config, config_file, default_flow_style=False) print ( "A config file has been generated in %s for server name" -- cgit 1.4.1 From 68c06039461a342084787f77125d4c7898ac5899 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:14:44 +0100 Subject: comment out ugly test logline --- synapse/config/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 63c8e36930..1a850f7058 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -78,7 +78,7 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - logger.info("Test") + #logger.info("Test") else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) -- cgit 1.4.1 From f4c9ebbc345d13c458f63ce2f946339af9eddd26 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 11:07:13 +0100 Subject: Delete ugly commented out log line. --- synapse/config/logger.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 1a850f7058..247b324816 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -78,7 +78,6 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - #logger.info("Test") else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) -- cgit 1.4.1 From d624e2a6383bbb179132b79eec80fa516e747bd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 04:24:44 +0100 Subject: Manually generate the default config yaml, remove most of the commandline arguments for synapse anticipating that people will use the yaml instead. Simpify implementing config options by not requiring the classes to hit the super class --- demo/start.sh | 27 ++++---- synapse/app/homeserver.py | 6 +- synapse/config/_base.py | 141 ++++++++++++++++++++++------------------- synapse/config/appservice.py | 18 ++---- synapse/config/captcha.py | 64 +++++++++---------- synapse/config/database.py | 73 +++++++++++---------- synapse/config/homeserver.py | 4 +- synapse/config/key.py | 139 ++++++++++++++++++---------------------- synapse/config/logger.py | 35 +++++++--- synapse/config/metrics.py | 29 ++++----- synapse/config/ratelimiting.py | 78 ++++++++++------------- synapse/config/registration.py | 47 ++++---------- synapse/config/repository.py | 35 +++++----- synapse/config/server.py | 117 ++++++++++++++++++++-------------- synapse/config/tls.py | 78 +++++++++++------------ synapse/config/voip.py | 43 ++++++------- synapse/server.py | 1 - 17 files changed, 457 insertions(+), 478 deletions(-) (limited to 'synapse/config') diff --git a/demo/start.sh b/demo/start.sh index 0485be8053..941eccd668 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,30 +16,29 @@ if [ $# -eq 1 ]; then fi fi +export PYTHONPATH=$(readlink -f $(pwd)) + + +echo $PYTHONPATH + for port in 8080 8081 8082; do echo "Starting server on port $port... " https_port=$((port + 400)) + mkdir -p demo/$port + pushd demo/$port + rm $DIR/etc/$port.config python -m synapse.app.homeserver \ - --generate-config \ - --config-path "demo/etc/$port.config" \ - -p "$https_port" \ - --unsecure-port "$port" \ - -H "localhost:$https_port" \ - -f "$DIR/$port.log" \ - -d "$DIR/$port.db" \ - -D --pid-file "$DIR/$port.pid" \ - --manhole $((port + 1000)) \ - --tls-dh-params-path "demo/demo.tls.dh" \ - --media-store-path "demo/media_store.$port" \ - $PARAMS $SYNAPSE_PARAMS \ - --enable-registration + --generate-config "localhost:$https_port" \ + --config-path "$DIR/etc/$port.config" \ python -m synapse.app.homeserver \ - --config-path "demo/etc/$port.config" \ + --config-path "$DIR/etc/$port.config" \ + -D \ -vv \ + popd done cd "$CWD" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5c6812f473..c16dd8acc3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -394,7 +394,6 @@ def setup(config_options): config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), - db_name=config.database_path, db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, @@ -407,9 +406,8 @@ def setup(config_options): redirect_root_to_web_client=True, ) - db_name = hs.get_db_name() - logger.info("Preparing database: %s...", db_name) + logger.info("Preparing database: %r...", config.database_config) try: db_conn = database_engine.module.connect( @@ -431,7 +429,7 @@ def setup(config_options): ) sys.exit(1) - logger.info("Database prepared in %s.", db_name) + logger.info("Database prepared in %r.", config.database_config) if config.manhole: f = twisted.manhole.telnet.ShellFactory() diff --git a/synapse/config/_base.py b/synapse/config/_base.py index b59f4e45e2..9f5da70948 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -14,9 +14,10 @@ # limitations under the License. import argparse -import sys import os import yaml +import sys +from textwrap import dedent class ConfigError(Exception): @@ -24,8 +25,6 @@ class ConfigError(Exception): class Config(object): - def __init__(self, args): - pass @staticmethod def parse_size(string): @@ -37,6 +36,22 @@ class Config(object): size = sizes[suffix] return int(string) * size + @staticmethod + def parse_duration(string): + second = 1000 + hour = 60 * 60 * second + day = 24 * hour + week = 7 * day + year = 365 * day + + sizes = {"s": second, "h": hour, "d": day, "w": week, "y": year} + size = 1 + suffix = string[-1] + if suffix in sizes: + string = string[:-1] + size = sizes[suffix] + return int(string) * size + @staticmethod def abspath(file_path): return os.path.abspath(file_path) if file_path else file_path @@ -77,17 +92,6 @@ class Config(object): with open(file_path) as file_stream: return file_stream.read() - @classmethod - def read_yaml_file(cls, file_path, config_name): - cls.check_file(file_path, config_name) - with open(file_path) as file_stream: - try: - return yaml.load(file_stream) - except: - raise ConfigError( - "Error parsing yaml in file %r" % (file_path,) - ) - @staticmethod def default_path(name): return os.path.abspath(os.path.join(os.path.curdir, name)) @@ -97,16 +101,33 @@ class Config(object): with open(file_path) as file_stream: return yaml.load(file_stream) - @classmethod - def add_arguments(cls, parser): - pass + def invoke_all(self, name, *args, **kargs): + results = [] + for cls in type(self).mro(): + if name in cls.__dict__: + results.append(getattr(cls, name)(self, *args, **kargs)) + return results - @classmethod - def generate_config(cls, args, config_dir_path): - pass + def generate_config(self, config_dir_path, server_name): + default_config = "# vim:ft=yaml\n" + + default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all( + "default_config", config_dir_path, server_name + )) + + config = yaml.load(default_config) + + if not os.path.exists(config_dir_path): + os.makedirs(config_dir_path) + + self.invoke_all("generate_keys", config) + + return default_config @classmethod def load_config(cls, description, argv, generate_section=None): + result = cls() + config_parser = argparse.ArgumentParser(add_help=False) config_parser.add_argument( "-c", "--config-path", @@ -115,66 +136,56 @@ class Config(object): ) config_parser.add_argument( "--generate-config", - action="store_true", - help="Generate config file" + metavar="SERVER_NAME", + help="Generate a config file for the server name" ) config_args, remaining_args = config_parser.parse_known_args(argv) - if config_args.generate_config: - if not config_args.config_path: - config_parser.error( - "Must specify where to generate the config file" - ) - config_dir_path = os.path.dirname(config_args.config_path) - if os.path.exists(config_args.config_path): - defaults = cls.read_config_file(config_args.config_path) - else: - defaults = {} - else: - if config_args.config_path: - defaults = cls.read_config_file(config_args.config_path) - else: - defaults = {} - - parser = argparse.ArgumentParser( - parents=[config_parser], - description=description, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - cls.add_arguments(parser) - parser.set_defaults(**defaults) - - args = parser.parse_args(remaining_args) + if not config_args.config_path: + config_parser.error( + "Must supply a config file.\nA config file can be automatically" + " generated using \"--generate-config SERVER_NAME" + " -c CONFIG-FILE\"" + ) if config_args.generate_config: + server_name = config_args.generate_config + config_path = config_args.config_path + if os.path.exists(config_path): + print "Config file %r already exists. Not overwriting" % ( + config_args.config_path + ) + sys.exit(0) config_dir_path = os.path.dirname(config_args.config_path) config_dir_path = os.path.abspath(config_dir_path) - if not os.path.exists(config_dir_path): - os.makedirs(config_dir_path) - cls.generate_config(args, config_dir_path) - config = {} - for key, value in vars(args).items(): - if (key not in set(["config_path", "generate_config"]) - and value is not None): - config[key] = value - with open(config_args.config_path, "w") as config_file: - # TODO(mark/paul) We might want to output emacs-style mode - # markers as well as vim-style mode markers into the file, - # to further hint to people this is a YAML file. - config_file.write("# vim:ft=yaml\n") - yaml.dump(config, config_file, default_flow_style=False) + with open(config_path, "wb") as config_file: + config_file.write( + result.generate_config(config_dir_path, server_name) + ) print ( "A config file has been generated in %s for server name" " '%s' with corresponding SSL keys and self-signed" " certificates. Please review this file and customise it to" " your needs." - ) % ( - config_args.config_path, config['server_name'] - ) + ) % (config_path, server_name) print ( "If this server name is incorrect, you will need to regenerate" " the SSL certificates" ) sys.exit(0) - return cls(args) + config = cls.read_config_file(config_args.config_path) + result.invoke_all("read_config", config) + + parser = argparse.ArgumentParser( + parents=[config_parser], + description=description, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + result.invoke_all("add_arguments", parser) + args = parser.parse_args(remaining_args) + + result.invoke_all("read_arguments", args) + + return result diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 399a716d80..38f41933b7 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -17,15 +17,11 @@ from ._base import Config class AppServiceConfig(Config): - def __init__(self, args): - super(AppServiceConfig, self).__init__(args) - self.app_service_config_files = args.app_service_config_files + def read_config(self, config): + self.app_service_config_files = config.get("app_service_config_files", []) - @classmethod - def add_arguments(cls, parser): - super(AppServiceConfig, cls).add_arguments(parser) - group = parser.add_argument_group("appservice") - group.add_argument( - "--app-service-config-files", type=str, nargs='+', - help="A list of application service config files to use." - ) + def default_config(cls, config_dir_path, server_name): + return """\ + # A list of application service config file to use + app_service_config_files: [] + """ diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 07fbfadc0f..ba7037aeb1 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -17,40 +17,34 @@ from ._base import Config class CaptchaConfig(Config): - def __init__(self, args): - super(CaptchaConfig, self).__init__(args) - self.recaptcha_private_key = args.recaptcha_private_key - self.recaptcha_public_key = args.recaptcha_public_key - self.enable_registration_captcha = args.enable_registration_captcha + def read_config(self, config): + self.recaptcha_private_key = config["recaptcha_private_key"] + self.recaptcha_public_key = config["recaptcha_public_key"] + self.enable_registration_captcha = config["enable_registration_captcha"] self.captcha_ip_origin_is_x_forwarded = ( - args.captcha_ip_origin_is_x_forwarded - ) - self.captcha_bypass_secret = args.captcha_bypass_secret - - @classmethod - def add_arguments(cls, parser): - super(CaptchaConfig, cls).add_arguments(parser) - group = parser.add_argument_group("recaptcha") - group.add_argument( - "--recaptcha-public-key", type=str, default="YOUR_PUBLIC_KEY", - help="This Home Server's ReCAPTCHA public key." - ) - group.add_argument( - "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY", - help="This Home Server's ReCAPTCHA private key." - ) - group.add_argument( - "--enable-registration-captcha", type=bool, default=False, - help="Enables ReCaptcha checks when registering, preventing signup" - + " unless a captcha is answered. Requires a valid ReCaptcha " - + "public/private key." - ) - group.add_argument( - "--captcha_ip_origin_is_x_forwarded", type=bool, default=False, - help="When checking captchas, use the X-Forwarded-For (XFF) header" - + " as the client IP and not the actual client IP." - ) - group.add_argument( - "--captcha_bypass_secret", type=str, - help="A secret key used to bypass the captcha test entirely." + config["captcha_ip_origin_is_x_forwarded"] ) + self.captcha_bypass_secret = config.get("captcha_bypass_secret") + + def default_config(self, config_dir_path, server_name): + return """\ + ## Captcha ## + + # This Home Server's ReCAPTCHA public key. + recaptcha_private_key: "YOUR_PUBLIC_KEY" + + # This Home Server's ReCAPTCHA private key. + recaptcha_public_key: "YOUR_PRIVATE_KEY" + + # Enables ReCaptcha checks when registering, preventing signup + # unless a captcha is answered. Requires a valid ReCaptcha + # public/private key. + enable_registration_captcha: False + + # When checking captchas, use the X-Forwarded-For (XFF) header + # as the client IP and not the actual client IP. + captcha_ip_origin_is_x_forwarded: False + + # A secret key used to bypass the captcha test entirely. + captcha_bypass_secret: ~ + """ diff --git a/synapse/config/database.py b/synapse/config/database.py index 190d119df4..ccd96c4f0f 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -14,28 +14,21 @@ # limitations under the License. from ._base import Config -import os -import yaml class DatabaseConfig(Config): - def __init__(self, args): - super(DatabaseConfig, self).__init__(args) - if args.database_path == ":memory:": - self.database_path = ":memory:" - else: - self.database_path = self.abspath(args.database_path) - self.event_cache_size = self.parse_size(args.event_cache_size) - if args.database_config: - with open(args.database_config) as f: - self.database_config = yaml.safe_load(f) - else: + def read_config(self, config): + self.event_cache_size = self.parse_size( + config.get("event_cache_size", "10K") + ) + + self.database_config = config.get("database") + + if self.database_config is None: self.database_config = { "name": "sqlite3", - "args": { - "database": self.database_path, - }, + "args": {}, } name = self.database_config.get("name", None) @@ -50,24 +43,36 @@ class DatabaseConfig(Config): else: raise RuntimeError("Unsupported database type '%s'" % (name,)) - @classmethod - def add_arguments(cls, parser): - super(DatabaseConfig, cls).add_arguments(parser) + self.set_databasepath(config.get("database_path")) + + def default_config(self, config, config_dir_path): + database_path = self.abspath("homeserver.db") + return """\ + # Database configuration + database: + # The database engine name + name: "sqlite3" + # Arguments to pass to the engine + args: + # Path to the database + database: "%(database_path)s" + # Number of events to cache in memory. + event_cache_size: "10K" + """ % locals() + + def read_arguments(self, args): + self.set_databasepath(args.database_path) + + def set_databasepath(self, database_path): + if database_path != ":memory:": + database_path = self.abspath(database_path) + if self.database_config.get("name", None) == "sqlite3": + if database_path is not None: + self.database_config["database"] = database_path + + def add_arguments(self, parser): db_group = parser.add_argument_group("database") db_group.add_argument( - "-d", "--database-path", default="homeserver.db", - metavar="SQLITE_DATABASE_PATH", help="The database name." - ) - db_group.add_argument( - "--event-cache-size", default="100K", - help="Number of events to cache in memory." + "-d", "--database-path", metavar="SQLITE_DATABASE_PATH", + help="The path to a sqlite database to use." ) - db_group.add_argument( - "--database-config", default=None, - help="Location of the database configuration file." - ) - - @classmethod - def generate_config(cls, args, config_dir_path): - super(DatabaseConfig, cls).generate_config(args, config_dir_path) - args.database_path = os.path.abspath(args.database_path) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 1c8ff38465..f9b4807a35 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -36,4 +36,6 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, if __name__ == '__main__': import sys - HomeServerConfig.load_config("Generate config", sys.argv[1:], "HomeServer") + sys.stdout.write( + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2]) + ) diff --git a/synapse/config/key.py b/synapse/config/key.py index a2de6d5c17..a63f7d841b 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -24,44 +24,53 @@ from syutil.base64util import decode_base64 class KeyConfig(Config): - def __init__(self, args): - super(KeyConfig, self).__init__(args) - self.signing_key = self.read_signing_key(args.signing_key_path) + def read_config(self, config): + self.signing_key = self.read_signing_key(config["signing_key_path"]) self.old_signing_keys = self.read_old_signing_keys( - args.old_signing_key_path + config["old_signing_keys"] + ) + self.key_refresh_interval = self.parse_duration( + config["key_refresh_interval"] ) - self.key_refresh_interval = args.key_refresh_interval self.perspectives = self.read_perspectives( - args.perspectives_config_path + config["perspectives"] ) - @classmethod - def add_arguments(cls, parser): - super(KeyConfig, cls).add_arguments(parser) - key_group = parser.add_argument_group("keys") - key_group.add_argument("--signing-key-path", - help="The signing key to sign messages with") - key_group.add_argument("--old-signing-key-path", - help="The keys that the server used to sign" - " sign messages with but won't use" - " to sign new messages. E.g. it has" - " lost its private key") - key_group.add_argument("--key-refresh-interval", - default=24 * 60 * 60 * 1000, # 1 Day - help="How long a key response is valid for." - " Used to set the exipiry in /key/v2/." - " Controls how frequently servers will" - " query what keys are still valid") - key_group.add_argument("--perspectives-config-path", - help="The trusted servers to download signing" - " keys from") - - def read_perspectives(self, perspectives_config_path): - config = self.read_yaml_file( - perspectives_config_path, "perspectives_config_path" - ) + def default_config(self, config_dir_path, server_name): + base_key_name = os.path.join(config_dir_path, server_name) + return """\ + ## Signing Keys ## + + # Path to the signing key to sign messages with + signing_key_path: "%(base_key_name)s.signing.key" + + # The keys that the server used to sign messages with but won't use + # to sign new messages. E.g. it has lost its private key + old_signing_keys: {} + # "ed25519:auto": + # # Base64 encoded public key + # key: "The public part of your old signing key." + # # Millisecond POSIX timestamp when the key expired. + # expired_ts: 123456789123 + + # How long key response published by this server is valid for. + # Used to set the valid_until_ts in /key/v2 APIs. + # Determines how quickly servers will query to check which keys + # are still valid. + key_refresh_interval: "1d" # 1 Day. + + # The trusted servers to download signing keys from. + perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + """ % locals() + + def read_perspectives(self, perspectives_config): servers = {} - for server_name, server_config in config["servers"].items(): + for server_name, server_config in perspectives_config["servers"].items(): for key_id, key_data in server_config["verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_base64 = key_data["key"] @@ -82,37 +91,31 @@ class KeyConfig(Config): " Try running again with --generate-config" ) - def read_old_signing_keys(self, old_signing_key_path): - old_signing_keys = self.read_file( - old_signing_key_path, "old_signing_key" - ) - try: - return syutil.crypto.signing_key.read_old_signing_keys( - old_signing_keys.splitlines(True) - ) - except Exception: - raise ConfigError( - "Error reading old signing keys." - ) - - @classmethod - def generate_config(cls, args, config_dir_path): - super(KeyConfig, cls).generate_config(args, config_dir_path) - base_key_name = os.path.join(config_dir_path, args.server_name) - - args.pid_file = os.path.abspath(args.pid_file) - - if not args.signing_key_path: - args.signing_key_path = base_key_name + ".signing.key" + def read_old_signing_keys(self, old_signing_keys): + keys = {} + for key_id, key_data in old_signing_keys.items(): + if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_key.expired_ts = key_data["expired_ts"] + keys[key_id] = verify_key + else: + raise ConfigError( + "Unsupported signing algorithm for old key: %r" % (key_id,) + ) + return keys - if not os.path.exists(args.signing_key_path): - with open(args.signing_key_path, "w") as signing_key_file: + def generate_keys(self, config): + signing_key_path = config["signing_key_path"] + if not os.path.exists(signing_key_path): + with open(signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, (syutil.crypto.signing_key.generate_signing_key("auto"),), ) else: - signing_keys = cls.read_file(args.signing_key_path, "signing_key") + signing_keys = self.read_file(signing_key_path, "signing_key") if len(signing_keys.split("\n")[0].split()) == 1: # handle keys in the old format. key = syutil.crypto.signing_key.decode_signing_key_base64( @@ -120,28 +123,8 @@ class KeyConfig(Config): "auto", signing_keys.split("\n")[0] ) - with open(args.signing_key_path, "w") as signing_key_file: + with open(signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, (key,), ) - - if not args.old_signing_key_path: - args.old_signing_key_path = base_key_name + ".old.signing.keys" - - if not os.path.exists(args.old_signing_key_path): - with open(args.old_signing_key_path, "w"): - pass - - if not args.perspectives_config_path: - args.perspectives_config_path = base_key_name + ".perspectives" - - if not os.path.exists(args.perspectives_config_path): - with open(args.perspectives_config_path, "w") as perspectives_file: - perspectives_file.write( - 'servers:\n' - ' matrix.org:\n' - ' verify_keys:\n' - ' "ed25519:auto":\n' - ' key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"\n' - ) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 247b324816..37b3d5342c 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -22,22 +22,41 @@ import yaml class LoggingConfig(Config): - def __init__(self, args): - super(LoggingConfig, self).__init__(args) - self.verbosity = int(args.verbose) if args.verbose else None - self.log_config = self.abspath(args.log_config) - self.log_file = self.abspath(args.log_file) - @classmethod + def read_config(self, config): + self.verbosity = config.get("verbose", 0) + self.log_config = self.abspath(config.get("log_config")) + self.log_file = self.abspath(config.get("log_file")) + + def default_config(self, config_dir_path, server_name): + log_file = self.abspath("homeserver.log") + return """ + # Logging verbosity level. + verbose: 0 + + # File to write logging to + log_file: "%(log_file)s" + + # A yaml python logging config file + #log_config: "your.log.config.yaml" + """ % locals() + + def read_arguments(self, args): + if args.verbose is not None: + self.verbosity = args.verbose + if args.log_config is not None: + self.log_config = args.log_config + if args.log_file is not None: + self.log_file = args.log_file + def add_arguments(cls, parser): - super(LoggingConfig, cls).add_arguments(parser) logging_group = parser.add_argument_group("logging") logging_group.add_argument( '-v', '--verbose', dest="verbose", action='count', help="The verbosity level." ) logging_group.add_argument( - '-f', '--log-file', dest="log_file", default="homeserver.log", + '-f', '--log-file', dest="log_file", help="File to log to." ) logging_group.add_argument( diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 901a429c76..06e2e7ccff 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -17,20 +17,17 @@ from ._base import Config class MetricsConfig(Config): - def __init__(self, args): - super(MetricsConfig, self).__init__(args) - self.enable_metrics = args.enable_metrics - self.metrics_port = args.metrics_port + def read_config(self, config): + self.enable_metrics = config["enable_metrics"] + self.metrics_port = config["metrics_port"] - @classmethod - def add_arguments(cls, parser): - super(MetricsConfig, cls).add_arguments(parser) - metrics_group = parser.add_argument_group("metrics") - metrics_group.add_argument( - '--enable-metrics', dest="enable_metrics", action="store_true", - help="Enable collection and rendering of performance metrics" - ) - metrics_group.add_argument( - '--metrics-port', metavar="PORT", type=int, - help="Separate port to accept metrics requests on (on localhost)" - ) + def default_config(self, config_dir_path, server_name): + return """\ + ## Metrics ### + + # Enable collection and rendering of performance metrics + enable_metrics: False + + # Separate port to accept metrics requests on (on localhost) + metrics_port: ~ + """ diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 862c07ef8c..76d9970e5b 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -17,56 +17,42 @@ from ._base import Config class RatelimitConfig(Config): - def __init__(self, args): - super(RatelimitConfig, self).__init__(args) - self.rc_messages_per_second = args.rc_messages_per_second - self.rc_message_burst_count = args.rc_message_burst_count + def read_config(self, config): + self.rc_messages_per_second = config["rc_messages_per_second"] + self.rc_message_burst_count = config["rc_message_burst_count"] - self.federation_rc_window_size = args.federation_rc_window_size - self.federation_rc_sleep_limit = args.federation_rc_sleep_limit - self.federation_rc_sleep_delay = args.federation_rc_sleep_delay - self.federation_rc_reject_limit = args.federation_rc_reject_limit - self.federation_rc_concurrent = args.federation_rc_concurrent + self.federation_rc_window_size = config["federation_rc_window_size"] + self.federation_rc_sleep_limit = config["federation_rc_sleep_limit"] + self.federation_rc_sleep_delay = config["federation_rc_sleep_delay"] + self.federation_rc_reject_limit = config["federation_rc_reject_limit"] + self.federation_rc_concurrent = config["federation_rc_concurrent"] - @classmethod - def add_arguments(cls, parser): - super(RatelimitConfig, cls).add_arguments(parser) - rc_group = parser.add_argument_group("ratelimiting") - rc_group.add_argument( - "--rc-messages-per-second", type=float, default=0.2, - help="number of messages a client can send per second" - ) - rc_group.add_argument( - "--rc-message-burst-count", type=float, default=10, - help="number of message a client can send before being throttled" - ) + def default_config(self, config_dir_path, server_name): + return """\ + ## Ratelimiting ## - rc_group.add_argument( - "--federation-rc-window-size", type=int, default=10000, - help="The federation window size in milliseconds", - ) + # Number of messages a client can send per second + rc_messages_per_second: 0.2 - rc_group.add_argument( - "--federation-rc-sleep-limit", type=int, default=10, - help="The number of federation requests from a single server" - " in a window before the server will delay processing the" - " request.", - ) + # Number of message a client can send before being throttled + rc_message_burst_count: 10.0 - rc_group.add_argument( - "--federation-rc-sleep-delay", type=int, default=500, - help="The duration in milliseconds to delay processing events from" - " remote servers by if they go over the sleep limit.", - ) + # The federation window size in milliseconds + federation_rc_window_size: 1000 - rc_group.add_argument( - "--federation-rc-reject-limit", type=int, default=50, - help="The maximum number of concurrent federation requests allowed" - " from a single server", - ) + # The number of federation requests from a single server in a window + # before the server will delay processing the request. + federation_rc_sleep_limit: 10 - rc_group.add_argument( - "--federation-rc-concurrent", type=int, default=3, - help="The number of federation requests to concurrently process" - " from a single server", - ) + # The duration in milliseconds to delay processing events from + # remote servers by if they go over the sleep limit. + federation_rc_sleep_delay: 500 + + # The maximum number of concurrent federation requests allowed + # from a single server + federation_rc_reject_limit: 50 + + # The number of federation requests to concurrently process from a + # single server + federation_rc_concurrent: 3 + """ diff --git a/synapse/config/registration.py b/synapse/config/registration.py index f412a72f59..82684e4dc9 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -22,40 +22,21 @@ import distutils.util class RegistrationConfig(Config): - def __init__(self, args): - super(RegistrationConfig, self).__init__(args) - - # `args.enable_registration` may either be a bool or a string depending - # on if the option was given a value (e.g. --enable-registration=true - # would set `args.enable_registration` to "true" not True.) + def read_config(self, config): self.disable_registration = not bool( - distutils.util.strtobool(str(args.enable_registration)) - ) - self.registration_shared_secret = args.registration_shared_secret - - @classmethod - def add_arguments(cls, parser): - super(RegistrationConfig, cls).add_arguments(parser) - reg_group = parser.add_argument_group("registration") - - reg_group.add_argument( - "--enable-registration", - const=True, - default=False, - nargs='?', - help="Enable registration for new users.", - ) - reg_group.add_argument( - "--registration-shared-secret", type=str, - help="If set, allows registration by anyone who also has the shared" - " secret, even if registration is otherwise disabled.", + distutils.util.strtobool(str(config["enable_registration"])) ) + self.registration_shared_secret = config.get("registration_shared_secret") + + def default_config(self, config_dir, server_name): + registration_shared_secret = random_string_with_symbols(50) + return """\ + ## Registration ## - @classmethod - def generate_config(cls, args, config_dir_path): - super(RegistrationConfig, cls).generate_config(args, config_dir_path) - if args.enable_registration is None: - args.enable_registration = False + # Enable registration for new users. + enable_registration: True - if args.registration_shared_secret is None: - args.registration_shared_secret = random_string_with_symbols(50) + # If set, allows registration by anyone who also has the shared + # secret, even if registration is otherwise disabled. + registration_shared_secret: "%(registration_shared_secret)s" + """ % locals() diff --git a/synapse/config/repository.py b/synapse/config/repository.py index e1827f05e4..bf727285d7 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -17,11 +17,10 @@ from ._base import Config class ContentRepositoryConfig(Config): - def __init__(self, args): - super(ContentRepositoryConfig, self).__init__(args) - self.max_upload_size = self.parse_size(args.max_upload_size) - self.max_image_pixels = self.parse_size(args.max_image_pixels) - self.media_store_path = self.ensure_directory(args.media_store_path) + def read_config(self, config): + self.max_upload_size = self.parse_size(config["max_upload_size"]) + self.max_image_pixels = self.parse_size(config["max_image_pixels"]) + self.media_store_path = self.ensure_directory(config["media_store_path"]) def parse_size(self, string): sizes = {"K": 1024, "M": 1024 * 1024} @@ -32,17 +31,15 @@ class ContentRepositoryConfig(Config): size = sizes[suffix] return int(string) * size - @classmethod - def add_arguments(cls, parser): - super(ContentRepositoryConfig, cls).add_arguments(parser) - db_group = parser.add_argument_group("content_repository") - db_group.add_argument( - "--max-upload-size", default="10M" - ) - db_group.add_argument( - "--media-store-path", default=cls.default_path("media_store") - ) - db_group.add_argument( - "--max-image-pixels", default="32M", - help="Maximum number of pixels that will be thumbnailed" - ) + def default_config(self, config_dir_path, server_name): + media_store = self.default_path("media_store") + return """ + # Directory where uploaded images and attachments are stored. + media_store_path: "%(media_store)s" + + # The largest allowed upload size in bytes + max_upload_size: "10M" + + # Maximum number of pixels that will be thumbnailed + max_image_pixels: "32M" + """ % locals() diff --git a/synapse/config/server.py b/synapse/config/server.py index c25feb4c58..fe1b63469a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -17,64 +17,85 @@ from ._base import Config class ServerConfig(Config): - def __init__(self, args): - super(ServerConfig, self).__init__(args) - self.server_name = args.server_name - self.bind_port = args.bind_port - self.bind_host = args.bind_host - self.unsecure_port = args.unsecure_port - self.daemonize = args.daemonize - self.pid_file = self.abspath(args.pid_file) - self.web_client = args.web_client - self.manhole = args.manhole - self.soft_file_limit = args.soft_file_limit - if not args.content_addr: - host = args.server_name + def read_config(self, config): + self.server_name = config["server_name"] + self.bind_port = config["bind_port"] + self.bind_host = config["bind_host"] + self.unsecure_port = config["unsecure_port"] + self.manhole = config["manhole"] + self.pid_file = self.abspath(config.get("pid_file")) + self.web_client = config["web_client"] + self.soft_file_limit = config["soft_file_limit"] + + # Attempt to guess the content_addr for the v0 content repostitory + content_addr = config.get("content_addr") + if not content_addr: + host = self.server_name if ':' not in host: - host = "%s:%d" % (host, args.unsecure_port) + host = "%s:%d" % (host, self.unsecure_port) else: host = host.split(':')[0] - host = "%s:%d" % (host, args.unsecure_port) - args.content_addr = "http://%s" % (host,) + host = "%s:%d" % (host, self.unsecure_port) + content_addr = "http://%s" % (host,) + + self.content_addr = content_addr + + def default_config(self, config_dir_path, server_name): + if ":" in server_name: + bind_port = int(server_name.split(":")[1]) + unsecure_port = bind_port - 400 + else: + bind_port = 8448 + unsecure_port = 8008 + + pid_file = self.abspath("homeserver.pid") + return """\ + ## Server ## + + # The domain name of the server, with optional explicit port. + # This is used by remote servers to connect to this server, + # e.g. matrix.org, localhost:8080, etc. + server_name: "%(server_name)s" + + # The port to listen for HTTPS requests on. + # For when matrix traffic is sent directly to synapse. + bind_port: %(bind_port)s - self.content_addr = args.content_addr + # The port to listen for HTTP requests on. + # For when matrix traffic passes through loadbalancer that unwraps TLS. + unsecure_port: %(unsecure_port)s + + # Local interface to listen on. + # The empty string will cause synapse to listen on all interfaces. + bind_host: "" + + # When running as a daemon, the file to store the pid in + pid_file: %(pid_file)s + + # Whether to serve a web client from the HTTP/HTTPS root resource. + web_client: True + + # Set the soft limit on the number of file descriptors synapse can use + # Zero is used to indicate synapse should set the soft limit to the + # hard limit. + soft_file_limit: 0 + + # Turn on the twisted telnet manhole service on localhost on the given + # port. + manhole: ~ + """ % locals() + + def read_arguments(self, args): + if args.manhole is not None: + self.manhole = args.manhole + self.daemonize = args.daemonize - @classmethod - def add_arguments(cls, parser): - super(ServerConfig, cls).add_arguments(parser) + def add_arguments(self, parser): server_group = parser.add_argument_group("server") - server_group.add_argument( - "-H", "--server-name", default="localhost", - help="The domain name of the server, with optional explicit port. " - "This is used by remote servers to connect to this server, " - "e.g. matrix.org, localhost:8080, etc." - ) - server_group.add_argument("-p", "--bind-port", metavar="PORT", - type=int, help="https port to listen on", - default=8448) - server_group.add_argument("--unsecure-port", metavar="PORT", - type=int, help="http port to listen on", - default=8008) - server_group.add_argument("--bind-host", default="", - help="Local interface to listen on") server_group.add_argument("-D", "--daemonize", action='store_true', help="Daemonize the home server") - server_group.add_argument('--pid-file', default="homeserver.pid", - help="When running as a daemon, the file to" - " store the pid in") - server_group.add_argument('--web_client', default=True, type=bool, - help="Whether or not to serve a web client") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, help="Turn on the twisted telnet manhole" " service on the given port.") - server_group.add_argument("--content-addr", default=None, - help="The host and scheme to use for the " - "content repository") - server_group.add_argument("--soft-file-limit", type=int, default=0, - help="Set the soft limit on the number of " - "file descriptors synapse can use. " - "Zero is used to indicate synapse " - "should set the soft limit to the hard" - "limit.") diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 034f9a7bf0..e70bc1cd2c 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -23,37 +23,44 @@ GENERATE_DH_PARAMS = False class TlsConfig(Config): - def __init__(self, args): - super(TlsConfig, self).__init__(args) + def read_config(self, config): self.tls_certificate = self.read_tls_certificate( - args.tls_certificate_path + config.get("tls_certificate_path") ) - self.no_tls = args.no_tls + self.no_tls = config.get("no_tls", False) if self.no_tls: self.tls_private_key = None else: self.tls_private_key = self.read_tls_private_key( - args.tls_private_key_path + config.get("tls_private_key_path") ) self.tls_dh_params_path = self.check_file( - args.tls_dh_params_path, "tls_dh_params" + config.get("tls_dh_params_path"), "tls_dh_params" ) - @classmethod - def add_arguments(cls, parser): - super(TlsConfig, cls).add_arguments(parser) - tls_group = parser.add_argument_group("tls") - tls_group.add_argument("--tls-certificate-path", - help="PEM encoded X509 certificate for TLS") - tls_group.add_argument("--tls-private-key-path", - help="PEM encoded private key for TLS") - tls_group.add_argument("--tls-dh-params-path", - help="PEM dh parameters for ephemeral keys") - tls_group.add_argument("--no-tls", action='store_true', - help="Don't bind to the https port.") + def default_config(self, config_dir_path, server_name): + base_key_name = os.path.join(config_dir_path, server_name) + + tls_certificate_path = base_key_name + ".tls.crt" + tls_private_key_path = base_key_name + ".tls.key" + tls_dh_params_path = base_key_name + ".tls.dh" + + return """\ + # PEM encoded X509 certificate for TLS + tls_certificate_path: "%(tls_certificate_path)s" + + # PEM encoded private key for TLS + tls_private_key_path: "%(tls_private_key_path)s" + + # PEM dh parameters for ephemeral keys + tls_dh_params_path: "%(tls_dh_params_path)s" + + # Don't bind to the https port + no_tls: False + """ % locals() def read_tls_certificate(self, cert_path): cert_pem = self.read_file(cert_path, "tls_certificate") @@ -63,22 +70,13 @@ class TlsConfig(Config): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - @classmethod - def generate_config(cls, args, config_dir_path): - super(TlsConfig, cls).generate_config(args, config_dir_path) - base_key_name = os.path.join(config_dir_path, args.server_name) - - if args.tls_certificate_path is None: - args.tls_certificate_path = base_key_name + ".tls.crt" - - if args.tls_private_key_path is None: - args.tls_private_key_path = base_key_name + ".tls.key" - - if args.tls_dh_params_path is None: - args.tls_dh_params_path = base_key_name + ".tls.dh" + def generate_keys(self, config): + tls_certificate_path = config["tls_certificate_path"] + tls_private_key_path = config["tls_private_key_path"] + tls_dh_params_path = config["tls_dh_params_path"] - if not os.path.exists(args.tls_private_key_path): - with open(args.tls_private_key_path, "w") as private_key_file: + if not os.path.exists(tls_private_key_path): + with open(tls_private_key_path, "w") as private_key_file: tls_private_key = crypto.PKey() tls_private_key.generate_key(crypto.TYPE_RSA, 2048) private_key_pem = crypto.dump_privatekey( @@ -86,17 +84,17 @@ class TlsConfig(Config): ) private_key_file.write(private_key_pem) else: - with open(args.tls_private_key_path) as private_key_file: + with open(tls_private_key_path) as private_key_file: private_key_pem = private_key_file.read() tls_private_key = crypto.load_privatekey( crypto.FILETYPE_PEM, private_key_pem ) - if not os.path.exists(args.tls_certificate_path): - with open(args.tls_certificate_path, "w") as certifcate_file: + if not os.path.exists(tls_certificate_path): + with open(tls_certificate_path, "w") as certifcate_file: cert = crypto.X509() subject = cert.get_subject() - subject.CN = args.server_name + subject.CN = config["server_name"] cert.set_serial_number(1000) cert.gmtime_adj_notBefore(0) @@ -110,16 +108,16 @@ class TlsConfig(Config): certifcate_file.write(cert_pem) - if not os.path.exists(args.tls_dh_params_path): + if not os.path.exists(tls_dh_params_path): if GENERATE_DH_PARAMS: subprocess.check_call([ "openssl", "dhparam", "-outform", "PEM", - "-out", args.tls_dh_params_path, + "-out", tls_dh_params_path, "2048" ]) else: - with open(args.tls_dh_params_path, "w") as dh_params_file: + with open(tls_dh_params_path, "w") as dh_params_file: dh_params_file.write( "2048-bit DH parameters taken from rfc3526\n" "-----BEGIN DH PARAMETERS-----\n" diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 65162d21b7..a1707223d3 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -17,28 +17,21 @@ from ._base import Config class VoipConfig(Config): - def __init__(self, args): - super(VoipConfig, self).__init__(args) - self.turn_uris = args.turn_uris - self.turn_shared_secret = args.turn_shared_secret - self.turn_user_lifetime = args.turn_user_lifetime - - @classmethod - def add_arguments(cls, parser): - super(VoipConfig, cls).add_arguments(parser) - group = parser.add_argument_group("voip") - group.add_argument( - "--turn-uris", type=str, default=None, action='append', - help="The public URIs of the TURN server to give to clients" - ) - group.add_argument( - "--turn-shared-secret", type=str, default=None, - help=( - "The shared secret used to compute passwords for the TURN" - " server" - ) - ) - group.add_argument( - "--turn-user-lifetime", type=int, default=(1000 * 60 * 60), - help="How long generated TURN credentials last, in ms" - ) + def read_config(self, config): + self.turn_uris = config.get("turn_uris", []) + self.turn_shared_secret = config["turn_shared_secret"] + self.turn_user_lifetime = self.parse_duration(config["turn_user_lifetime"]) + + def default_config(self, config_dir_path, server_name): + return """\ + ## Turn ## + + # The public URIs of the TURN server to give to clients + turn_uris: [] + + # The shared secret used to compute passwords for the TURN server + turn_shared_secret: "YOUR_SHARED_SECRET" + + # How long generated TURN credentials last + turn_user_lifetime: "1h" + """ diff --git a/synapse/server.py b/synapse/server.py index d61a228c36..8b3dc675cc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -59,7 +59,6 @@ class BaseHomeServer(object): 'config', 'clock', 'http_client', - 'db_name', 'db_pool', 'persistence_service', 'replication_layer', -- cgit 1.4.1 From 6b69ddd17a9fe75544ce32b402042f2d50826874 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 04:26:29 +0100 Subject: remove duplicate parse_size method --- synapse/app/homeserver.py | 1 - synapse/config/repository.py | 9 --------- 2 files changed, 10 deletions(-) (limited to 'synapse/config') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index c16dd8acc3..e6a34561c1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -406,7 +406,6 @@ def setup(config_options): redirect_root_to_web_client=True, ) - logger.info("Preparing database: %r...", config.database_config) try: diff --git a/synapse/config/repository.py b/synapse/config/repository.py index bf727285d7..adaf4e4bb2 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -22,15 +22,6 @@ class ContentRepositoryConfig(Config): self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) - def parse_size(self, string): - sizes = {"K": 1024, "M": 1024 * 1024} - size = 1 - suffix = string[-1] - if suffix in sizes: - string = string[:-1] - size = sizes[suffix] - return int(string) * size - def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") return """ -- cgit 1.4.1 From 1aa11cf7cef83f529bb6f48a76f2d2fe10a7cfe4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 13:48:15 +0100 Subject: Allow multiple config files, set up a default config before applying the config files --- demo/start.sh | 3 ++- synapse/config/_base.py | 58 ++++++++++++++++++++++++++++---------------- synapse/config/homeserver.py | 2 +- 3 files changed, 40 insertions(+), 23 deletions(-) (limited to 'synapse/config') diff --git a/demo/start.sh b/demo/start.sh index 941eccd668..ef4be2d5ff 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -30,7 +30,8 @@ for port in 8080 8081 8082; do rm $DIR/etc/$port.config python -m synapse.app.homeserver \ - --generate-config "localhost:$https_port" \ + --generate-config \ + -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ python -m synapse.app.homeserver \ diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 9f5da70948..d98b6aaedf 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -117,51 +117,59 @@ class Config(object): config = yaml.load(default_config) - if not os.path.exists(config_dir_path): - os.makedirs(config_dir_path) - - self.invoke_all("generate_keys", config) - - return default_config + return default_config, config @classmethod def load_config(cls, description, argv, generate_section=None): - result = cls() + obj = cls() config_parser = argparse.ArgumentParser(add_help=False) config_parser.add_argument( "-c", "--config-path", + action="append", metavar="CONFIG_FILE", help="Specify config file" ) config_parser.add_argument( "--generate-config", - metavar="SERVER_NAME", + action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "-H", "--server-name", + help="The server name to generate a config file for" + ) config_args, remaining_args = config_parser.parse_known_args(argv) if not config_args.config_path: config_parser.error( "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config SERVER_NAME" + " generated using \"--generate-config -h SERVER_NAME" " -c CONFIG-FILE\"" ) + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) if config_args.generate_config: - server_name = config_args.generate_config - config_path = config_args.config_path + server_name = config_args.server_name + if not server_name: + print "Most specify a server_name to a generate config for." + sys.exit(1) + (config_path,) = config_args.config_path if os.path.exists(config_path): print "Config file %r already exists. Not overwriting" % ( config_args.config_path ) - sys.exit(0) - config_dir_path = os.path.dirname(config_args.config_path) - config_dir_path = os.path.abspath(config_dir_path) + sys.exit(1) + if not os.path.exists(config_dir_path): + os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: - config_file.write( - result.generate_config(config_dir_path, server_name) + + config_bytes, config = obj.generate_config( + config_dir_path, server_name ) + obj.invoke_all("generate_keys", config) + config_file.write(config_bytes) print ( "A config file has been generated in %s for server name" " '%s' with corresponding SSL keys and self-signed" @@ -174,8 +182,16 @@ class Config(object): ) sys.exit(0) - config = cls.read_config_file(config_args.config_path) - result.invoke_all("read_config", config) + specified_config = {} + for config_path in config_args.config_path: + yaml_config = cls.read_config_file(config_path) + specified_config.update(yaml_config) + + server_name = specified_config["server_name"] + _, config = obj.generate_config(config_dir_path, server_name) + config.update(specified_config) + + obj.invoke_all("read_config", config) parser = argparse.ArgumentParser( parents=[config_parser], @@ -183,9 +199,9 @@ class Config(object): formatter_class=argparse.RawDescriptionHelpFormatter, ) - result.invoke_all("add_arguments", parser) + obj.invoke_all("add_arguments", parser) args = parser.parse_args(remaining_args) - result.invoke_all("read_arguments", args) + obj.invoke_all("read_arguments", args) - return result + return obj diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index f9b4807a35..fe0ccb6eb7 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -37,5 +37,5 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, if __name__ == '__main__': import sys sys.stdout.write( - HomeServerConfig().generate_config(sys.argv[1], sys.argv[2]) + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0] ) -- cgit 1.4.1 From d89a9f72833acfd472ec557a8fe3927320efea93 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 13:58:13 +0100 Subject: Add an access_log SYN-161 #resolve --- synapse/app/homeserver.py | 25 ++++++++++++++++++++++--- synapse/config/captcha.py | 2 ++ synapse/config/logger.py | 5 +++++ 3 files changed, 29 insertions(+), 3 deletions(-) (limited to 'synapse/config') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5c6812f473..0aa5c34c81 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,6 +35,7 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site +from twisted.web.http import proxiedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource @@ -225,10 +226,18 @@ class SynapseHomeServer(HomeServer): def start_listening(self): config = self.get_config() + log_formatter = None + if config.captcha_ip_origin_is_x_forwarded: + log_formatter = proxiedLogFormatter + if not config.no_tls and config.bind_port is not None: reactor.listenSSL( config.bind_port, - Site(self.root_resource), + Site( + self.root_resource, + logPath=config.access_log_file, + logFormatter=log_formatter, + ), self.tls_context_factory, interface=config.bind_host ) @@ -237,7 +246,11 @@ class SynapseHomeServer(HomeServer): if config.unsecure_port is not None: reactor.listenTCP( config.unsecure_port, - Site(self.root_resource), + Site( + self.root_resource, + logPath=config.access_log_file, + logFormatter=log_formatter, + ), interface=config.bind_host ) logger.info("Synapse now listening on port %d", config.unsecure_port) @@ -245,7 +258,13 @@ class SynapseHomeServer(HomeServer): metrics_resource = self.get_resource_for_metrics() if metrics_resource and config.metrics_port is not None: reactor.listenTCP( - config.metrics_port, Site(metrics_resource), interface="127.0.0.1", + config.metrics_port, + Site( + metrics_resource, + logPath=config.access_log_file, + logFormatter=log_formatter, + ), + interface="127.0.0.1", ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 07fbfadc0f..456ce9c632 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -22,6 +22,8 @@ class CaptchaConfig(Config): self.recaptcha_private_key = args.recaptcha_private_key self.recaptcha_public_key = args.recaptcha_public_key self.enable_registration_captcha = args.enable_registration_captcha + + # XXX: This is used for more than just captcha self.captcha_ip_origin_is_x_forwarded = ( args.captcha_ip_origin_is_x_forwarded ) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 247b324816..559cbe7963 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -27,6 +27,7 @@ class LoggingConfig(Config): self.verbosity = int(args.verbose) if args.verbose else None self.log_config = self.abspath(args.log_config) self.log_file = self.abspath(args.log_file) + self.access_log_file = self.abspath(args.access_log_file) @classmethod def add_arguments(cls, parser): @@ -44,6 +45,10 @@ class LoggingConfig(Config): '--log-config', dest="log_config", default=None, help="Python logging config file" ) + logging_group.add_argument( + '--access-log-file', dest="access_log_file", default="access.log", + help="File to log server access to" + ) def setup_logging(self): log_format = ( -- cgit 1.4.1 From c9e62927f29991ca32e21215e6647b37962c84dc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 14:34:09 +0100 Subject: Use disable_registration keys if they are present --- synapse/config/registration.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 82684e4dc9..095c3d3b00 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -26,6 +26,11 @@ class RegistrationConfig(Config): self.disable_registration = not bool( distutils.util.strtobool(str(config["enable_registration"])) ) + if "disable_registration" in config: + self.disable_registration = bool( + distutils.util.strtobool(str(config["disable_registration"])) + ) + self.registration_shared_secret = config.get("registration_shared_secret") def default_config(self, config_dir, server_name): -- cgit 1.4.1 From 265f30bd3f080f11b5cf6644d293943932c592e4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 15:04:06 +0100 Subject: Allow --enable-registration to be passed on the commandline --- synapse/config/registration.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 095c3d3b00..971965a5da 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -17,18 +17,18 @@ from ._base import Config from synapse.util.stringutils import random_string_with_symbols -import distutils.util +from distutils.util import strtobool class RegistrationConfig(Config): def read_config(self, config): self.disable_registration = not bool( - distutils.util.strtobool(str(config["enable_registration"])) + strtobool(str(config["enable_registration"])) ) if "disable_registration" in config: self.disable_registration = bool( - distutils.util.strtobool(str(config["disable_registration"])) + strtobool(str(config["disable_registration"])) ) self.registration_shared_secret = config.get("registration_shared_secret") @@ -45,3 +45,16 @@ class RegistrationConfig(Config): # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" """ % locals() + + def add_arguments(self, parser): + reg_group = parser.add_argument_group("registration") + reg_group.add_argument( + "--enable-registration", action="store_true", + help="Enable registration for new users." + ) + + def read_arguments(self, args): + if args.enable_registration is not None: + self.disable_registration = not bool( + strtobool(str(args.enable_registration)) + ) -- cgit 1.4.1 From c28f1d16f072b22ae34be34d3cbcc8a06e394f6f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 15:13:14 +0100 Subject: Add a random string to the auto generated key id --- synapse/config/key.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/key.py b/synapse/config/key.py index a63f7d841b..4a18a94775 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -20,6 +20,7 @@ from syutil.crypto.signing_key import ( is_signing_algorithm_supported, decode_verify_key_bytes ) from syutil.base64util import decode_base64 +from synapse.util.stringutils import random_string class KeyConfig(Config): @@ -110,9 +111,10 @@ class KeyConfig(Config): signing_key_path = config["signing_key_path"] if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: + key_id = "a_" + random_string(4) syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.signing_key.generate_signing_key("auto"),), + (syutil.crypto.signing_key.generate_signing_key(key_id),), ) else: signing_keys = self.read_file(signing_key_path, "signing_key") -- cgit 1.4.1 From 74aaacf82aa6b592b100f8b930938e67bfd99000 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 16:04:02 +0100 Subject: Don't break when sizes or durations are given as integers --- demo/start.sh | 6 +++--- synapse/config/_base.py | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) (limited to 'synapse/config') diff --git a/demo/start.sh b/demo/start.sh index ef4be2d5ff..75ff9861d8 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -26,9 +26,9 @@ for port in 8080 8081 8082; do https_port=$((port + 400)) mkdir -p demo/$port - pushd demo/$port +# pushd demo/$port - rm $DIR/etc/$port.config + #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ --generate-config \ -H "localhost:$https_port" \ @@ -39,7 +39,7 @@ for port in 8080 8081 8082; do -D \ -vv \ - popd + # popd done cd "$CWD" diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d98b6aaedf..e0c203cb1f 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,30 +27,33 @@ class ConfigError(Exception): class Config(object): @staticmethod - def parse_size(string): + def parse_size(value): + if isinstance(value, int) or isinstance(value, long): + return value sizes = {"K": 1024, "M": 1024 * 1024} size = 1 - suffix = string[-1] + suffix = value[-1] if suffix in sizes: - string = string[:-1] + value = value[:-1] size = sizes[suffix] - return int(string) * size + return int(value) * size @staticmethod - def parse_duration(string): + def parse_duration(value): + if isinstance(value, int) or isinstance(value, long): + return value second = 1000 hour = 60 * 60 * second day = 24 * hour week = 7 * day year = 365 * day - sizes = {"s": second, "h": hour, "d": day, "w": week, "y": year} size = 1 - suffix = string[-1] + suffix = value[-1] if suffix in sizes: - string = string[:-1] + value = value[:-1] size = sizes[suffix] - return int(string) * size + return int(value) * size @staticmethod def abspath(file_path): -- cgit 1.4.1 From 054aa0d58c22ae76d3e094fc2fd6495456ffd2cf Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 16:17:27 +0100 Subject: Do access log using python's logging stuff, just under a separate logger name --- synapse/app/homeserver.py | 42 ++++++++++++++++++++++++++++-------------- synapse/config/logger.py | 11 +++++++++++ 2 files changed, 39 insertions(+), 14 deletions(-) (limited to 'synapse/config') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0aa5c34c81..3ce5fa4a43 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,7 +35,7 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site -from twisted.web.http import proxiedLogFormatter +from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource @@ -226,17 +226,13 @@ class SynapseHomeServer(HomeServer): def start_listening(self): config = self.get_config() - log_formatter = None - if config.captcha_ip_origin_is_x_forwarded: - log_formatter = proxiedLogFormatter - if not config.no_tls and config.bind_port is not None: reactor.listenSSL( config.bind_port, - Site( + SynapseSite( + "synapse.access.https", + config, self.root_resource, - logPath=config.access_log_file, - logFormatter=log_formatter, ), self.tls_context_factory, interface=config.bind_host @@ -246,10 +242,10 @@ class SynapseHomeServer(HomeServer): if config.unsecure_port is not None: reactor.listenTCP( config.unsecure_port, - Site( + SynapseSite( + "synapse.access.http", + config, self.root_resource, - logPath=config.access_log_file, - logFormatter=log_formatter, ), interface=config.bind_host ) @@ -259,10 +255,10 @@ class SynapseHomeServer(HomeServer): if metrics_resource and config.metrics_port is not None: reactor.listenTCP( config.metrics_port, - Site( + SynapseSite( + "synapse.access.metrics", + config, metrics_resource, - logPath=config.access_log_file, - logFormatter=log_formatter, ), interface="127.0.0.1", ) @@ -484,6 +480,24 @@ class SynapseService(service.Service): return self._port.stopListening() +class SynapseSite(Site): + """ + Subclass of a twisted http Site that does access logging with python's + standard logging + """ + def __init__(self, logger_name, config, resource, *args, **kwargs): + Site.__init__(self, resource, *args, **kwargs) + if config.captcha_ip_origin_is_x_forwarded: + self._log_formatter = proxiedLogFormatter + else: + self._log_formatter = combinedLogFormatter + self.access_logger = logging.getLogger(logger_name) + + def log(self, request): + line = self._log_formatter(self._logDateTime, request) + self.access_logger.info(line) + + def run(hs): def in_thread(): diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 559cbe7963..077f20497a 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -83,6 +83,17 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) + + if self.access_log_file: + access_logger = logging.getLogger('synapse.access') + # we log to both files by default + access_logger.propagate = 1 + access_log_handler = logging.handlers.RotatingFileHandler( + self.access_log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 + ) + access_log_formatter = logging.Formatter('%(message)s') + access_log_handler.setFormatter(access_log_formatter) + access_logger.addHandler(access_log_handler) else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) -- cgit 1.4.1 From 5b02f334519964ffae6812df5413fcdae84db6ba Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 16:20:12 +0100 Subject: Undo changes to logger config, ie. remove the access_log_file option: decision is to support this through log_config rather tan adding an option. --- synapse/config/logger.py | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 077f20497a..247b324816 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -27,7 +27,6 @@ class LoggingConfig(Config): self.verbosity = int(args.verbose) if args.verbose else None self.log_config = self.abspath(args.log_config) self.log_file = self.abspath(args.log_file) - self.access_log_file = self.abspath(args.access_log_file) @classmethod def add_arguments(cls, parser): @@ -45,10 +44,6 @@ class LoggingConfig(Config): '--log-config', dest="log_config", default=None, help="Python logging config file" ) - logging_group.add_argument( - '--access-log-file', dest="access_log_file", default="access.log", - help="File to log server access to" - ) def setup_logging(self): log_format = ( @@ -83,17 +78,6 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - - if self.access_log_file: - access_logger = logging.getLogger('synapse.access') - # we log to both files by default - access_logger.propagate = 1 - access_log_handler = logging.handlers.RotatingFileHandler( - self.access_log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 - ) - access_log_formatter = logging.Formatter('%(message)s') - access_log_handler.setFormatter(access_log_formatter) - access_logger.addHandler(access_log_handler) else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) -- cgit 1.4.1 From 2f1348f3395ba44cece492205ce7fa87ac519bee Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 16:52:57 +0100 Subject: Write a default log_config when generating config --- demo/start.sh | 4 ++-- synapse/config/_base.py | 3 ++- synapse/config/key.py | 2 +- synapse/config/logger.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++- synapse/config/tls.py | 2 +- 5 files changed, 59 insertions(+), 6 deletions(-) (limited to 'synapse/config') diff --git a/demo/start.sh b/demo/start.sh index 75ff9861d8..5b3daef57f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -26,7 +26,7 @@ for port in 8080 8081 8082; do https_port=$((port + 400)) mkdir -p demo/$port -# pushd demo/$port + pushd demo/$port #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ @@ -39,7 +39,7 @@ for port in 8080 8081 8082; do -D \ -vv \ - # popd + popd done cd "$CWD" diff --git a/synapse/config/_base.py b/synapse/config/_base.py index e0c203cb1f..d7ccfcd98c 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -171,7 +171,7 @@ class Config(object): config_bytes, config = obj.generate_config( config_dir_path, server_name ) - obj.invoke_all("generate_keys", config) + obj.invoke_all("generate_files", config) config_file.write(config_bytes) print ( "A config file has been generated in %s for server name" @@ -192,6 +192,7 @@ class Config(object): server_name = specified_config["server_name"] _, config = obj.generate_config(config_dir_path, server_name) + config.pop("log_config") config.update(specified_config) obj.invoke_all("read_config", config) diff --git a/synapse/config/key.py b/synapse/config/key.py index 4a18a94775..27e0d2906e 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -107,7 +107,7 @@ class KeyConfig(Config): ) return keys - def generate_keys(self, config): + def generate_files(self, config): signing_key_path = config["signing_key_path"] if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 37b3d5342c..fa542623b7 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -19,6 +19,47 @@ from twisted.python.log import PythonLoggingObserver import logging import logging.config import yaml +from string import Template +import os + + +DEFAULT_LOG_CONFIG = Template(""" +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s\ +- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: ${log_file} + maxBytes: 104857600 + backupCount: 10 + filters: [context] + level: INFO + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse: + level: INFO + + synapse.storage.SQL: + level: INFO + +root: + level: INFO + handlers: [file, console] +""") class LoggingConfig(Config): @@ -30,6 +71,9 @@ class LoggingConfig(Config): def default_config(self, config_dir_path, server_name): log_file = self.abspath("homeserver.log") + log_config = self.abspath( + os.path.join(config_dir_path, server_name + ".log.config") + ) return """ # Logging verbosity level. verbose: 0 @@ -38,7 +82,7 @@ class LoggingConfig(Config): log_file: "%(log_file)s" # A yaml python logging config file - #log_config: "your.log.config.yaml" + log_config: "%(log_config)s" """ % locals() def read_arguments(self, args): @@ -64,6 +108,14 @@ class LoggingConfig(Config): help="Python logging config file" ) + def generate_files(self, config): + log_config = config.get("log_config") + if log_config and not os.path.exists(log_config): + with open(log_config, "wb") as log_config_file: + log_config_file.write( + DEFAULT_LOG_CONFIG.substitute(log_file=config["log_file"]) + ) + def setup_logging(self): log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e70bc1cd2c..ecb2d42c1f 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -70,7 +70,7 @@ class TlsConfig(Config): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - def generate_keys(self, config): + def generate_files(self, config): tls_certificate_path = config["tls_certificate_path"] tls_private_key_path = config["tls_private_key_path"] tls_dh_params_path = config["tls_dh_params_path"] -- cgit 1.4.1 From 95cbfee8ae1a1bf27e0c8aa10668e936c7d775c0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:52:20 +0100 Subject: Update metrics.py --- synapse/config/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 06e2e7ccff..71a1b1d189 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -19,7 +19,7 @@ from ._base import Config class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] - self.metrics_port = config["metrics_port"] + self.metrics_port = config.get("metrics_port") def default_config(self, config_dir_path, server_name): return """\ @@ -29,5 +29,5 @@ class MetricsConfig(Config): enable_metrics: False # Separate port to accept metrics requests on (on localhost) - metrics_port: ~ + # metrics_port: 8081 """ -- cgit 1.4.1 From 62cebee8ee320f530f8aa9cd72419a4cca2f94ea Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:54:01 +0100 Subject: Update key.py --- synapse/config/key.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/key.py b/synapse/config/key.py index 27e0d2906e..0494c0cb77 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -120,9 +120,10 @@ class KeyConfig(Config): signing_keys = self.read_file(signing_key_path, "signing_key") if len(signing_keys.split("\n")[0].split()) == 1: # handle keys in the old format. + key_id = "a_" + random_string(4) key = syutil.crypto.signing_key.decode_signing_key_base64( syutil.crypto.signing_key.NACL_ED25519, - "auto", + key_id, signing_keys.split("\n")[0] ) with open(signing_key_path, "w") as signing_key_file: -- cgit 1.4.1 From 345995fcde82cd69ed04277d55a14d32b6dbe589 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 18:10:19 +0100 Subject: Remove the ~, comment the lines instead --- synapse/config/captcha.py | 2 +- synapse/config/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index ee5c238314..d8fe577e34 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -47,5 +47,5 @@ class CaptchaConfig(Config): captcha_ip_origin_is_x_forwarded: False # A secret key used to bypass the captcha test entirely. - captcha_bypass_secret: ~ + #captcha_bypass_secret: "YOUR_SECRET_HERE" """ diff --git a/synapse/config/server.py b/synapse/config/server.py index fe1b63469a..cd9ca56158 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -83,7 +83,7 @@ class ServerConfig(Config): # Turn on the twisted telnet manhole service on localhost on the given # port. - manhole: ~ + #manhole: 9000 """ % locals() def read_arguments(self, args): -- cgit 1.4.1 From 50c87b8eed628c0cae2e4bf2a60dff47ff21bbd4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 18:11:47 +0100 Subject: Allow "manhole" to be ommited from the config --- synapse/config/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index cd9ca56158..f20d3fd389 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -23,7 +23,7 @@ class ServerConfig(Config): self.bind_port = config["bind_port"] self.bind_host = config["bind_host"] self.unsecure_port = config["unsecure_port"] - self.manhole = config["manhole"] + self.manhole = config.get("manhole") self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] -- cgit 1.4.1 From 46a65c282fc92d2c783af84423ef4a0690a2cf48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 1 May 2015 13:54:38 +0100 Subject: Allow generate-config to run against an existing config file to generate default keys --- synapse/config/_base.py | 35 +++++++++++++++++++++++------------ synapse/config/database.py | 3 ++- synapse/config/registration.py | 2 +- 3 files changed, 26 insertions(+), 14 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d7ccfcd98c..cd4bd28e8c 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -159,26 +159,37 @@ class Config(object): print "Most specify a server_name to a generate config for." sys.exit(1) (config_path,) = config_args.config_path - if os.path.exists(config_path): - print "Config file %r already exists. Not overwriting" % ( - config_args.config_path - ) - sys.exit(1) if not os.path.exists(config_dir_path): os.makedirs(config_dir_path) + if os.path.exists(config_path): + print "Config file %r already exists" % (config_path,) + yaml_config = cls.read_config_file(config_path) + yaml_name = yaml_config["server_name"] + if server_name != yaml_name: + print ( + "Config file %r has a different server_name: " + " %r != %r" % (config_path, server_name, yaml_name) + ) + sys.exit(1) + config_bytes, config = obj.generate_config( + config_dir_path, server_name + ) + config.update(yaml_config) + print "Generating any missing keys for %r" % (server_name,) + obj.invoke_all("generate_files", config) + sys.exit(0) with open(config_path, "wb") as config_file: - config_bytes, config = obj.generate_config( config_dir_path, server_name ) obj.invoke_all("generate_files", config) config_file.write(config_bytes) - print ( - "A config file has been generated in %s for server name" - " '%s' with corresponding SSL keys and self-signed" - " certificates. Please review this file and customise it to" - " your needs." - ) % (config_path, server_name) + print ( + "A config file has been generated in %s for server name" + " '%s' with corresponding SSL keys and self-signed" + " certificates. Please review this file and customise it to" + " your needs." + ) % (config_path, server_name) print ( "If this server name is incorrect, you will need to regenerate" " the SSL certificates" diff --git a/synapse/config/database.py b/synapse/config/database.py index ccd96c4f0f..f0611e8884 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -56,6 +56,7 @@ class DatabaseConfig(Config): args: # Path to the database database: "%(database_path)s" + # Number of events to cache in memory. event_cache_size: "10K" """ % locals() @@ -68,7 +69,7 @@ class DatabaseConfig(Config): database_path = self.abspath(database_path) if self.database_config.get("name", None) == "sqlite3": if database_path is not None: - self.database_config["database"] = database_path + self.database_config["args"]["database"] = database_path def add_arguments(self, parser): db_group = parser.add_argument_group("database") diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 971965a5da..b39989a87f 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -49,7 +49,7 @@ class RegistrationConfig(Config): def add_arguments(self, parser): reg_group = parser.add_argument_group("registration") reg_group.add_argument( - "--enable-registration", action="store_true", + "--enable-registration", action="store_true", default=None, help="Enable registration for new users." ) -- cgit 1.4.1 From 3bcdf3664c34110d5cdb1f1936cd97c3b3fe4b42 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 1 May 2015 14:34:48 +0100 Subject: Use the daemonize key from the config if it exists --- synapse/config/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index f20d3fd389..78195b3a4f 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -27,6 +27,7 @@ class ServerConfig(Config): self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] + self.daemonize = config.get("daemonize") # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") @@ -89,11 +90,13 @@ class ServerConfig(Config): def read_arguments(self, args): if args.manhole is not None: self.manhole = args.manhole - self.daemonize = args.daemonize + if args.daemonize is not None: + self.daemonize = args.daemonize def add_arguments(self, parser): server_group = parser.add_argument_group("server") server_group.add_argument("-D", "--daemonize", action='store_true', + default=None, help="Daemonize the home server") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, -- cgit 1.4.1 From e45b05647e9242ba543562e3ad2bb4141e85ab8c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 17:38:10 +0100 Subject: Fix the --help option for synapse --- synapse/config/_base.py | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index cd4bd28e8c..2807abbc90 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -144,16 +144,17 @@ class Config(object): ) config_args, remaining_args = config_parser.parse_known_args(argv) - if not config_args.config_path: - config_parser.error( - "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config -h SERVER_NAME" - " -c CONFIG-FILE\"" - ) - - config_dir_path = os.path.dirname(config_args.config_path[0]) - config_dir_path = os.path.abspath(config_dir_path) if config_args.generate_config: + if not config_args.config_path: + config_parser.error( + "Must supply a config file.\nA config file can be automatically" + " generated using \"--generate-config -h SERVER_NAME" + " -c CONFIG-FILE\"" + ) + + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) + server_name = config_args.server_name if not server_name: print "Most specify a server_name to a generate config for." @@ -196,6 +197,25 @@ class Config(object): ) sys.exit(0) + parser = argparse.ArgumentParser( + parents=[config_parser], + description=description, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + obj.invoke_all("add_arguments", parser) + args = parser.parse_args(remaining_args) + + if not config_args.config_path: + config_parser.error( + "Must supply a config file.\nA config file can be automatically" + " generated using \"--generate-config -h SERVER_NAME" + " -c CONFIG-FILE\"" + ) + + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) + specified_config = {} for config_path in config_args.config_path: yaml_config = cls.read_config_file(config_path) @@ -208,15 +228,6 @@ class Config(object): obj.invoke_all("read_config", config) - parser = argparse.ArgumentParser( - parents=[config_parser], - description=description, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - obj.invoke_all("add_arguments", parser) - args = parser.parse_args(remaining_args) - obj.invoke_all("read_arguments", args) return obj -- cgit 1.4.1