From 8ce100c7b48a38e5f74ed6ac560de1f50c288379 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:29:36 +0000 Subject: Convert directory paths to absolute paths before daemonizing --- synapse/config/_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index dfc115d8e8..9b0f8c3c32 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -50,8 +50,9 @@ class Config(object): ) return cls.abspath(file_path) - @staticmethod - def ensure_directory(dir_path): + @classmethod + def ensure_directory(cls, dir_path): + dir_path = cls.abspath(dir_path) if not os.path.exists(dir_path): os.makedirs(dir_path) if not os.path.isdir(dir_path): -- cgit 1.5.1 From 30595b466f1f6b61eef15dc41562f6336e57e6fe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:50:33 +0000 Subject: Use yaml logging config format because it is much nicer --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index f9568ebd21..b18d08ec61 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -18,6 +18,7 @@ from synapse.util.logcontext import LoggingContextFilter from twisted.python.log import PythonLoggingObserver import logging import logging.config +import yaml class LoggingConfig(Config): @@ -79,7 +80,7 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.fileConfig(self.log_config) + logging.config.dictConfig(yaml.load(self.log_config)) observer = PythonLoggingObserver() observer.start() -- cgit 1.5.1 From f91345bdb5b76ce76b99d4fc969ee4f41cbbba4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:57:31 +0000 Subject: yaml.load expects strings to be a yaml rather than file --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/logger.py b/synapse/config/logger.py index b18d08ec61..63c8e36930 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -80,7 +80,8 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.dictConfig(yaml.load(self.log_config)) + with open(self.log_config, 'r') as f: + logging.config.dictConfig(yaml.load(f)) observer = PythonLoggingObserver() observer.start() -- cgit 1.5.1 From f5a70e0d2e890adea53b3f6565a3bbe92512a506 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 11 Feb 2015 15:01:15 +0000 Subject: Add a cache for get_event --- synapse/config/_base.py | 10 ++++++++++ synapse/config/database.py | 5 +++++ synapse/storage/__init__.py | 3 +++ synapse/storage/_base.py | 24 +++++++++++++++++++++--- tests/storage/test_base.py | 5 +++-- tests/utils.py | 1 + 6 files changed, 43 insertions(+), 5 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 9b0f8c3c32..87cdbf1d30 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,6 +27,16 @@ class Config(object): def __init__(self, args): pass + @staticmethod + def parse_size(string): + sizes = {"K": 1024, "M": 1024 * 1024} + size = 1 + suffix = string[-1] + if suffix in sizes: + string = string[:-1] + size = sizes[suffix] + return int(string) * size + @staticmethod def abspath(file_path): return os.path.abspath(file_path) if file_path else file_path diff --git a/synapse/config/database.py b/synapse/config/database.py index daa161c952..87efe54645 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -24,6 +24,7 @@ class DatabaseConfig(Config): self.database_path = ":memory:" else: self.database_path = self.abspath(args.database_path) + self.event_cache_size = self.parse_size(args.event_cache_size) @classmethod def add_arguments(cls, parser): @@ -33,6 +34,10 @@ class DatabaseConfig(Config): "-d", "--database-path", default="homeserver.db", help="The database name." ) + db_group.add_argument( + "--event-cache-size", default="100K", + help="Number of events to cache in memory." + ) @classmethod def generate_config(cls, args, config_dir_path): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a63c59a8a2..1170d8b6ec 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -164,6 +164,9 @@ class DataStore(RoomMemberStore, RoomStore, stream_ordering=None, is_new_state=True, current_state=None): + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 3e1ab0a159..f13b8f4fad 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,6 +19,7 @@ from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext, LoggingContext +from synapse.util.lrucache import LruCache from twisted.internet import defer @@ -128,6 +129,8 @@ class SQLBaseStore(object): self._txn_perf_counters = PerformanceCounters() self._get_event_counters = PerformanceCounters() + self._get_event_cache = LruCache(hs.config.event_cache_size) + def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -579,6 +582,20 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): + + start_time = time.time() * 1000 + update_counter = self._get_event_counters.update + + try: + cache = self._get_event_cache.setdefault(event_id, {}) + # Separate cache entries for each way to invoke _get_event_txn + return cache[(check_redacted, get_prev_content, allow_rejected)] + except KeyError: + pass + finally: + start_time = update_counter("event_cache", start_time) + + sql = ( "SELECT e.internal_metadata, e.json, r.event_id, rej.reason " "FROM event_json as e " @@ -588,7 +605,6 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -599,14 +615,16 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res - self._get_event_counters.update("select_event", start_time) + start_time = update_counter("select_event", start_time) if allow_rejected or not rejected_reason: - return self._get_event_from_row_txn( + result = self._get_event_from_row_txn( txn, internal_metadata, js, redacted, check_redacted=check_redacted, get_prev_content=get_prev_content, ) + cache[(check_redacted, get_prev_content, allow_rejected)] = result + return result else: return None diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index a0a24ce096..55fbffa7a2 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -38,8 +38,9 @@ class SQLBaseStoreTestCase(unittest.TestCase): return defer.succeed(func(self.mock_txn, *args, **kwargs)) self.db_pool.runInteraction = runInteraction - hs = HomeServer("test", - db_pool=self.db_pool) + config = Mock() + config.event_cache_size = 1 + hs = HomeServer("test", db_pool=self.db_pool, config=config) self.datastore = SQLBaseStore(hs) diff --git a/tests/utils.py b/tests/utils.py index 25c33492a5..39895c739f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -41,6 +41,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): if config is None: config = Mock() config.signing_key = [MockKey()] + config.event_cache_size = 1 if datastore is None: db_pool = SQLiteMemoryDbPool() -- cgit 1.5.1 From c3eb7dd9c572cfcea831e4d5f41b5a34cb3c57e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 11:50:49 +0000 Subject: Add config option to set the soft fd limit on start --- synapse/app/homeserver.py | 20 +++++++++++++++++--- synapse/config/server.py | 7 +++++++ 2 files changed, 24 insertions(+), 3 deletions(-) (limited to 'synapse/config') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ea20de1434..07386f6f28 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -52,6 +52,7 @@ import synapse import logging import os import re +import resource import subprocess import sqlite3 import syweb @@ -269,6 +270,15 @@ def get_version_string(): return ("Synapse/%s" % (synapse.__version__,)).encode("ascii") +def change_resource_limit(soft_file_no): + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) + logger.info("Set file limit to: %d", soft_file_no) + except (ValueError, resource.error) as e: + logger.warn("Failed to set file limit: %s", e) + + def setup(): config = HomeServerConfig.load_config( "Synapse Homeserver", @@ -345,10 +355,11 @@ def setup(): if config.daemonize: print config.pid_file + daemon = Daemonize( app="synapse-homeserver", pid=config.pid_file, - action=run, + action=lambda: run(config), auto_close_fds=False, verbose=True, logger=logger, @@ -356,11 +367,14 @@ def setup(): daemon.start() else: - reactor.run() + run(config) -def run(): +def run(config): with LoggingContext("run"): + if config.soft_file_limit: + change_resource_limit(config.soft_file_limit) + reactor.run() diff --git a/synapse/config/server.py b/synapse/config/server.py index 31e44cc857..a3edf208e9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -31,6 +31,7 @@ class ServerConfig(Config): self.webclient = True self.manhole = args.manhole self.no_tls = args.no_tls + self.soft_file_limit = args.soft_file_limit if not args.content_addr: host = args.server_name @@ -77,6 +78,12 @@ class ServerConfig(Config): "content repository") server_group.add_argument("--no-tls", action='store_true', help="Don't bind to the https port.") + server_group.add_argument("--soft-file-limit", type=int, default=0, + help="Set the limit on the number of file " + "descriptors synapse can use. Zero " + "is used to indicate synapse should " + "not change the limit from system " + "default.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") -- cgit 1.5.1 From 81163f822e5133ae6e724f18e70e9b84b2081a4e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Feb 2015 14:16:53 +0000 Subject: Add config option to disable registration. --- synapse/config/homeserver.py | 3 ++- synapse/config/registration.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 synapse/config/registration.py (limited to 'synapse/config') diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index b0fe217459..c024535f52 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -22,11 +22,12 @@ from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig from .email import EmailConfig from .voip import VoipConfig +from .registration import RegistrationConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - EmailConfig, VoipConfig): + EmailConfig, VoipConfig, RegistrationConfig,): pass diff --git a/synapse/config/registration.py b/synapse/config/registration.py new file mode 100644 index 0000000000..cca8ab5676 --- /dev/null +++ b/synapse/config/registration.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class RegistrationConfig(Config): + + def __init__(self, args): + super(RegistrationConfig, self).__init__(args) + self.disable_registration = args.disable_registration + + @classmethod + def add_arguments(cls, parser): + super(RegistrationConfig, cls).add_arguments(parser) + reg_group = parser.add_argument_group("registration") + reg_group.add_argument( + "--disable-registration", + action='store_true', + help="Disable registration of new users." + ) -- cgit 1.5.1 From 7c56210f204ef735444129c4f7b2a393eb7539ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Feb 2015 16:09:44 +0000 Subject: By default set soft limit to hard limit --- synapse/app/homeserver.py | 8 ++++++-- synapse/config/server.py | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'synapse/config') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5f671dfd23..89af091f10 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -273,7 +273,12 @@ def get_version_string(): def change_resource_limit(soft_file_no): try: soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + + if not soft_file_no: + soft_file_no = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) + logger.info("Set file limit to: %d", soft_file_no) except (ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) @@ -372,8 +377,7 @@ def setup(): def run(config): with LoggingContext("run"): - if config.soft_file_limit: - change_resource_limit(config.soft_file_limit) + change_resource_limit(config.soft_file_limit) reactor.run() diff --git a/synapse/config/server.py b/synapse/config/server.py index a3edf208e9..4e4892d40b 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -79,11 +79,11 @@ class ServerConfig(Config): server_group.add_argument("--no-tls", action='store_true', help="Don't bind to the https port.") server_group.add_argument("--soft-file-limit", type=int, default=0, - help="Set the limit on the number of file " - "descriptors synapse can use. Zero " - "is used to indicate synapse should " - "not change the limit from system " - "default.") + help="Set the soft limit on the number of " + "file descriptors synapse can use. " + "Zero is used to indicate synapse " + "should set the soft limit to the hard" + "limit.") def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") -- cgit 1.5.1 From 255f989c7bc20d17ec37fed00cb2d107553d6f73 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Feb 2015 20:57:58 +0000 Subject: turns uris config options should append since it's a list --- synapse/config/voip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/voip.py b/synapse/config/voip.py index a2b822719f..65162d21b7 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -28,7 +28,7 @@ class VoipConfig(Config): super(VoipConfig, cls).add_arguments(parser) group = parser.add_argument_group("voip") group.add_argument( - "--turn-uris", type=str, default=None, + "--turn-uris", type=str, default=None, action='append', help="The public URIs of the TURN server to give to clients" ) group.add_argument( -- cgit 1.5.1 From 9d9b230501915c326136567349b0995623c48a21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Mar 2015 11:33:45 +0000 Subject: Make the federation server ratelimiting configurable. --- synapse/config/ratelimiting.py | 36 ++++++++++++++++++++++++++++++++ synapse/federation/transport/__init__.py | 10 ++++----- 2 files changed, 41 insertions(+), 5 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 17c7e64ce7..862c07ef8c 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -22,6 +22,12 @@ class RatelimitConfig(Config): self.rc_messages_per_second = args.rc_messages_per_second self.rc_message_burst_count = args.rc_message_burst_count + self.federation_rc_window_size = args.federation_rc_window_size + self.federation_rc_sleep_limit = args.federation_rc_sleep_limit + self.federation_rc_sleep_delay = args.federation_rc_sleep_delay + self.federation_rc_reject_limit = args.federation_rc_reject_limit + self.federation_rc_concurrent = args.federation_rc_concurrent + @classmethod def add_arguments(cls, parser): super(RatelimitConfig, cls).add_arguments(parser) @@ -34,3 +40,33 @@ class RatelimitConfig(Config): "--rc-message-burst-count", type=float, default=10, help="number of message a client can send before being throttled" ) + + rc_group.add_argument( + "--federation-rc-window-size", type=int, default=10000, + help="The federation window size in milliseconds", + ) + + rc_group.add_argument( + "--federation-rc-sleep-limit", type=int, default=10, + help="The number of federation requests from a single server" + " in a window before the server will delay processing the" + " request.", + ) + + rc_group.add_argument( + "--federation-rc-sleep-delay", type=int, default=500, + help="The duration in milliseconds to delay processing events from" + " remote servers by if they go over the sleep limit.", + ) + + rc_group.add_argument( + "--federation-rc-reject-limit", type=int, default=50, + help="The maximum number of concurrent federation requests allowed" + " from a single server", + ) + + rc_group.add_argument( + "--federation-rc-concurrent", type=int, default=3, + help="The number of federation requests to concurrently process" + " from a single server", + ) diff --git a/synapse/federation/transport/__init__.py b/synapse/federation/transport/__init__.py index f0283b5105..2a671b9aec 100644 --- a/synapse/federation/transport/__init__.py +++ b/synapse/federation/transport/__init__.py @@ -66,9 +66,9 @@ class TransportLayer(TransportLayerServer, TransportLayerClient): self.ratelimiter = FederationRateLimiter( self.clock, - window_size=10000, - sleep_limit=10, - sleep_msec=500, - reject_limit=50, - concurrent_requests=3, + window_size=homeserver.config.federation_rc_window_size, + sleep_limit=homeserver.config.federation_rc_sleep_limit, + sleep_msec=homeserver.config.federation_rc_sleep_delay, + reject_limit=homeserver.config.federation_rc_reject_limit, + concurrent_requests=homeserver.config.federation_rc_concurrent, ) -- cgit 1.5.1 From 3ce8540484a3cc29ce2970ebf6608b6fd3359931 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 11:34:06 +0000 Subject: Don't look for an TLS private key if we have set --no-tls --- synapse/config/server.py | 3 --- synapse/config/tls.py | 17 +++++++++++++---- synapse/crypto/context_factory.py | 5 ++++- 3 files changed, 17 insertions(+), 8 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index 4e4892d40b..b042d4eed9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -30,7 +30,6 @@ class ServerConfig(Config): self.pid_file = self.abspath(args.pid_file) self.webclient = True self.manhole = args.manhole - self.no_tls = args.no_tls self.soft_file_limit = args.soft_file_limit if not args.content_addr: @@ -76,8 +75,6 @@ class ServerConfig(Config): server_group.add_argument("--content-addr", default=None, help="The host and scheme to use for the " "content repository") - server_group.add_argument("--no-tls", action='store_true', - help="Don't bind to the https port.") server_group.add_argument("--soft-file-limit", type=int, default=0, help="Set the soft limit on the number of " "file descriptors synapse can use. " diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 384b29e7ba..a45bf6d521 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import Config +from ._base import Config, ConfigError from OpenSSL import crypto import subprocess @@ -28,9 +28,16 @@ class TlsConfig(Config): self.tls_certificate = self.read_tls_certificate( args.tls_certificate_path ) - self.tls_private_key = self.read_tls_private_key( - args.tls_private_key_path - ) + + self.no_tls = args.no_tls + + if self.no_tls: + self.tls_private_key = None + else: + self.tls_private_key = self.read_tls_private_key( + args.tls_private_key_path + ) + self.tls_dh_params_path = self.check_file( args.tls_dh_params_path, "tls_dh_params" ) @@ -45,6 +52,8 @@ class TlsConfig(Config): help="PEM encoded private key for TLS") tls_group.add_argument("--tls-dh-params-path", help="PEM dh parameters for ephemeral keys") + tls_group.add_argument("--no-tls", action='store_true', + help="Don't bind to the https port.") def read_tls_certificate(self, cert_path): cert_pem = self.read_file(cert_path, "tls_certificate") diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 24d4abf3e9..2f8618a0df 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -38,7 +38,10 @@ class ServerContextFactory(ssl.ContextFactory): logger.exception("Failed to enable eliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate(config.tls_certificate) - context.use_privatekey(config.tls_private_key) + + if not config.no_tls: + context.use_privatekey(config.tls_private_key) + context.load_tmp_dh(config.tls_dh_params_path) context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") -- cgit 1.5.1 From e49d6b1568eab259dc5eea434da2d0e65876e492 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Mar 2015 11:37:24 +0000 Subject: Unused import --- synapse/config/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/tls.py b/synapse/config/tls.py index a45bf6d521..034f9a7bf0 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import Config, ConfigError +from ._base import Config from OpenSSL import crypto import subprocess -- cgit 1.5.1