From d624e2a6383bbb179132b79eec80fa516e747bd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 04:24:44 +0100 Subject: Manually generate the default config yaml, remove most of the commandline arguments for synapse anticipating that people will use the yaml instead. Simpify implementing config options by not requiring the classes to hit the super class --- demo/start.sh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'demo') diff --git a/demo/start.sh b/demo/start.sh index 0485be8053..941eccd668 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,30 +16,29 @@ if [ $# -eq 1 ]; then fi fi +export PYTHONPATH=$(readlink -f $(pwd)) + + +echo $PYTHONPATH + for port in 8080 8081 8082; do echo "Starting server on port $port... " https_port=$((port + 400)) + mkdir -p demo/$port + pushd demo/$port + rm $DIR/etc/$port.config python -m synapse.app.homeserver \ - --generate-config \ - --config-path "demo/etc/$port.config" \ - -p "$https_port" \ - --unsecure-port "$port" \ - -H "localhost:$https_port" \ - -f "$DIR/$port.log" \ - -d "$DIR/$port.db" \ - -D --pid-file "$DIR/$port.pid" \ - --manhole $((port + 1000)) \ - --tls-dh-params-path "demo/demo.tls.dh" \ - --media-store-path "demo/media_store.$port" \ - $PARAMS $SYNAPSE_PARAMS \ - --enable-registration + --generate-config "localhost:$https_port" \ + --config-path "$DIR/etc/$port.config" \ python -m synapse.app.homeserver \ - --config-path "demo/etc/$port.config" \ + --config-path "$DIR/etc/$port.config" \ + -D \ -vv \ + popd done cd "$CWD" -- cgit 1.5.1 From 1aa11cf7cef83f529bb6f48a76f2d2fe10a7cfe4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 13:48:15 +0100 Subject: Allow multiple config files, set up a default config before applying the config files --- demo/start.sh | 3 ++- synapse/config/_base.py | 58 ++++++++++++++++++++++++++++---------------- synapse/config/homeserver.py | 2 +- 3 files changed, 40 insertions(+), 23 deletions(-) (limited to 'demo') diff --git a/demo/start.sh b/demo/start.sh index 941eccd668..ef4be2d5ff 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -30,7 +30,8 @@ for port in 8080 8081 8082; do rm $DIR/etc/$port.config python -m synapse.app.homeserver \ - --generate-config "localhost:$https_port" \ + --generate-config \ + -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ python -m synapse.app.homeserver \ diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 9f5da70948..d98b6aaedf 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -117,51 +117,59 @@ class Config(object): config = yaml.load(default_config) - if not os.path.exists(config_dir_path): - os.makedirs(config_dir_path) - - self.invoke_all("generate_keys", config) - - return default_config + return default_config, config @classmethod def load_config(cls, description, argv, generate_section=None): - result = cls() + obj = cls() config_parser = argparse.ArgumentParser(add_help=False) config_parser.add_argument( "-c", "--config-path", + action="append", metavar="CONFIG_FILE", help="Specify config file" ) config_parser.add_argument( "--generate-config", - metavar="SERVER_NAME", + action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "-H", "--server-name", + help="The server name to generate a config file for" + ) config_args, remaining_args = config_parser.parse_known_args(argv) if not config_args.config_path: config_parser.error( "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config SERVER_NAME" + " generated using \"--generate-config -h SERVER_NAME" " -c CONFIG-FILE\"" ) + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) if config_args.generate_config: - server_name = config_args.generate_config - config_path = config_args.config_path + server_name = config_args.server_name + if not server_name: + print "Most specify a server_name to a generate config for." + sys.exit(1) + (config_path,) = config_args.config_path if os.path.exists(config_path): print "Config file %r already exists. Not overwriting" % ( config_args.config_path ) - sys.exit(0) - config_dir_path = os.path.dirname(config_args.config_path) - config_dir_path = os.path.abspath(config_dir_path) + sys.exit(1) + if not os.path.exists(config_dir_path): + os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: - config_file.write( - result.generate_config(config_dir_path, server_name) + + config_bytes, config = obj.generate_config( + config_dir_path, server_name ) + obj.invoke_all("generate_keys", config) + config_file.write(config_bytes) print ( "A config file has been generated in %s for server name" " '%s' with corresponding SSL keys and self-signed" @@ -174,8 +182,16 @@ class Config(object): ) sys.exit(0) - config = cls.read_config_file(config_args.config_path) - result.invoke_all("read_config", config) + specified_config = {} + for config_path in config_args.config_path: + yaml_config = cls.read_config_file(config_path) + specified_config.update(yaml_config) + + server_name = specified_config["server_name"] + _, config = obj.generate_config(config_dir_path, server_name) + config.update(specified_config) + + obj.invoke_all("read_config", config) parser = argparse.ArgumentParser( parents=[config_parser], @@ -183,9 +199,9 @@ class Config(object): formatter_class=argparse.RawDescriptionHelpFormatter, ) - result.invoke_all("add_arguments", parser) + obj.invoke_all("add_arguments", parser) args = parser.parse_args(remaining_args) - result.invoke_all("read_arguments", args) + obj.invoke_all("read_arguments", args) - return result + return obj diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index f9b4807a35..fe0ccb6eb7 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -37,5 +37,5 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, if __name__ == '__main__': import sys sys.stdout.write( - HomeServerConfig().generate_config(sys.argv[1], sys.argv[2]) + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0] ) -- cgit 1.5.1 From 74aaacf82aa6b592b100f8b930938e67bfd99000 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 16:04:02 +0100 Subject: Don't break when sizes or durations are given as integers --- demo/start.sh | 6 +++--- synapse/config/_base.py | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) (limited to 'demo') diff --git a/demo/start.sh b/demo/start.sh index ef4be2d5ff..75ff9861d8 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -26,9 +26,9 @@ for port in 8080 8081 8082; do https_port=$((port + 400)) mkdir -p demo/$port - pushd demo/$port +# pushd demo/$port - rm $DIR/etc/$port.config + #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ --generate-config \ -H "localhost:$https_port" \ @@ -39,7 +39,7 @@ for port in 8080 8081 8082; do -D \ -vv \ - popd + # popd done cd "$CWD" diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d98b6aaedf..e0c203cb1f 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,30 +27,33 @@ class ConfigError(Exception): class Config(object): @staticmethod - def parse_size(string): + def parse_size(value): + if isinstance(value, int) or isinstance(value, long): + return value sizes = {"K": 1024, "M": 1024 * 1024} size = 1 - suffix = string[-1] + suffix = value[-1] if suffix in sizes: - string = string[:-1] + value = value[:-1] size = sizes[suffix] - return int(string) * size + return int(value) * size @staticmethod - def parse_duration(string): + def parse_duration(value): + if isinstance(value, int) or isinstance(value, long): + return value second = 1000 hour = 60 * 60 * second day = 24 * hour week = 7 * day year = 365 * day - sizes = {"s": second, "h": hour, "d": day, "w": week, "y": year} size = 1 - suffix = string[-1] + suffix = value[-1] if suffix in sizes: - string = string[:-1] + value = value[:-1] size = sizes[suffix] - return int(string) * size + return int(value) * size @staticmethod def abspath(file_path): -- cgit 1.5.1 From 2f1348f3395ba44cece492205ce7fa87ac519bee Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 16:52:57 +0100 Subject: Write a default log_config when generating config --- demo/start.sh | 4 ++-- synapse/config/_base.py | 3 ++- synapse/config/key.py | 2 +- synapse/config/logger.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++- synapse/config/tls.py | 2 +- 5 files changed, 59 insertions(+), 6 deletions(-) (limited to 'demo') diff --git a/demo/start.sh b/demo/start.sh index 75ff9861d8..5b3daef57f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -26,7 +26,7 @@ for port in 8080 8081 8082; do https_port=$((port + 400)) mkdir -p demo/$port -# pushd demo/$port + pushd demo/$port #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ @@ -39,7 +39,7 @@ for port in 8080 8081 8082; do -D \ -vv \ - # popd + popd done cd "$CWD" diff --git a/synapse/config/_base.py b/synapse/config/_base.py index e0c203cb1f..d7ccfcd98c 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -171,7 +171,7 @@ class Config(object): config_bytes, config = obj.generate_config( config_dir_path, server_name ) - obj.invoke_all("generate_keys", config) + obj.invoke_all("generate_files", config) config_file.write(config_bytes) print ( "A config file has been generated in %s for server name" @@ -192,6 +192,7 @@ class Config(object): server_name = specified_config["server_name"] _, config = obj.generate_config(config_dir_path, server_name) + config.pop("log_config") config.update(specified_config) obj.invoke_all("read_config", config) diff --git a/synapse/config/key.py b/synapse/config/key.py index 4a18a94775..27e0d2906e 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -107,7 +107,7 @@ class KeyConfig(Config): ) return keys - def generate_keys(self, config): + def generate_files(self, config): signing_key_path = config["signing_key_path"] if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 37b3d5342c..fa542623b7 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -19,6 +19,47 @@ from twisted.python.log import PythonLoggingObserver import logging import logging.config import yaml +from string import Template +import os + + +DEFAULT_LOG_CONFIG = Template(""" +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s\ +- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: ${log_file} + maxBytes: 104857600 + backupCount: 10 + filters: [context] + level: INFO + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse: + level: INFO + + synapse.storage.SQL: + level: INFO + +root: + level: INFO + handlers: [file, console] +""") class LoggingConfig(Config): @@ -30,6 +71,9 @@ class LoggingConfig(Config): def default_config(self, config_dir_path, server_name): log_file = self.abspath("homeserver.log") + log_config = self.abspath( + os.path.join(config_dir_path, server_name + ".log.config") + ) return """ # Logging verbosity level. verbose: 0 @@ -38,7 +82,7 @@ class LoggingConfig(Config): log_file: "%(log_file)s" # A yaml python logging config file - #log_config: "your.log.config.yaml" + log_config: "%(log_config)s" """ % locals() def read_arguments(self, args): @@ -64,6 +108,14 @@ class LoggingConfig(Config): help="Python logging config file" ) + def generate_files(self, config): + log_config = config.get("log_config") + if log_config and not os.path.exists(log_config): + with open(log_config, "wb") as log_config_file: + log_config_file.write( + DEFAULT_LOG_CONFIG.substitute(log_file=config["log_file"]) + ) + def setup_logging(self): log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e70bc1cd2c..ecb2d42c1f 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -70,7 +70,7 @@ class TlsConfig(Config): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - def generate_keys(self, config): + def generate_files(self, config): tls_certificate_path = config["tls_certificate_path"] tls_private_key_path = config["tls_private_key_path"] tls_dh_params_path = config["tls_dh_params_path"] -- cgit 1.5.1