From a52e1a3b6c5a255f4a51d48dec739751238fa1f9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 24 Jun 2019 18:37:20 +0100 Subject: Factor out "generate_config_from_template" ... and inline generate_secrets --- docker/start.py | 122 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 41 deletions(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index a7a54dacf7..eab39f3c93 100755 --- a/docker/start.py +++ b/docker/start.py @@ -7,37 +7,109 @@ import subprocess import glob import codecs + # Utility functions -convert = lambda src, dst, environ: open(dst, "w").write( - jinja2.Template(open(src).read()).render(**environ) -) +def log(txt): + print(txt, file=sys.stderr) + + +def error(txt): + log(txt) + sys.exit(2) + + +def convert(src, dst, environ): + """Generate a file from a template + + Args: + src (str): path to input file + dst (str): path to file to write + environ (dict): environment dictionary, for replacement mappings. + """ + with open(src) as infile: + template = infile.read() + rendered = jinja2.Template(template).render(**environ) + with open(dst, "w") as outfile: + outfile.write(rendered) def check_arguments(environ, args): for argument in args: if argument not in environ: - print("Environment variable %s is mandatory, exiting." % argument) - sys.exit(2) + error("Environment variable %s is mandatory, exiting." % argument) + + +def generate_config_from_template(environ, ownership): + """Generate a homeserver.yaml from environment variables + + Args: + environ (dict): environment dictionary + ownership (str): ":" string which will be used to set + ownership of the generated configs + Returns: + path to generated config file + """ + for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): + if v not in environ: + error( + "Environment variable '%s' is mandatory when generating a config " + "file on-the-fly." % (v,) + ) + + # populate some params from data files (if they exist, else create new ones) + environ = environ.copy() + secrets = { + "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", + "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", + } -def generate_secrets(environ, secrets): for name, secret in secrets.items(): if secret not in environ: filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name) + + # if the file already exists, load in the existing value; otherwise, + # generate a new secret and write it to a file + if os.path.exists(filename): with open(filename) as handle: value = handle.read() else: - print("Generating a random secret for {}".format(name)) + log("Generating a random secret for {}".format(name)) value = codecs.encode(os.urandom(32), "hex").decode() with open(filename, "w") as handle: handle.write(value) environ[secret] = value + environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") + if not os.path.exists("/compiled"): + os.mkdir("/compiled") + + config_path = "/compiled/homeserver.yaml" + + # Convert SYNAPSE_NO_TLS to boolean if exists + if "SYNAPSE_NO_TLS" in environ: + tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"]) + if tlsanswerstring in ("true", "on", "1", "yes"): + environ["SYNAPSE_NO_TLS"] = True + else: + if tlsanswerstring in ("false", "off", "0", "no"): + environ["SYNAPSE_NO_TLS"] = False + else: + error( + 'Environment variable "SYNAPSE_NO_TLS" found but value "' + + tlsanswerstring + + '" unrecognized; exiting.' + ) + + convert("/conf/homeserver.yaml", config_path, environ) + convert("/conf/log.config", "/compiled/log.config", environ) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + return config_path + # Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None -environ = os.environ.copy() ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver"] @@ -62,39 +134,7 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: config_path = environ["SYNAPSE_CONFIG_PATH"] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) - generate_secrets( - environ, - { - "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", - "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", - }, - ) - environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") - if not os.path.exists("/compiled"): - os.mkdir("/compiled") - - config_path = "/compiled/homeserver.yaml" - - # Convert SYNAPSE_NO_TLS to boolean if exists - if "SYNAPSE_NO_TLS" in environ: - tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"]) - if tlsanswerstring in ("true", "on", "1", "yes"): - environ["SYNAPSE_NO_TLS"] = True - else: - if tlsanswerstring in ("false", "off", "0", "no"): - environ["SYNAPSE_NO_TLS"] = False - else: - print( - 'Environment variable "SYNAPSE_NO_TLS" found but value "' - + tlsanswerstring - + '" unrecognized; exiting.' - ) - sys.exit(2) - - convert("/conf/homeserver.yaml", config_path, environ) - convert("/conf/log.config", "/compiled/log.config", environ) - subprocess.check_output(["chown", "-R", ownership, "/data"]) + config_path = generate_config_from_template(environ, ownership) args += [ "--config-path", -- cgit 1.5.1 From b1fddb7f699a595d39507e876d950f256ab0f8aa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 24 Jun 2019 18:55:37 +0100 Subject: Factor out a run_generate_config function --- docker/start.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index eab39f3c93..70942bcbee 100755 --- a/docker/start.py +++ b/docker/start.py @@ -33,12 +33,6 @@ def convert(src, dst, environ): outfile.write(rendered) -def check_arguments(environ, args): - for argument in args: - if argument not in environ: - error("Environment variable %s is mandatory, exiting." % argument) - - def generate_config_from_template(environ, ownership): """Generate a homeserver.yaml from environment variables @@ -108,17 +102,22 @@ def generate_config_from_template(environ, ownership): return config_path -# Prepare the configuration -mode = sys.argv[1] if len(sys.argv) > 1 else None -ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) -args = ["python", "-m", "synapse.app.homeserver"] +def run_generate_config(environ): + """Run synapse with a --generate-config param to generate a template config file -# In generate mode, generate a configuration, missing keys, then exit -if mode == "generate": - check_arguments( - environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH") - ) - args += [ + Args: + environ (dict): environment dictionary + + Never returns. + """ + for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"): + if v not in environ: + error("Environment variable '%s' is mandatory in `generate` mode." % (v,)) + + args = [ + "python", + "-m", + "synapse.app.homeserver", "--server-name", environ["SYNAPSE_SERVER_NAME"], "--report-stats", @@ -129,6 +128,15 @@ if mode == "generate": ] os.execv("/usr/local/bin/python", args) + +# Prepare the configuration +mode = sys.argv[1] if len(sys.argv) > 1 else None +ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) + +# In generate mode, generate a configuration, missing keys, then exit +if mode == "generate": + run_generate_config(environ) + # In normal mode, generate missing keys if any, then run synapse else: if "SYNAPSE_CONFIG_PATH" in environ: @@ -136,7 +144,10 @@ else: else: config_path = generate_config_from_template(environ, ownership) - args += [ + args = [ + "python", + "-m", + "synapse.app.homeserver", "--config-path", config_path, # tell synapse to put any generated keys in /data rather than /compiled -- cgit 1.5.1 From 3f24e4dce7d4dfb50f0f8d958f6f4daf2dbdc70c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 14:23:24 +0100 Subject: Add a main() function --- docker/start.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index 70942bcbee..df907fe15c 100755 --- a/docker/start.py +++ b/docker/start.py @@ -129,16 +129,15 @@ def run_generate_config(environ): os.execv("/usr/local/bin/python", args) -# Prepare the configuration -mode = sys.argv[1] if len(sys.argv) > 1 else None -ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) +def main(args, environ): + mode = args[1] if len(args) > 1 else None + ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) -# In generate mode, generate a configuration, missing keys, then exit -if mode == "generate": - run_generate_config(environ) + # In generate mode, generate a configuration, missing keys, then exit + if mode == "generate": + return run_generate_config(environ) -# In normal mode, generate missing keys if any, then run synapse -else: + # In normal mode, generate missing keys if any, then run synapse if "SYNAPSE_CONFIG_PATH" in environ: config_path = environ["SYNAPSE_CONFIG_PATH"] else: @@ -158,3 +157,7 @@ else: # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) os.execv("/sbin/su-exec", ["su-exec", ownership] + args) + + +if __name__ == "__main__": + main(sys.argv, os.environ) -- cgit 1.5.1 From 5375c3a9b8121d121cc68756654f4fc20c481995 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 15:10:36 +0100 Subject: isort --- docker/start.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index df907fe15c..bdb703aebd 100755 --- a/docker/start.py +++ b/docker/start.py @@ -1,11 +1,12 @@ #!/usr/local/bin/python -import jinja2 +import codecs +import glob import os -import sys import subprocess -import glob -import codecs +import sys + +import jinja2 # Utility functions -- cgit 1.5.1 From a1732bbff995345e6b6bd79823bb5309038bbe56 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 14:59:02 +0100 Subject: improve logging for generate_config_from_template --- docker/start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index bdb703aebd..9ca139c3ea 100755 --- a/docker/start.py +++ b/docker/start.py @@ -67,10 +67,11 @@ def generate_config_from_template(environ, ownership): # generate a new secret and write it to a file if os.path.exists(filename): + log("Reading %s from %s" % (secret, filename)) with open(filename) as handle: value = handle.read() else: - log("Generating a random secret for {}".format(name)) + log("Generating a random secret for {}".format(secret)) value = codecs.encode(os.urandom(32), "hex").decode() with open(filename, "w") as handle: handle.write(value) -- cgit 1.5.1 From a5fba9c27c5b27fc3c0b1035e087e46fc11e4d79 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 18:09:09 +0100 Subject: Docker: only run --generate-keys when generating config on-the-fly. We don't want to generate any missing configs when running from a precanned config. (There's a strong argument that we don't want to do this at all, since generating a new signing key on each invocation sounds disasterous, but I don't fancy unpicking that for now.) --- docker/start.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index 9ca139c3ea..59527a5883 100755 --- a/docker/start.py +++ b/docker/start.py @@ -101,6 +101,24 @@ def generate_config_from_template(environ, ownership): convert("/conf/homeserver.yaml", config_path, environ) convert("/conf/log.config", "/compiled/log.config", environ) subprocess.check_output(["chown", "-R", ownership, "/data"]) + + # Hopefully we already have a signing key, but generate one if not. + subprocess.check_output( + [ + "su-exec", + ownership, + "python", + "-m", + "synapse.app.homeserver", + "--config-path", + config_path, + # tell synapse to put generated keys in /data rather than /compiled + "--keys-directory", + "/data", + "--generate-keys", + ] + ) + return config_path @@ -146,19 +164,15 @@ def main(args, environ): config_path = generate_config_from_template(environ, ownership) args = [ + "su-exec", + ownership, "python", "-m", "synapse.app.homeserver", "--config-path", config_path, - # tell synapse to put any generated keys in /data rather than /compiled - "--keys-directory", - "/data", ] - - # Generate missing keys and start synapse - subprocess.check_output(args + ["--generate-keys"]) - os.execv("/sbin/su-exec", ["su-exec", ownership] + args) + os.execv("/sbin/su-exec", args) if __name__ == "__main__": -- cgit 1.5.1 From c58a6e6108d2891d10ce7f29e4b0169a3e026787 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 15:09:22 +0100 Subject: document supported env vars for docker 'generate' option --- docker/README.md | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'docker') diff --git a/docker/README.md b/docker/README.md index 5a596eecb9..10af88844b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -179,3 +179,10 @@ docker run -d --name synapse \ -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml \ matrixdotorg/synapse:latest ``` + +The following environment variables are supported in this mode: + +* `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname. +* `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable + anonymous statistics reporting. +* `SYNAPSE_CONFIG_PATH` (mandatory): path to the file to be generated. -- cgit 1.5.1 From 7e433beb65946a39442f025705429227bfa3e429 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 15:18:30 +0100 Subject: Docker image: add support for SYNAPSE_DATA_DIR parameter Fixes #4830. --- docker/README.md | 4 ++++ docker/start.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) (limited to 'docker') diff --git a/docker/README.md b/docker/README.md index 10af88844b..5c85c538a8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -186,3 +186,7 @@ The following environment variables are supported in this mode: * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable anonymous statistics reporting. * `SYNAPSE_CONFIG_PATH` (mandatory): path to the file to be generated. +* `SYNAPSE_DATA_DIR`: where the generated config will put persistent data + such as the datatase and media store. Defaults to `/data`. +* `UID`, `GID`: the user id and group id to use for creating the data + directories. Defaults to `991`, `991`. diff --git a/docker/start.py b/docker/start.py index bdb703aebd..ad968b2a67 100755 --- a/docker/start.py +++ b/docker/start.py @@ -103,11 +103,12 @@ def generate_config_from_template(environ, ownership): return config_path -def run_generate_config(environ): +def run_generate_config(environ, ownership): """Run synapse with a --generate-config param to generate a template config file Args: - environ (dict): environment dictionary + environ (dict): env var dict + ownership (str): "userid:groupid" arg for chmod Never returns. """ @@ -115,6 +116,11 @@ def run_generate_config(environ): if v not in environ: error("Environment variable '%s' is mandatory in `generate` mode." % (v,)) + data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") + + # make sure that synapse has perms to write to the data dir. + subprocess.check_output(["chown", ownership, data_dir]) + args = [ "python", "-m", @@ -125,8 +131,11 @@ def run_generate_config(environ): environ["SYNAPSE_REPORT_STATS"], "--config-path", environ["SYNAPSE_CONFIG_PATH"], + "--data-directory", + data_dir, "--generate-config", ] + # log("running %s" % (args, )) os.execv("/usr/local/bin/python", args) @@ -134,9 +143,9 @@ def main(args, environ): mode = args[1] if len(args) > 1 else None ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) - # In generate mode, generate a configuration, missing keys, then exit + # In generate mode, generate a configuration and missing keys, then exit if mode == "generate": - return run_generate_config(environ) + return run_generate_config(environ, ownership) # In normal mode, generate missing keys if any, then run synapse if "SYNAPSE_CONFIG_PATH" in environ: -- cgit 1.5.1 From 6347dc1beddf71411d0615f4e4101b814457a836 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 18:21:32 +0100 Subject: Add support for SYNAPSE_CONFIG_DIR --- docker/README.md | 2 ++ docker/start.py | 3 +++ 2 files changed, 5 insertions(+) (limited to 'docker') diff --git a/docker/README.md b/docker/README.md index 5c85c538a8..a42d1d24b6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -186,6 +186,8 @@ The following environment variables are supported in this mode: * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable anonymous statistics reporting. * `SYNAPSE_CONFIG_PATH` (mandatory): path to the file to be generated. +* `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config + and event signing key) will be stored. Defaults to `/data`. * `SYNAPSE_DATA_DIR`: where the generated config will put persistent data such as the datatase and media store. Defaults to `/data`. * `UID`, `GID`: the user id and group id to use for creating the data diff --git a/docker/start.py b/docker/start.py index ad968b2a67..99107a9d54 100755 --- a/docker/start.py +++ b/docker/start.py @@ -116,6 +116,7 @@ def run_generate_config(environ, ownership): if v not in environ: error("Environment variable '%s' is mandatory in `generate` mode." % (v,)) + config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") # make sure that synapse has perms to write to the data dir. @@ -131,6 +132,8 @@ def run_generate_config(environ, ownership): environ["SYNAPSE_REPORT_STATS"], "--config-path", environ["SYNAPSE_CONFIG_PATH"], + "--config-directory", + config_dir, "--data-directory", data_dir, "--generate-config", -- cgit 1.5.1 From 28e30c6581b051c13d53d5abc6eab8b6c7d0f96f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 25 Jun 2019 18:23:17 +0100 Subject: Docker: generate our own log config When running under docker, we want to use docker's own logging stuff rather than losing the logs somewhere on the container's filesystem, so let's use log configs that spit logs out to stdout instead. --- docker/start.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/start.py b/docker/start.py index 99107a9d54..06b38ec9db 100755 --- a/docker/start.py +++ b/docker/start.py @@ -116,9 +116,16 @@ def run_generate_config(environ, ownership): if v not in environ: error("Environment variable '%s' is mandatory in `generate` mode." % (v,)) + server_name = environ["SYNAPSE_SERVER_NAME"] config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") + # create a suitable log config from our template + log_config_file = "%s/%s.log.config" % (config_dir, server_name) + if not os.path.exists(log_config_file): + log("Creating log config %s" % (log_config_file,)) + convert("/conf/log.config", log_config_file, environ) + # make sure that synapse has perms to write to the data dir. subprocess.check_output(["chown", ownership, data_dir]) @@ -127,7 +134,7 @@ def run_generate_config(environ, ownership): "-m", "synapse.app.homeserver", "--server-name", - environ["SYNAPSE_SERVER_NAME"], + server_name, "--report-stats", environ["SYNAPSE_REPORT_STATS"], "--config-path", -- cgit 1.5.1 From 2f7ebc2a559409abca9424bfb02cb287c4a21dc6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 27 Jun 2019 13:49:48 +0100 Subject: Deprecate the env var way of running the docker image (#5566) This is mostly a documentation change, but also adds a default value for SYNAPSE_CONFIG_PATH, so that running from the generated config is the default, and will Just Work provided your config is in the right place. --- changelog.d/5566.feature | 1 + contrib/docker/README.md | 4 + docker/README.md | 216 +++++++++++++++-------------------------------- docker/start.py | 32 +++++-- 4 files changed, 99 insertions(+), 154 deletions(-) create mode 100644 changelog.d/5566.feature (limited to 'docker') diff --git a/changelog.d/5566.feature b/changelog.d/5566.feature new file mode 100644 index 0000000000..85380bc517 --- /dev/null +++ b/changelog.d/5566.feature @@ -0,0 +1 @@ +Update Docker image to deprecate the use of environment variables for configuration, and make the use of a static configuration the default. diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 05254e5192..af102f7594 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,5 +1,9 @@ # Synapse Docker +FIXME: this is out-of-date as of +https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up +to date would be welcome. + ### Automated configuration It is recommended that you use Docker Compose to run your containers, including diff --git a/docker/README.md b/docker/README.md index a42d1d24b6..7f7d27ed33 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,39 +6,11 @@ postgres database. The image also does *not* provide a TURN server. -## Run - -### Using docker-compose (easier) - -This image is designed to run either with an automatically generated -configuration file or with a custom configuration that requires manual editing. - -An easy way to make use of this image is via docker-compose. See the -[contrib/docker](https://github.com/matrix-org/synapse/tree/master/contrib/docker) section of the synapse project for -examples. - -### Without Compose (harder) - -If you do not wish to use Compose, you may still run this image using plain -Docker commands. Note that the following is just a guideline and you may need -to add parameters to the docker run command to account for the network situation -with your postgres database. - -``` -docker run \ - -d \ - --name synapse \ - --mount type=volume,src=synapse-data,dst=/data \ - -e SYNAPSE_SERVER_NAME=my.matrix.host \ - -e SYNAPSE_REPORT_STATS=yes \ - -p 8448:8448 \ - matrixdotorg/synapse:latest -``` - ## Volumes -The image expects a single volume, located at ``/data``, that will hold: +By default, the image expects a single volume, located at ``/data``, that will hold: +* configuration files; * temporary files during uploads; * uploaded media and thumbnails; * the SQLite database if you do not configure postgres; @@ -53,142 +25,90 @@ In order to setup an application service, simply create an ``appservices`` directory in the data volume and write the application service Yaml configuration file there. Multiple application services are supported. -## TLS certificates - -Synapse requires a valid TLS certificate. You can do one of the following: - - * Provide your own certificate and key (as - `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.crt` and - `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.key`, or elsewhere by providing an - entire config as `${SYNAPSE_CONFIG_PATH}`). In this case, you should forward - traffic to port 8448 in the container, for example with `-p 443:8448`. - - * Use a reverse proxy to terminate incoming TLS, and forward the plain http - traffic to port 8008 in the container. In this case you should set `-e - SYNAPSE_NO_TLS=1`. - - * Use the ACME (Let's Encrypt) support built into Synapse. This requires - `${SYNAPSE_SERVER_NAME}` port 80 to be forwarded to port 8009 in the - container, for example with `-p 80:8009`. To enable it in the docker - container, set `-e SYNAPSE_ACME=1`. - -If you don't do any of these, Synapse will fail to start with an error similar to: - - synapse.config._base.ConfigError: Error accessing file '/data/.tls.crt' (config for tls_certificate): No such file or directory - -## Environment - -Unless you specify a custom path for the configuration file, a very generic -file will be generated, based on the following environment settings. -These are a good starting point for setting up your own deployment. - -Global settings: - -* ``UID``, the user id Synapse will run as [default 991] -* ``GID``, the group id Synapse will run as [default 991] -* ``SYNAPSE_CONFIG_PATH``, path to a custom config file - -If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file -then customize it manually: see [Generating a config -file](#generating-a-config-file). - -Otherwise, a dynamic configuration file will be used. - -### Environment variables used to build a dynamic configuration file - -The following environment variables are used to build the configuration file -when ``SYNAPSE_CONFIG_PATH`` is not set. - -* ``SYNAPSE_SERVER_NAME`` (mandatory), the server public hostname. -* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous - statistics reporting back to the Matrix project which helps us to get funding. -* `SYNAPSE_NO_TLS`, (accepts `true`, `false`, `on`, `off`, `1`, `0`, `yes`, `no`]): disable - TLS in Synapse (use this if you run your own TLS-capable reverse proxy). Defaults - to `false` (ie, TLS is enabled by default). -* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on - the Synapse instance. -* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. -* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. -* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public - key in order to enable recaptcha upon registration. -* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private - key in order to enable recaptcha upon registration. -* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN - uris to enable TURN for this homeserver. -* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. -* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size - [default `10M`]. -* ``SYNAPSE_ACME``: set this to enable the ACME certificate renewal support. - -Shared secrets, that will be initialized to random values if not set: - -* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if - registration is disable. -* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens - to the server. - -Database specific values (will use SQLite if not set): - -* `POSTGRES_DB` - The database name for the synapse postgres - database. [default: `synapse`] -* `POSTGRES_HOST` - The host of the postgres database if you wish to use - postgresql instead of sqlite3. [default: `db` which is useful when using a - container on the same docker network in a compose file where the postgres - service is called `db`] -* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If - this is set then postgres will be used instead of sqlite3.** [default: none] - **NOTE**: You are highly encouraged to use postgresql! Please use the compose - file to make it easier to deploy. -* `POSTGRES_USER` - The user for the synapse postgres database. [default: - `synapse`] - -Mail server specific values (will not send emails if not set): - -* ``SYNAPSE_SMTP_HOST``, hostname to the mail server. -* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default - ``25``]. -* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if - any. -* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail - server if any. - -### Generating a config file - -It is possible to generate a basic configuration file for use with -`SYNAPSE_CONFIG_PATH` using the `generate` commandline option. You will need to -specify values for `SYNAPSE_CONFIG_PATH`, `SYNAPSE_SERVER_NAME` and -`SYNAPSE_REPORT_STATS`, and mount a docker volume to store the data on. For -example: +## Generating a configuration file + +The first step is to genearte a valid config file. To do this, you can run the +image with the `generate` commandline option. + +You will need to specify values for the `SYNAPSE_SERVER_NAME` and +`SYNAPSE_REPORT_STATS` environment variable, and mount a docker volume to store +the configuration on. For example: ``` docker run -it --rm \ --mount type=volume,src=synapse-data,dst=/data \ - -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ -e SYNAPSE_REPORT_STATS=yes \ matrixdotorg/synapse:latest generate ``` -This will generate a `homeserver.yaml` in (typically) -`/var/lib/docker/volumes/synapse-data/_data`, which you can then customise and -use with: +For information on picking a suitable server name, see +https://github.com/matrix-org/synapse/blob/master/INSTALL.md. -``` -docker run -d --name synapse \ - --mount type=volume,src=synapse-data,dst=/data \ - -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml \ - matrixdotorg/synapse:latest -``` +The above command will generate a `homeserver.yaml` in (typically) +`/var/lib/docker/volumes/synapse-data/_data`. You should check this file, and +customise it to your needs. -The following environment variables are supported in this mode: +The following environment variables are supported in `generate` mode: * `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname. * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable anonymous statistics reporting. -* `SYNAPSE_CONFIG_PATH` (mandatory): path to the file to be generated. * `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config and event signing key) will be stored. Defaults to `/data`. +* `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to + `/homeserver.yaml`. * `SYNAPSE_DATA_DIR`: where the generated config will put persistent data such as the datatase and media store. Defaults to `/data`. * `UID`, `GID`: the user id and group id to use for creating the data directories. Defaults to `991`, `991`. + + +## Running synapse + +Once you have a valid configuration file, you can start synapse as follows: + +``` +docker run -d --name synapse \ + --mount type=volume,src=synapse-data,dst=/data \ + -p 8008:8008 \ + matrixdotorg/synapse:latest +``` + +You can then check that it has started correctly with: + +``` +docker logs synapse +``` + +If all is well, you should now be able to connect to http://localhost:8008 and +see a confirmation message. + +The following environment variables are supported in run mode: + +* `SYNAPSE_CONFIG_DIR`: where additional config files are stored. Defaults to + `/data`. +* `SYNAPSE_CONFIG_PATH`: path to the config file. Defaults to + `/homeserver.yaml`. +* `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`. + +## TLS support + +The default configuration exposes a single HTTP port: http://localhost:8008. It +is suitable for local testing, but for any practical use, you will either need +to use a reverse proxy, or configure Synapse to expose an HTTPS port. + +For documentation on using a reverse proxy, see +https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst. + +For more information on enabling TLS support in synapse itself, see +https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of +course, you will need to expose the TLS port from the container with a `-p` +argument to `docker run`. + +## Legacy dynamic configuration file support + +For backwards-compatibility only, the docker image supports creating a dynamic +configuration file based on environment variables. This is now deprecated, but +is enabled when the `SYNAPSE_SERVER_NAME` variable is set (and `generate` is +not given). diff --git a/docker/start.py b/docker/start.py index fdc3d59d86..89ec9ebaf8 100755 --- a/docker/start.py +++ b/docker/start.py @@ -131,12 +131,13 @@ def run_generate_config(environ, ownership): Never returns. """ - for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"): + for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): if v not in environ: error("Environment variable '%s' is mandatory in `generate` mode." % (v,)) server_name = environ["SYNAPSE_SERVER_NAME"] config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") + config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") # create a suitable log config from our template @@ -157,7 +158,7 @@ def run_generate_config(environ, ownership): "--report-stats", environ["SYNAPSE_REPORT_STATS"], "--config-path", - environ["SYNAPSE_CONFIG_PATH"], + config_path, "--config-directory", config_dir, "--data-directory", @@ -176,11 +177,30 @@ def main(args, environ): if mode == "generate": return run_generate_config(environ, ownership) - # In normal mode, generate missing keys if any, then run synapse - if "SYNAPSE_CONFIG_PATH" in environ: - config_path = environ["SYNAPSE_CONFIG_PATH"] - else: + if "SYNAPSE_SERVER_NAME" in environ: + # backwards-compatibility generate-a-config-on-the-fly mode + if "SYNAPSE_CONFIG_PATH" in environ: + error( + "SYNAPSE_SERVER_NAME and SYNAPSE_CONFIG_PATH are mutually exclusive " + "except in `generate` mode." + ) + config_path = generate_config_from_template(environ, ownership) + else: + config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") + config_path = environ.get( + "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml" + ) + if not os.path.exists(config_path): + error( + "Config file '%s' does not exist. You should either create a new " + "config file by running with the `generate` argument (and then edit " + "the resulting file before restarting) or specify the path to an " + "existing config file with the SYNAPSE_CONFIG_PATH variable." + % (config_path,) + ) + + log("Starting synapse with config file " + config_path) args = [ "su-exec", -- cgit 1.5.1 From 1ddc7b39c90d0d823fc6935857eda5153b542d83 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 27 Jun 2019 13:50:10 +0100 Subject: Docker image: open the non-TLS port by default. (#5568) There's not much point in binding to localhost when it's in a docker container. --- changelog.d/5568.feature | 1 + docker/start.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/5568.feature (limited to 'docker') diff --git a/changelog.d/5568.feature b/changelog.d/5568.feature new file mode 100644 index 0000000000..59b9e5f96d --- /dev/null +++ b/changelog.d/5568.feature @@ -0,0 +1 @@ +Docker image: open the non-TLS port by default. diff --git a/docker/start.py b/docker/start.py index 89ec9ebaf8..2a13308dae 100755 --- a/docker/start.py +++ b/docker/start.py @@ -164,6 +164,7 @@ def run_generate_config(environ, ownership): "--data-directory", data_dir, "--generate-config", + "--open-private-ports", ] # log("running %s" % (args, )) os.execv("/usr/local/bin/python", args) -- cgit 1.5.1 From 555b6fa0d534ac897b139ee1e57a651e50e1aec5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 27 Jun 2019 13:52:40 +0100 Subject: Docker image: Add a migrate_config mode (#5567) ... to help people escape env var hell --- changelog.d/5567.feature | 1 + docker/README.md | 16 +++++++++++++ docker/conf/homeserver.yaml | 2 +- docker/start.py | 56 ++++++++++++++++++++++++++++++++------------- 4 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 changelog.d/5567.feature (limited to 'docker') diff --git a/changelog.d/5567.feature b/changelog.d/5567.feature new file mode 100644 index 0000000000..85380bc517 --- /dev/null +++ b/changelog.d/5567.feature @@ -0,0 +1 @@ +Update Docker image to deprecate the use of environment variables for configuration, and make the use of a static configuration the default. diff --git a/docker/README.md b/docker/README.md index 7f7d27ed33..b62417c281 100644 --- a/docker/README.md +++ b/docker/README.md @@ -112,3 +112,19 @@ For backwards-compatibility only, the docker image supports creating a dynamic configuration file based on environment variables. This is now deprecated, but is enabled when the `SYNAPSE_SERVER_NAME` variable is set (and `generate` is not given). + +To migrate from a dynamic configuration file to a static one, run the docker +container once with the environment variables set, and `migrate_config` +commandline option. For example: + +``` +docker run -it --rm \ + --mount type=volume,src=synapse-data,dst=/data \ + -e SYNAPSE_SERVER_NAME=my.matrix.host \ + -e SYNAPSE_REPORT_STATS=yes \ + matrixdotorg/synapse:latest migrate_config +``` + +This will generate the same configuration file as the legacy mode used, but +will store it in `/data/homeserver.yaml` instead of a temporary location. You +can then use it as shown above at [Running synapse](#running-synapse). diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index babd5bef9e..b0267b1c60 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -21,7 +21,7 @@ server_name: "{{ SYNAPSE_SERVER_NAME }}" pid_file: /homeserver.pid web_client: False soft_file_limit: 0 -log_config: "/compiled/log.config" +log_config: "{{ SYNAPSE_LOG_CONFIG }}" ## Ports ## diff --git a/docker/start.py b/docker/start.py index 2a13308dae..40a861f200 100755 --- a/docker/start.py +++ b/docker/start.py @@ -34,22 +34,21 @@ def convert(src, dst, environ): outfile.write(rendered) -def generate_config_from_template(environ, ownership): +def generate_config_from_template(config_dir, config_path, environ, ownership): """Generate a homeserver.yaml from environment variables Args: + config_dir (str): where to put generated config files + config_path (str): where to put the main config file environ (dict): environment dictionary ownership (str): ":" string which will be used to set ownership of the generated configs - - Returns: - path to generated config file """ for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): if v not in environ: error( - "Environment variable '%s' is mandatory when generating a config " - "file on-the-fly." % (v,) + "Environment variable '%s' is mandatory when generating a config file." + % (v,) ) # populate some params from data files (if they exist, else create new ones) @@ -78,10 +77,8 @@ def generate_config_from_template(environ, ownership): environ[secret] = value environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") - if not os.path.exists("/compiled"): - os.mkdir("/compiled") - - config_path = "/compiled/homeserver.yaml" + if not os.path.exists(config_dir): + os.mkdir(config_dir) # Convert SYNAPSE_NO_TLS to boolean if exists if "SYNAPSE_NO_TLS" in environ: @@ -98,8 +95,16 @@ def generate_config_from_template(environ, ownership): + '" unrecognized; exiting.' ) + if "SYNAPSE_LOG_CONFIG" not in environ: + environ["SYNAPSE_LOG_CONFIG"] = config_dir + "/log.config" + + log("Generating synapse config file " + config_path) convert("/conf/homeserver.yaml", config_path, environ) - convert("/conf/log.config", "/compiled/log.config", environ) + + log_config_file = environ["SYNAPSE_LOG_CONFIG"] + log("Generating log config file " + log_config_file) + convert("/conf/log.config", log_config_file, environ) + subprocess.check_output(["chown", "-R", ownership, "/data"]) # Hopefully we already have a signing key, but generate one if not. @@ -114,13 +119,11 @@ def generate_config_from_template(environ, ownership): config_path, # tell synapse to put generated keys in /data rather than /compiled "--keys-directory", - "/data", + config_dir, "--generate-keys", ] ) - return config_path - def run_generate_config(environ, ownership): """Run synapse with a --generate-config param to generate a template config file @@ -178,15 +181,36 @@ def main(args, environ): if mode == "generate": return run_generate_config(environ, ownership) + if mode == "migrate_config": + # generate a config based on environment vars. + config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") + config_path = environ.get( + "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml" + ) + return generate_config_from_template( + config_dir, config_path, environ, ownership + ) + + if mode is not None: + error("Unknown execution mode '%s'" % (mode,)) + if "SYNAPSE_SERVER_NAME" in environ: # backwards-compatibility generate-a-config-on-the-fly mode if "SYNAPSE_CONFIG_PATH" in environ: error( "SYNAPSE_SERVER_NAME and SYNAPSE_CONFIG_PATH are mutually exclusive " - "except in `generate` mode." + "except in `generate` or `migrate_config` mode." ) - config_path = generate_config_from_template(environ, ownership) + config_path = "/compiled/homeserver.yaml" + log( + "Generating config file '%s' on-the-fly from environment variables.\n" + "Note that this mode is deprecated. You can migrate to a static config\n" + "file by running with 'migrate_config'. See the README for more details." + % (config_path,) + ) + + generate_config_from_template("/compiled", config_path, environ, ownership) else: config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get( -- cgit 1.5.1