From d378c3da784416449d3e03ed45f05d55dfcfe26e Mon Sep 17 00:00:00 2001 From: Juho Vanhanen Date: Mon, 6 Jul 2020 15:37:39 +0300 Subject: Add libwebp dependency to Dockerfile (#7791) * Add libwebp dependency to Dockerfile Signed-off-by: Juho Vanhanen --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) (limited to 'docker') diff --git a/docker/Dockerfile b/docker/Dockerfile index 9a3cf7b3f5..093e89af6c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,6 +24,7 @@ RUN apk add \ build-base \ libffi-dev \ libjpeg-turbo-dev \ + libwebp-dev \ libressl-dev \ libxslt-dev \ linux-headers \ @@ -61,6 +62,7 @@ FROM docker.io/python:${PYTHON_VERSION}-alpine3.11 RUN apk add --no-cache --virtual .runtime_deps \ libffi \ libjpeg-turbo \ + libwebp \ libressl \ libxslt \ libpq \ -- cgit 1.5.1 From a5545cf86d6642cf583f3cad7156b8ba14efe81f Mon Sep 17 00:00:00 2001 From: Christopher May-Townsend Date: Fri, 17 Jul 2020 17:40:53 +0100 Subject: Switch to Debian:Slim from Alpine for the docker image (#7839) As mentioned in #7397, switching to a debian base should help with multi-arch work to save time on compiling. This is unashamedly based on #6373, but without the extra functionality. Switch python version back to generic 3.7 to always pull the latest. Essentially, keeping this as small as possible. The image is bigger though unfortunately. --- changelog.d/7839.docker | 1 + docker/Dockerfile | 57 ++++++++++++++++++++----------------------------- docker/start.py | 12 +++++------ 3 files changed, 30 insertions(+), 40 deletions(-) create mode 100644 changelog.d/7839.docker (limited to 'docker') diff --git a/changelog.d/7839.docker b/changelog.d/7839.docker new file mode 100644 index 0000000000..cdf3c9631c --- /dev/null +++ b/changelog.d/7839.docker @@ -0,0 +1 @@ +Base docker image on Debian Buster rather than Alpine Linux. Contributed by @maquis196. diff --git a/docker/Dockerfile b/docker/Dockerfile index 093e89af6c..8b3a4246a5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,35 +16,31 @@ ARG PYTHON_VERSION=3.7 ### ### Stage 0: builder ### -FROM docker.io/python:${PYTHON_VERSION}-alpine3.11 as builder +FROM docker.io/python:${PYTHON_VERSION}-slim as builder # install the OS build deps -RUN apk add \ - build-base \ - libffi-dev \ - libjpeg-turbo-dev \ - libwebp-dev \ - libressl-dev \ - libxslt-dev \ - linux-headers \ - postgresql-dev \ - zlib-dev -# build things which have slow build steps, before we copy synapse, so that -# the layer can be cached. -# -# (we really just care about caching a wheel here, as the "pip install" below -# will install them again.) +RUN apt-get update && apt-get install -y \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* +# Build dependencies that are not available as wheels, to speed up rebuilds RUN pip install --prefix="/install" --no-warn-script-location \ - cryptography \ - msgpack-python \ - pillow \ - pynacl + frozendict \ + jaeger-client \ + opentracing \ + prometheus-client \ + psycopg2 \ + pycparser \ + pyrsistent \ + pyyaml \ + simplejson \ + threadloop \ + thrift # now install synapse and all of the python deps to /install. - COPY synapse /synapse/synapse/ COPY scripts /synapse/scripts/ COPY MANIFEST.in README.rst setup.py synctl /synapse/ @@ -56,20 +52,13 @@ RUN pip install --prefix="/install" --no-warn-script-location \ ### Stage 1: runtime ### -FROM docker.io/python:${PYTHON_VERSION}-alpine3.11 +FROM docker.io/python:${PYTHON_VERSION}-slim -# xmlsec is required for saml support -RUN apk add --no-cache --virtual .runtime_deps \ - libffi \ - libjpeg-turbo \ - libwebp \ - libressl \ - libxslt \ - libpq \ - zlib \ - su-exec \ - tzdata \ - xmlsec +RUN apt-get update && apt-get install -y \ + libpq5 \ + xmlsec1 \ + gosu \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py diff --git a/docker/start.py b/docker/start.py index 2a25c9380e..9f08134158 100755 --- a/docker/start.py +++ b/docker/start.py @@ -120,7 +120,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): if ownership is not None: subprocess.check_output(["chown", "-R", ownership, "/data"]) - args = ["su-exec", ownership] + args + args = ["gosu", ownership] + args subprocess.check_output(args) @@ -172,8 +172,8 @@ def run_generate_config(environ, ownership): # make sure that synapse has perms to write to the data dir. subprocess.check_output(["chown", ownership, data_dir]) - args = ["su-exec", ownership] + args - os.execv("/sbin/su-exec", args) + args = ["gosu", ownership] + args + os.execv("/usr/sbin/gosu", args) else: os.execv("/usr/local/bin/python", args) @@ -189,7 +189,7 @@ def main(args, environ): ownership = "{}:{}".format(desired_uid, desired_gid) if ownership is None: - log("Will not perform chmod/su-exec as UserID already matches request") + log("Will not perform chmod/gosu as UserID already matches request") # In generate mode, generate a configuration and missing keys, then exit if mode == "generate": @@ -236,8 +236,8 @@ running with 'migrate_config'. See the README for more details. args = ["python", "-m", synapse_worker, "--config-path", config_path] if ownership is not None: - args = ["su-exec", ownership] + args - os.execv("/sbin/su-exec", args) + args = ["gosu", ownership] + args + os.execv("/usr/sbin/gosu", args) else: os.execv("/usr/local/bin/python", args) -- cgit 1.5.1 From 1ec2961b3b214c7af4b541396adc94948b626c47 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Fri, 17 Jul 2020 18:25:48 +0100 Subject: Add help for creating a user via docker (#7885) --- changelog.d/7885.doc | 1 + docker/README.md | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 changelog.d/7885.doc (limited to 'docker') diff --git a/changelog.d/7885.doc b/changelog.d/7885.doc new file mode 100644 index 0000000000..cbe9de4082 --- /dev/null +++ b/changelog.d/7885.doc @@ -0,0 +1 @@ +Provide instructions on using `register_new_matrix_user` via docker. diff --git a/docker/README.md b/docker/README.md index 8c337149ca..008a9ff708 100644 --- a/docker/README.md +++ b/docker/README.md @@ -94,6 +94,21 @@ The following environment variables are supported in run mode: * `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`. * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`. +## Generating an (admin) user + +After synapse is running, you may wish to create a user via `register_new_matrix_user`. + +This requires a `registration_shared_secret` to be set in your config file. Synapse +must be restarted to pick up this change. + +You can then call the script: + +``` +docker exec -it synapse register_new_matrix_user http://localhost:8008 -c /data/homeserver.yaml --help +``` + +Remember to remove the `registration_shared_secret` and restart if you no-longer need it. + ## TLS support The default configuration exposes a single HTTP port: http://localhost:8008. It -- cgit 1.5.1 From 0304ad0c3d79e44e78f9658e71f1e1533e3aa4e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Aug 2020 11:39:29 +0100 Subject: Move setting of Filter into code. We do this to prevent foot guns. The default config uses a MemoryFilter, but users are free to change to logging to files directly. If they do then they have to ensure to set the `filters: [context]` on the right handler, otherwise records get written with the wrong context. Instead we move the logic to happen when we generate a record, which is when we *log* rather than *handle*. (It's possible to add filters to loggers in the config, however they don't apply to descendant loggers and so they have to be manually set on *every* logger used in the code base) --- docker/conf/log.config | 6 ------ docs/sample_log_config.yaml | 7 ------- synapse/config/logger.py | 24 ++++++++++++++++-------- 3 files changed, 16 insertions(+), 21 deletions(-) (limited to 'docker') diff --git a/docker/conf/log.config b/docker/conf/log.config index ed418a57cd..491bbcc87a 100644 --- a/docker/conf/log.config +++ b/docker/conf/log.config @@ -4,16 +4,10 @@ formatters: precise: format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -filters: - context: - (): synapse.logging.context.LoggingContextFilter - request: "" - handlers: console: class: logging.StreamHandler formatter: precise - filters: [context] loggers: synapse.storage.SQL: diff --git a/docs/sample_log_config.yaml b/docs/sample_log_config.yaml index 403ac005ee..55a48a9ed6 100644 --- a/docs/sample_log_config.yaml +++ b/docs/sample_log_config.yaml @@ -11,11 +11,6 @@ formatters: precise: format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -filters: - context: - (): synapse.logging.context.LoggingContextFilter - request: "" - handlers: file: class: logging.handlers.TimedRotatingFileHandler @@ -30,7 +25,6 @@ handlers: # logs will still be flushed immediately. buffer: class: logging.handlers.MemoryHandler - filters: [context] target: file # The capacity is the number of log lines that are buffered before # being written to disk. Increasing this will lead to better @@ -44,7 +38,6 @@ handlers: console: class: logging.StreamHandler formatter: precise - filters: [context] loggers: synapse.storage.SQL: diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 493e98462d..c96e6ef62a 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -55,11 +55,6 @@ formatters: format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - \ %(request)s - %(message)s' -filters: - context: - (): synapse.logging.context.LoggingContextFilter - request: "" - handlers: file: class: logging.handlers.TimedRotatingFileHandler @@ -74,7 +69,6 @@ handlers: # logs will still be flushed immediately. buffer: class: logging.handlers.MemoryHandler - filters: [context] target: file # The capacity is the number of log lines that are buffered before # being written to disk. Increasing this will lead to better @@ -88,7 +82,6 @@ handlers: console: class: logging.StreamHandler formatter: precise - filters: [context] loggers: synapse.storage.SQL: @@ -199,11 +192,26 @@ def _setup_stdlib_logging(config, log_config, logBeginner: LogBeginner): handler = logging.StreamHandler() handler.setFormatter(formatter) - handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) else: logging.config.dictConfig(log_config) + # We add a log record factory that runs all messages through the + # LoggingContextFilter so that we get the context *at the time we log* + # rather than when we write to a handler. This can be done in config using + # filter options, but care must when using e.g. MemoryHandler to buffer + # writes. + + log_filter = LoggingContextFilter(request="") + old_factory = logging.getLogRecordFactory() + + def factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + log_filter.filter(record) + return record + + logging.setLogRecordFactory(factory) + # Route Twisted's native logging through to the standard library logging # system. observer = STDLibLogObserver() -- cgit 1.5.1 From 64e8a4697abd84329aa59877b9bca5704d9e0f4c Mon Sep 17 00:00:00 2001 From: Christopher May-Townsend Date: Mon, 24 Aug 2020 18:15:18 +0100 Subject: Add healthcheck for default localhost 8008 port on /health endpoint. (#8147) --- changelog.d/8147.docker | 1 + docker/Dockerfile | 4 ++++ docker/README.md | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 changelog.d/8147.docker (limited to 'docker') diff --git a/changelog.d/8147.docker b/changelog.d/8147.docker new file mode 100644 index 0000000000..dcc951d8f5 --- /dev/null +++ b/changelog.d/8147.docker @@ -0,0 +1 @@ +Added curl for healthcheck support and readme updates for the change. Contributed by @maquis196. diff --git a/docker/Dockerfile b/docker/Dockerfile index 8b3a4246a5..432d56a8ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -55,6 +55,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \ FROM docker.io/python:${PYTHON_VERSION}-slim RUN apt-get update && apt-get install -y \ + curl \ libpq5 \ xmlsec1 \ gosu \ @@ -69,3 +70,6 @@ VOLUME ["/data"] EXPOSE 8008/tcp 8009/tcp 8448/tcp ENTRYPOINT ["/start.py"] + +HEALTHCHECK --interval=1m --timeout=5s \ + CMD curl -fSs http://localhost:8008/health || exit 1 diff --git a/docker/README.md b/docker/README.md index 008a9ff708..d0da34778e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -162,3 +162,32 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile . You can choose to build a different docker image by changing the value of the `-f` flag to point to another Dockerfile. + +## Disabling the healthcheck + +If you are using a non-standard port or tls inside docker you can disable the healthcheck +whilst running the above `docker run` commands. + +``` + --no-healthcheck +``` +## Setting custom healthcheck on docker run + +If you wish to point the healthcheck at a different port with docker command, add the following + +``` + --health-cmd 'curl -fSs http://localhost:1234/health' +``` + +## Setting the healthcheck in docker-compose file + +You can add the following to set a custom healthcheck in a docker compose file. +You will need version >2.1 for this to work. + +``` +healthcheck: + test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"] + interval: 1m + timeout: 10s + retries: 3 +``` -- cgit 1.5.1 From ed18f32e1b7bf734303e040400a2da2e27501154 Mon Sep 17 00:00:00 2001 From: Christopher May-Townsend Date: Wed, 26 Aug 2020 15:03:20 +0100 Subject: Add required Debian dependencies to allow docker builds on the arm platform (#8144) Signed-off-by: Christopher May-Townsend --- changelog.d/8144.docker | 1 + docker/Dockerfile | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 changelog.d/8144.docker (limited to 'docker') diff --git a/changelog.d/8144.docker b/changelog.d/8144.docker new file mode 100644 index 0000000000..9bb5881fa8 --- /dev/null +++ b/changelog.d/8144.docker @@ -0,0 +1 @@ +Fix builds of the Docker image on non-x86 platforms. diff --git a/docker/Dockerfile b/docker/Dockerfile index 432d56a8ee..27512f8600 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,11 +19,16 @@ ARG PYTHON_VERSION=3.7 FROM docker.io/python:${PYTHON_VERSION}-slim as builder # install the OS build deps - - RUN apt-get update && apt-get install -y \ build-essential \ + libffi-dev \ + libjpeg-dev \ libpq-dev \ + libssl-dev \ + libwebp-dev \ + libxml++2.6-dev \ + libxslt1-dev \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* # Build dependencies that are not available as wheels, to speed up rebuilds @@ -56,9 +61,11 @@ FROM docker.io/python:${PYTHON_VERSION}-slim RUN apt-get update && apt-get install -y \ curl \ + gosu \ + libjpeg62-turbo \ libpq5 \ + libwebp6 \ xmlsec1 \ - gosu \ && rm -rf /var/lib/apt/lists/* COPY --from=builder /install /usr/local -- cgit 1.5.1