diff --git a/changelog.d/9162.misc b/changelog.d/9162.misc
new file mode 100644
index 0000000000..1083da8a7a
--- /dev/null
+++ b/changelog.d/9162.misc
@@ -0,0 +1 @@
+Add a dockerfile for running Synapse in worker-mode under Complement.
\ No newline at end of file
diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
new file mode 100644
index 0000000000..969cf97286
--- /dev/null
+++ b/docker/Dockerfile-workers
@@ -0,0 +1,23 @@
+# Inherit from the official Synapse docker image
+FROM matrixdotorg/synapse
+
+# Install deps
+RUN apt-get update
+RUN apt-get install -y supervisor redis nginx
+
+# Remove the default nginx sites
+RUN rm /etc/nginx/sites-enabled/default
+
+# Copy Synapse worker, nginx and supervisord configuration template files
+COPY ./docker/conf-workers/* /conf/
+
+# Expose nginx listener port
+EXPOSE 8080/tcp
+
+# Volume for user-editable config files, logs etc.
+VOLUME ["/data"]
+
+# A script to read environment variables and create the necessary
+# files to run the desired worker configuration. Will start supervisord.
+COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
+ENTRYPOINT ["/configure_workers_and_start.py"]
diff --git a/docker/README-testing.md b/docker/README-testing.md
new file mode 100644
index 0000000000..6a5baf9e28
--- /dev/null
+++ b/docker/README-testing.md
@@ -0,0 +1,140 @@
+# Running tests against a dockerised Synapse
+
+It's possible to run integration tests against Synapse
+using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
+compliance test suite for homeservers, and supports any homeserver docker image configured
+to listen on ports 8008/8448. This document contains instructions for building Synapse
+docker images that can be run inside Complement for testing purposes.
+
+Note that running Synapse's unit tests from within the docker image is not supported.
+
+## Testing with SQLite and single-process Synapse
+
+> Note that `scripts-dev/complement.sh` is a script that will automatically build
+> and run an SQLite-based, single-process of Synapse against Complement.
+
+The instructions below will set up Complement testing for a single-process,
+SQLite-based Synapse deployment.
+
+Start by building the base Synapse docker image. If you wish to run tests with the latest
+release of Synapse, instead of your current checkout, you can skip this step. From the
+root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, build the Synapse image for Complement. You will need a local checkout
+of Complement. Change to the root of your Complement checkout and run:
+
+```sh
+docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+how to run the tests, as well as the various available command line flags.
+
+## Testing with PostgreSQL and single or multi-process Synapse
+
+The above docker image only supports running Synapse with SQLite and in a
+single-process topology. The following instructions are used to build a Synapse image for
+Complement that supports either single or multi-process topology with a PostgreSQL
+database backend.
+
+As with the single-process image, build the base Synapse docker image. If you wish to run
+tests with the latest release of Synapse, instead of your current checkout, you can skip
+this step. From the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
+Again, from the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
+```
+
+This will build an image with the tag` matrixdotorg/synapse-workers`.
+
+It's worth noting at this point that this image is fully functional, and
+can be used for testing against locally. See instructions for using the container
+under
+[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
+below.
+
+Finally, build the Synapse image for Complement, which is based on
+`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
+the root of your Complement checkout and run:
+
+```sh
+docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+how to run the tests, as well as the various available command line flags.
+
+## Running the Dockerfile-worker image standalone
+
+For manual testing of a multi-process Synapse instance in Docker,
+[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
+bundling all necessary components together for a workerised homeserver instance.
+
+This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
+a redis for worker communication and a supervisord instance to start up and monitor all
+processes. You will need to provide your own postgres container to connect to, and TLS
+is not handled by the container.
+
+Once you've built the image using the above instructions, you can run it. Be sure
+you've set up a volume according to the [usual Synapse docker instructions](README.md).
+Then run something along the lines of:
+
+```
+docker run -d --name synapse \
+ --mount type=volume,src=synapse-data,dst=/data \
+ -p 8008:8008 \
+ -e SYNAPSE_SERVER_NAME=my.matrix.host \
+ -e SYNAPSE_REPORT_STATS=no \
+ -e POSTGRES_HOST=postgres \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=somesecret \
+ -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
+ -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
+ matrixdotorg/synapse-workers
+```
+
+...substituting `POSTGRES*` variables for those that match a postgres host you have
+available (usually a running postgres docker container).
+
+The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
+use when running the container. All possible worker names are defined by the keys of the
+`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
+Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
+files.
+
+Sharding is supported for a subset of workers, in line with the
+[worker documentation](../docs/workers.md). To run multiple instances of a given worker
+type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
+(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
+
+Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
+(leaving only the main process). The container is configured to use redis-based worker
+mode.
+
+Logs for workers and the main process are logged to stdout and can be viewed with
+standard `docker logs` tooling. Worker logs contain their worker name
+after the timestamp.
+
+Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
+`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
+00, according to the container's clock. Logging for the main process must still be
+configured by modifying the homeserver's log config in your Synapse data volume.
diff --git a/docker/README.md b/docker/README.md
index b65bcea636..a7d1e670fe 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -11,7 +11,7 @@ is not supported by this image.
## Volumes
-By default, the image expects a single volume, located at ``/data``, that will hold:
+By default, the image expects a single volume, located at `/data`, that will hold:
* configuration files;
* uploaded media and thumbnails;
@@ -19,11 +19,11 @@ By default, the image expects a single volume, located at ``/data``, that will h
* the appservices configuration.
You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` could be stored on a large but low
+disposal. For instance, `/data/media` could be stored on a large but low
performance hdd storage while other files could be stored on high performance
endpoints.
-In order to setup an application service, simply create an ``appservices``
+In order to setup an application service, simply create an `appservices`
directory in the data volume and write the application service Yaml
configuration file there. Multiple application services are supported.
@@ -56,6 +56,8 @@ The following environment variables are supported in `generate` mode:
* `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
* `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
anonymous statistics reporting.
+* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
+ Defaults to `8008`.
* `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
and event signing key) will be stored. Defaults to `/data`.
* `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
@@ -76,6 +78,8 @@ docker run -d --name synapse \
matrixdotorg/synapse:latest
```
+(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
+
You can then check that it has started correctly with:
```
@@ -211,4 +215,4 @@ healthcheck:
## Using jemalloc
Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.md)
+You can read about jemalloc by reading the Synapse [README](../README.md).
diff --git a/docker/conf-workers/nginx.conf.j2 b/docker/conf-workers/nginx.conf.j2
new file mode 100644
index 0000000000..1081979e06
--- /dev/null
+++ b/docker/conf-workers/nginx.conf.j2
@@ -0,0 +1,27 @@
+# This file contains the base config for the reverse proxy, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+{{ upstream_directives }}
+
+server {
+ # Listen on an unoccupied port number
+ listen 8008;
+ listen [::]:8008;
+
+ server_name localhost;
+
+ # Nginx by default only allows file uploads up to 1M in size
+ # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+ client_max_body_size 100M;
+
+{{ worker_locations }}
+
+ # Send all other traffic to the main process
+ location ~* ^(\\/_matrix|\\/_synapse) {
+ proxy_pass http://localhost:8080;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Host $host;
+ }
+}
diff --git a/docker/conf-workers/shared.yaml.j2 b/docker/conf-workers/shared.yaml.j2
new file mode 100644
index 0000000000..f94b8c6aca
--- /dev/null
+++ b/docker/conf-workers/shared.yaml.j2
@@ -0,0 +1,9 @@
+# This file contains the base for the shared homeserver config file between Synapse workers,
+# as part of ./Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+redis:
+ enabled: true
+
+{{ shared_worker_config }}
\ No newline at end of file
diff --git a/docker/conf-workers/supervisord.conf.j2 b/docker/conf-workers/supervisord.conf.j2
new file mode 100644
index 0000000000..0de2c6143b
--- /dev/null
+++ b/docker/conf-workers/supervisord.conf.j2
@@ -0,0 +1,41 @@
+# This file contains the base config for supervisord, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+[supervisord]
+nodaemon=true
+user=root
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+priority=500
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=www-data
+autorestart=true
+
+[program:redis]
+command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
+priority=1
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=redis
+autorestart=true
+
+[program:synapse_main]
+command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
+priority=10
+# Log startup failures to supervisord's stdout/err
+# Regular synapse logs will still go in the configured data directory
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+# Additional process blocks
+{{ worker_config }}
\ No newline at end of file
diff --git a/docker/conf-workers/worker.yaml.j2 b/docker/conf-workers/worker.yaml.j2
new file mode 100644
index 0000000000..42131afc95
--- /dev/null
+++ b/docker/conf-workers/worker.yaml.j2
@@ -0,0 +1,26 @@
+# This is a configuration template for a single worker instance, and is
+# used by Dockerfile-workers.
+# Values will be change depending on whichever workers are selected when
+# running that image.
+
+worker_app: "{{ app }}"
+worker_name: "{{ name }}"
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+ - type: http
+ port: {{ port }}
+{% if listener_resources %}
+ resources:
+ - names:
+{%- for resource in listener_resources %}
+ - {{ resource }}
+{%- endfor %}
+{% endif %}
+
+worker_log_config: {{ worker_log_config_filepath }}
+
+{{ worker_extra_conf }}
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index a792899540..2b23d7f428 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -40,7 +40,9 @@ listeners:
compress: false
{% endif %}
- - port: 8008
+ # Allow configuring in case we want to reverse proxy 8008
+ # using another process in the same container
+ - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
tls: false
bind_addresses: ['::']
type: http
diff --git a/docker/conf/log.config b/docker/conf/log.config
index 491bbcc87a..34572bc0f3 100644
--- a/docker/conf/log.config
+++ b/docker/conf/log.config
@@ -2,9 +2,34 @@ version: 1
formatters:
precise:
- format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% if worker_name %}
+ format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% else %}
+ format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% endif %}
handlers:
+ file:
+ class: logging.handlers.TimedRotatingFileHandler
+ formatter: precise
+ filename: {{ LOG_FILE_PATH or "homeserver.log" }}
+ when: "midnight"
+ backupCount: 6 # Does not include the current log file.
+ encoding: utf8
+
+ # Default to buffering writes to log file for efficiency. This means that
+ # there will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
+ # logs will still be flushed immediately.
+ buffer:
+ class: logging.handlers.MemoryHandler
+ target: file
+ # The capacity is the number of log lines that are buffered before
+ # being written to disk. Increasing this will lead to better
+ # performance, at the expensive of it taking longer for log lines to
+ # be written to disk.
+ capacity: 10
+ flushLevel: 30 # Flush for WARNING logs as well
+
console:
class: logging.StreamHandler
formatter: precise
@@ -17,6 +42,11 @@ loggers:
root:
level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
+
+{% if LOG_FILE_PATH %}
+ handlers: [console, buffer]
+{% else %}
handlers: [console]
+{% endif %}
disable_existing_loggers: false
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
new file mode 100755
index 0000000000..4be6afc65d
--- /dev/null
+++ b/docker/configure_workers_and_start.py
@@ -0,0 +1,558 @@
+#!/usr/bin/env python
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script reads environment variables and generates a shared Synapse worker,
+# nginx and supervisord configs depending on the workers requested.
+#
+# The environment variables it reads are:
+# * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
+# * SYNAPSE_REPORT_STATS: Whether to report stats.
+# * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
+# below. Leave empty for no workers, or set to '*' for all possible workers.
+#
+# NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
+# in the project's README), this script may be run multiple times, and functionality should
+# continue to work if so.
+
+import os
+import subprocess
+import sys
+
+import jinja2
+import yaml
+
+MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
+
+
+WORKERS_CONFIG = {
+ "pusher": {
+ "app": "synapse.app.pusher",
+ "listener_resources": [],
+ "endpoint_patterns": [],
+ "shared_extra_conf": {"start_pushers": False},
+ "worker_extra_conf": "",
+ },
+ "user_dir": {
+ "app": "synapse.app.user_dir",
+ "listener_resources": ["client"],
+ "endpoint_patterns": [
+ "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
+ ],
+ "shared_extra_conf": {"update_user_directory": False},
+ "worker_extra_conf": "",
+ },
+ "media_repository": {
+ "app": "synapse.app.media_repository",
+ "listener_resources": ["media"],
+ "endpoint_patterns": [
+ "^/_matrix/media/",
+ "^/_synapse/admin/v1/purge_media_cache$",
+ "^/_synapse/admin/v1/room/.*/media.*$",
+ "^/_synapse/admin/v1/user/.*/media.*$",
+ "^/_synapse/admin/v1/media/.*$",
+ "^/_synapse/admin/v1/quarantine_media/.*$",
+ ],
+ "shared_extra_conf": {"enable_media_repo": False},
+ "worker_extra_conf": "enable_media_repo: true",
+ },
+ "appservice": {
+ "app": "synapse.app.appservice",
+ "listener_resources": [],
+ "endpoint_patterns": [],
+ "shared_extra_conf": {"notify_appservices": False},
+ "worker_extra_conf": "",
+ },
+ "federation_sender": {
+ "app": "synapse.app.federation_sender",
+ "listener_resources": [],
+ "endpoint_patterns": [],
+ "shared_extra_conf": {"send_federation": False},
+ "worker_extra_conf": "",
+ },
+ "synchrotron": {
+ "app": "synapse.app.generic_worker",
+ "listener_resources": ["client"],
+ "endpoint_patterns": [
+ "^/_matrix/client/(v2_alpha|r0)/sync$",
+ "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
+ "^/_matrix/client/(api/v1|r0)/initialSync$",
+ "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
+ ],
+ "shared_extra_conf": {},
+ "worker_extra_conf": "",
+ },
+ "federation_reader": {
+ "app": "synapse.app.generic_worker",
+ "listener_resources": ["federation"],
+ "endpoint_patterns": [
+ "^/_matrix/federation/(v1|v2)/event/",
+ "^/_matrix/federation/(v1|v2)/state/",
+ "^/_matrix/federation/(v1|v2)/state_ids/",
+ "^/_matrix/federation/(v1|v2)/backfill/",
+ "^/_matrix/federation/(v1|v2)/get_missing_events/",
+ "^/_matrix/federation/(v1|v2)/publicRooms",
+ "^/_matrix/federation/(v1|v2)/query/",
+ "^/_matrix/federation/(v1|v2)/make_join/",
+ "^/_matrix/federation/(v1|v2)/make_leave/",
+ "^/_matrix/federation/(v1|v2)/send_join/",
+ "^/_matrix/federation/(v1|v2)/send_leave/",
+ "^/_matrix/federation/(v1|v2)/invite/",
+ "^/_matrix/federation/(v1|v2)/query_auth/",
+ "^/_matrix/federation/(v1|v2)/event_auth/",
+ "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
+ "^/_matrix/federation/(v1|v2)/user/devices/",
+ "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
+ "^/_matrix/key/v2/query",
+ ],
+ "shared_extra_conf": {},
+ "worker_extra_conf": "",
+ },
+ "federation_inbound": {
+ "app": "synapse.app.generic_worker",
+ "listener_resources": ["federation"],
+ "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
+ "shared_extra_conf": {},
+ "worker_extra_conf": "",
+ },
+ "event_persister": {
+ "app": "synapse.app.generic_worker",
+ "listener_resources": ["replication"],
+ "endpoint_patterns": [],
+ "shared_extra_conf": {},
+ "worker_extra_conf": "",
+ },
+ "background_worker": {
+ "app": "synapse.app.generic_worker",
+ "listener_resources": [],
+ "endpoint_patterns": [],
+ # This worker cannot be sharded. Therefore there should only ever be one background
+ # worker, and it should be named background_worker1
+ "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
+ "worker_extra_conf": "",
+ },
+ "event_creator": {
+ "app": "synapse.app.generic_worker",
+ "listener_resources": ["client"],
+ "endpoint_patterns": [
+ "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
+ "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
+ "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+ "^/_matrix/client/(api/v1|r0|unstable)/join/",
+ "^/_matrix/client/(api/v1|r0|unstable)/profile/",
+ ],
+ "shared_extra_conf": {},
+ "worker_extra_conf": "",
+ },
+ "frontend_proxy": {
+ "app": "synapse.app.frontend_proxy",
+ "listener_resources": ["client", "replication"],
+ "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
+ "shared_extra_conf": {},
+ "worker_extra_conf": (
+ "worker_main_http_uri: http://127.0.0.1:%d"
+ % (MAIN_PROCESS_HTTP_LISTENER_PORT,),
+ ),
+ },
+}
+
+# Templates for sections that may be inserted multiple times in config files
+SUPERVISORD_PROCESS_CONFIG_BLOCK = """
+[program:synapse_{name}]
+command=/usr/local/bin/python -m {app} \
+ --config-path="{config_path}" \
+ --config-path=/conf/workers/shared.yaml \
+ --config-path=/conf/workers/{name}.yaml
+autorestart=unexpected
+priority=500
+exitcodes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+"""
+
+NGINX_LOCATION_CONFIG_BLOCK = """
+ location ~* {endpoint} {
+ proxy_pass {upstream};
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Host $host;
+ }
+"""
+
+NGINX_UPSTREAM_CONFIG_BLOCK = """
+upstream {upstream_worker_type} {
+{body}
+}
+"""
+
+
+# Utility functions
+def log(txt: str):
+ """Log something to the stdout.
+
+ Args:
+ txt: The text to log.
+ """
+ print(txt)
+
+
+def error(txt: str):
+ """Log something and exit with an error code.
+
+ Args:
+ txt: The text to log in error.
+ """
+ log(txt)
+ sys.exit(2)
+
+
+def convert(src: str, dst: str, **template_vars):
+ """Generate a file from a template
+
+ Args:
+ src: Path to the input file.
+ dst: Path to write to.
+ template_vars: The arguments to replace placeholder variables in the template with.
+ """
+ # Read the template file
+ with open(src) as infile:
+ template = infile.read()
+
+ # Generate a string from the template. We disable autoescape to prevent template
+ # variables from being escaped.
+ rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
+
+ # Write the generated contents to a file
+ #
+ # We use append mode in case the files have already been written to by something else
+ # (for instance, as part of the instructions in a dockerfile).
+ with open(dst, "a") as outfile:
+ # In case the existing file doesn't end with a newline
+ outfile.write("\n")
+
+ outfile.write(rendered)
+
+
+def add_sharding_to_shared_config(
+ shared_config: dict,
+ worker_type: str,
+ worker_name: str,
+ worker_port: int,
+) -> None:
+ """Given a dictionary representing a config file shared across all workers,
+ append sharded worker information to it for the current worker_type instance.
+
+ Args:
+ shared_config: The config dict that all worker instances share (after being converted to YAML)
+ worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
+ worker_name: The name of the worker instance.
+ worker_port: The HTTP replication port that the worker instance is listening on.
+ """
+ # The instance_map config field marks the workers that write to various replication streams
+ instance_map = shared_config.setdefault("instance_map", {})
+
+ # Worker-type specific sharding config
+ if worker_type == "pusher":
+ shared_config.setdefault("pusher_instances", []).append(worker_name)
+
+ elif worker_type == "federation_sender":
+ shared_config.setdefault("federation_sender_instances", []).append(worker_name)
+
+ elif worker_type == "event_persister":
+ # Event persisters write to the events stream, so we need to update
+ # the list of event stream writers
+ shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
+ worker_name
+ )
+
+ # Map of stream writer instance names to host/ports combos
+ instance_map[worker_name] = {
+ "host": "localhost",
+ "port": worker_port,
+ }
+
+ elif worker_type == "media_repository":
+ # The first configured media worker will run the media background jobs
+ shared_config.setdefault("media_instance_running_background_jobs", worker_name)
+
+
+def generate_base_homeserver_config():
+ """Starts Synapse and generates a basic homeserver config, which will later be
+ modified for worker support.
+
+ Raises: CalledProcessError if calling start.py returned a non-zero exit code.
+ """
+ # start.py already does this for us, so just call that.
+ # note that this script is copied in in the official, monolith dockerfile
+ os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
+ subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
+
+
+def generate_worker_files(environ, config_path: str, data_dir: str):
+ """Read the desired list of workers from environment variables and generate
+ shared homeserver, nginx and supervisord configs.
+
+ Args:
+ environ: _Environ[str]
+ config_path: Where to output the generated Synapse main worker config file.
+ data_dir: The location of the synapse data directory. Where log and
+ user-facing config files live.
+ """
+ # Note that yaml cares about indentation, so care should be taken to insert lines
+ # into files at the correct indentation below.
+
+ # shared_config is the contents of a Synapse config file that will be shared amongst
+ # the main Synapse process as well as all workers.
+ # It is intended mainly for disabling functionality when certain workers are spun up,
+ # and adding a replication listener.
+
+ # First read the original config file and extract the listeners block. Then we'll add
+ # another listener for replication. Later we'll write out the result.
+ listeners = [
+ {
+ "port": 9093,
+ "bind_address": "127.0.0.1",
+ "type": "http",
+ "resources": [{"names": ["replication"]}],
+ }
+ ]
+ with open(config_path) as file_stream:
+ original_config = yaml.safe_load(file_stream)
+ original_listeners = original_config.get("listeners")
+ if original_listeners:
+ listeners += original_listeners
+
+ # The shared homeserver config. The contents of which will be inserted into the
+ # base shared worker jinja2 template.
+ #
+ # This config file will be passed to all workers, included Synapse's main process.
+ shared_config = {"listeners": listeners}
+
+ # The supervisord config. The contents of which will be inserted into the
+ # base supervisord jinja2 template.
+ #
+ # Supervisord will be in charge of running everything, from redis to nginx to Synapse
+ # and all of its worker processes. Load the config template, which defines a few
+ # services that are necessary to run.
+ supervisord_config = ""
+
+ # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
+ # ports of each worker. For example:
+ # {
+ # worker_type: {1234, 1235, ...}}
+ # }
+ # and will be used to construct 'upstream' nginx directives.
+ nginx_upstreams = {}
+
+ # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
+ # placed after the proxy_pass directive. The main benefit to representing this data as a
+ # dict over a str is that we can easily deduplicate endpoints across multiple instances
+ # of the same worker.
+ #
+ # An nginx site config that will be amended to depending on the workers that are
+ # spun up. To be placed in /etc/nginx/conf.d.
+ nginx_locations = {}
+
+ # Read the desired worker configuration from the environment
+ worker_types = environ.get("SYNAPSE_WORKER_TYPES")
+ if worker_types is None:
+ # No workers, just the main process
+ worker_types = []
+ else:
+ # Split type names by comma
+ worker_types = worker_types.split(",")
+
+ # Create the worker configuration directory if it doesn't already exist
+ os.makedirs("/conf/workers", exist_ok=True)
+
+ # Start worker ports from this arbitrary port
+ worker_port = 18009
+
+ # A counter of worker_type -> int. Used for determining the name for a given
+ # worker type when generating its config file, as each worker's name is just
+ # worker_type + instance #
+ worker_type_counter = {}
+
+ # For each worker type specified by the user, create config values
+ for worker_type in worker_types:
+ worker_type = worker_type.strip()
+
+ worker_config = WORKERS_CONFIG.get(worker_type)
+ if worker_config:
+ worker_config = worker_config.copy()
+ else:
+ log(worker_type + " is an unknown worker type! It will be ignored")
+ continue
+
+ new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
+ worker_type_counter[worker_type] = new_worker_count
+
+ # Name workers by their type concatenated with an incrementing number
+ # e.g. federation_reader1
+ worker_name = worker_type + str(new_worker_count)
+ worker_config.update(
+ {"name": worker_name, "port": worker_port, "config_path": config_path}
+ )
+
+ # Update the shared config with any worker-type specific options
+ shared_config.update(worker_config["shared_extra_conf"])
+
+ # Check if more than one instance of this worker type has been specified
+ worker_type_total_count = worker_types.count(worker_type)
+ if worker_type_total_count > 1:
+ # Update the shared config with sharding-related options if necessary
+ add_sharding_to_shared_config(
+ shared_config, worker_type, worker_name, worker_port
+ )
+
+ # Enable the worker in supervisord
+ supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
+
+ # Add nginx location blocks for this worker's endpoints (if any are defined)
+ for pattern in worker_config["endpoint_patterns"]:
+ # Determine whether we need to load-balance this worker
+ if worker_type_total_count > 1:
+ # Create or add to a load-balanced upstream for this worker
+ nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
+
+ # Upstreams are named after the worker_type
+ upstream = "http://" + worker_type
+ else:
+ upstream = "http://localhost:%d" % (worker_port,)
+
+ # Note that this endpoint should proxy to this upstream
+ nginx_locations[pattern] = upstream
+
+ # Write out the worker's logging config file
+
+ # Check whether we should write worker logs to disk, in addition to the console
+ extra_log_template_args = {}
+ if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+ extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+ dir=data_dir, name=worker_name
+ )
+
+ # Render and write the file
+ log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+ convert(
+ "/conf/log.config",
+ log_config_filepath,
+ worker_name=worker_name,
+ **extra_log_template_args,
+ )
+
+ # Then a worker config file
+ convert(
+ "/conf/worker.yaml.j2",
+ "/conf/workers/{name}.yaml".format(name=worker_name),
+ **worker_config,
+ worker_log_config_filepath=log_config_filepath,
+ )
+
+ worker_port += 1
+
+ # Build the nginx location config blocks
+ nginx_location_config = ""
+ for endpoint, upstream in nginx_locations.items():
+ nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(
+ endpoint=endpoint,
+ upstream=upstream,
+ )
+
+ # Determine the load-balancing upstreams to configure
+ nginx_upstream_config = ""
+ for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
+ body = ""
+ for port in upstream_worker_ports:
+ body += " server localhost:%d;\n" % (port,)
+
+ # Add to the list of configured upstreams
+ nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
+ upstream_worker_type=upstream_worker_type,
+ body=body,
+ )
+
+ # Finally, we'll write out the config files.
+
+ # Shared homeserver config
+ convert(
+ "/conf/shared.yaml.j2",
+ "/conf/workers/shared.yaml",
+ shared_worker_config=yaml.dump(shared_config),
+ )
+
+ # Nginx config
+ convert(
+ "/conf/nginx.conf.j2",
+ "/etc/nginx/conf.d/matrix-synapse.conf",
+ worker_locations=nginx_location_config,
+ upstream_directives=nginx_upstream_config,
+ )
+
+ # Supervisord config
+ convert(
+ "/conf/supervisord.conf.j2",
+ "/etc/supervisor/conf.d/supervisord.conf",
+ main_config_path=config_path,
+ worker_config=supervisord_config,
+ )
+
+ # Ensure the logging directory exists
+ log_dir = data_dir + "/logs"
+ if not os.path.exists(log_dir):
+ os.mkdir(log_dir)
+
+
+def start_supervisord():
+ """Starts up supervisord which then starts and monitors all other necessary processes
+
+ Raises: CalledProcessError if calling start.py return a non-zero exit code.
+ """
+ subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+
+
+def main(args, environ):
+ config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+ config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+ data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
+
+ # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,
+ # this needs to be handled by a frontend proxy
+ environ["SYNAPSE_NO_TLS"] = "yes"
+
+ # Generate the base homeserver config if one does not yet exist
+ if not os.path.exists(config_path):
+ log("Generating base homeserver config")
+ generate_base_homeserver_config()
+
+ # This script may be run multiple times (mostly by Complement, see note at top of file).
+ # Don't re-configure workers in this instance.
+ mark_filepath = "/conf/workers_have_been_configured"
+ if not os.path.exists(mark_filepath):
+ # Always regenerate all other config files
+ generate_worker_files(environ, config_path, data_dir)
+
+ # Mark workers as being configured
+ with open(mark_filepath, "w") as f:
+ f.write("")
+
+ # Start supervisord, which will start Synapse, all of the configured worker
+ # processes, redis, nginx etc. according to the config we created above.
+ start_supervisord()
+
+
+if __name__ == "__main__":
+ main(sys.argv, os.environ)
|
