major wip

7 files changed, 243 insertions, 3 deletions

diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
new file mode 100644
index 0000000000..88b2a5262f
--- /dev/null
+++ b/docker/Dockerfile-workers
@@ -0,0 +1,25 @@
+# Inherit from the official Synapse docker image
+FROM matrixdotorg/synapse
+
+# Install deps
+RUN apt-get update
+RUN apt-get install -y supervisor redis nginx
+
+# A script to read environment variables and create the necessary
+# files to run the desired worker configuration
+COPY ./docker/create_worker_config_files.py /create_worker_config_files.py
+RUN /create_worker_config_files.py
+
+# Create a volume for logging. The official Synapse docker image
+# only logs to console, however this is inconvenient for multi-process
+# containers.
+VOLUME ["/logs"]
+
+# Expose Synapse client, ACME challenge and federation ports
+EXPOSE 8008/tcp 8009/tcp 8448/tcp
+
+# Start supervisord
+COPY ./docker/worker_conf/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+ENTRYPOINT ["/usr/bin/supervisord"]
+
+# TODO: Healthcheck? Can we ask supervisord?
\ No newline at end of file
diff --git a/docker/MoveToComplement.Dockerfile b/docker/MoveToComplement.Dockerfile
new file mode 100644
index 0000000000..c91000bc7f
--- /dev/null
+++ b/docker/MoveToComplement.Dockerfile
@@ -0,0 +1,35 @@
+# This dockerfile builds on top of Dockerfile-worker and includes a built-in postgres instance.
+# It is intended to be used for Complement testing
+
+FROM matrixdotorg/synapse:workers
+
+# Install postgres
+RUN apt-get update
+RUN apt-get install -y postgres
+
+# Create required databases in postgres
+
+# Create a user without a password
+RUN sudo -u postgres createuser -w synapse_user
+
+# Then set their password
+RUN sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'somesecret';"
+
+# Create the synapse database
+RUN sudo -u postgres psql -c "CREATE DATABASE synapse \
+ ENCODING 'UTF8' \
+ LC_COLLATE='C' \
+ LC_CTYPE='C' \
+ template=template0 \
+ OWNER synapse_user;"
+
+# Modify Synapse's database config to point to the local postgres
+COPY ./docker/synapse_use_local_postgres.py /synapse_use_local_postgres.py
+RUN /synapse_use_local_postgres.py
+
+VOLUME ["/data"]
+
+EXPOSE 8008/tcp 8009/tcp 8448/tcp
+
+# Start supervisord
+CMD ["/usr/bin/supervisord"]
\ No newline at end of file
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index a808485c12..8434fc69f3 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -27,8 +27,7 @@ log_config: "{{ SYNAPSE_LOG_CONFIG }}"
 
 listeners:
   {% if not SYNAPSE_NO_TLS %}
-  -
-    port: 8448
+  - port: 8448
     bind_addresses: ['::']
     type: http
     tls: true
@@ -52,6 +51,15 @@ listeners:
       - names: [federation]
         compress: false
 
+  {% if SYNAPSE_WORKERS %}
+  # The HTTP replication port
+  - port: 9093
+    bind_address: '127.0.0.1'
+    type: http
+    resources:
+      - names: [replication]
+  {% endif %}
+
 ## Database ##
 
 {% if POSTGRES_PASSWORD %}
diff --git a/docker/generate_shared_worker_config.py b/docker/generate_shared_worker_config.py
new file mode 100644
index 0000000000..755a737344
--- /dev/null
+++ b/docker/generate_shared_worker_config.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script reads environment variables and generates a shared Synapse worker,
+# nginx and supervisord configs depending on the workers requested
+
+import os
+import sys
+
+
+def main(args, environ):
+    """Read the desired list of workers from environment variables and generate
+    shared homeserver, nginx and supervisord configs.
+
+    Args:
+        environ: _Environ[str]
+    """
+    # The contents of this string will be appended to the one generated by
+    # the homeserver, and is intended mainly for disabling functionality on the main
+    # when certain workers are spun up
+    homeserver_config = ""
+
+    # The contents of this string will be append to the base supervisord config
+    supervisord_config = ""
+
+    # An nginx site config. Will live in /etc/nginx/conf.d
+    nginx_config_template_header = """
+    server {
+        listen 80;
+        listen [::]:80;
+
+        # For the federation port
+        listen 8448 default_server;
+        listen [::]:8448 default_server;
+
+        server_name localhost;
+    """
+    nginx_config_body = ""
+    nginx_config_template_end = """
+        # Send all other traffic to the main process
+        location ~* ^(\/_matrix|\/_synapse) {
+            proxy_pass http://localhost:8008;
+            proxy_set_header X-Forwarded-For $remote_addr;
+
+            # TODO: Can we move this to the default nginx.conf so all locations are
+            # affected?
+            #
+            # Nginx by default only allows file uploads up to 1M in size
+            # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+            client_max_body_size 50M;
+        }
+    }
+    """
+
+    # Read desired worker configuration from environment
+    worker_types = environ.get("SYNAPSE_WORKERS")
+    worker_types = worker_types.split(",")
+
+    for worker_type in worker_types:
+        if worker_type == "pusher":
+            # Disable push handling from the main process
+            homeserver_config += """
+            start_pushers: false
+            """
+
+            # Enable the pusher worker in supervisord
+            supervisord_config += """
+            """
+
+            # This worker does not handle any REST endpoints
+
+        elif worker_type == "appservice":
+            # Disable appservice traffic sending from the main process
+            homeserver_config += """
+            notify_appservices: false
+            """
+
+            # Enable the pusher worker in supervisord
+            supervisord_config += """
+            [program:synapse_user_dir]
+            command=/usr/local/bin/python -m synapse.app.user_dir \
+                --config-path=/config/homeserver.yaml \
+                --config-path=/config/workers/user_dir.yaml
+            autorestart=unexpected
+            exitcodes=0
+            """
+
+            # This worker does not handle any REST endpoints
+
+        elif worker_type == "user_dir":
+            # Disable user directory updates on the main process
+            homeserver_config += """
+            update_user_directory: false
+            """
+
+            # Enable the user directory worker in supervisord
+            supervisord_config += """
+            [program:synapse_user_dir]
+            command=/usr/local/bin/python -m synapse.app.user_dir \
+                --config-path=/config/homeserver.yaml \
+                --config-path=/config/workers/user_dir.yaml
+            autorestart=unexpected
+            exitcodes=0
+            """
+
+            # Route user directory requests to this worker
+            nginx_config_body += """
+            location ~* ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
+                proxy_pass http://localhost:8010;
+                proxy_set_header X-Forwarded-For $remote_addr;
+            }
+            """
+
+    # Write out the config files
+
+    # Main homeserver config
+    with open("/config/main.yaml", "a") as f:
+        f.write(homeserver_config)
+
+    # Nginx config
+    with open("/config/nginx.conf", "w") as f:
+        f.write(nginx_config_template_header)
+        f.write(nginx_config_body)
+        f.write(nginx_config_template_end)
+
+    # Supervisord config
+    with open("/config/supervisord.conf", "a") as f:
+        f.write(supervisord_config)
+
+
+if __name__ == "__main__":
+    main(sys.argv, os.environ)
diff --git a/docker/start.py b/docker/start.py
index 0d2c590b88..c3a6d96861 100755
--- a/docker/start.py
+++ b/docker/start.py
@@ -177,7 +177,6 @@ def run_generate_config(environ, ownership):
     else:
         os.execv("/usr/local/bin/python", args)
 
-
 def main(args, environ):
     mode = args[1] if len(args) > 1 else "run"
     desired_uid = int(environ.get("UID", "991"))
diff --git a/docker/worker_conf/main.conf b/docker/worker_conf/main.conf
new file mode 100644
index 0000000000..917b82500d
--- /dev/null
+++ b/docker/worker_conf/main.conf
@@ -0,0 +1,9 @@
+# A bit of Synapse config file that will be appended to the main homeserver config file.
+# It is intended for generate_shared_worker_config.py to add entries to this file to
+# disable functionality as equivalent workers are spun up.
+
+# TODO: extend the existing `listeners` section. This defines the ports that the
+# main process will listen on.
+
+redis:
+    enabled: true
diff --git a/docker/worker_conf/supervisord.conf b/docker/worker_conf/supervisord.conf
new file mode 100644
index 0000000000..63c898e75b
--- /dev/null
+++ b/docker/worker_conf/supervisord.conf
@@ -0,0 +1,19 @@
+[supervisord]
+nodaemon=true
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+priority=900
+stdout_logfile= /dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=www-data
+autorestart=true
+
+[program:synapse_main]
+command=/usr/local/bin/python -m synapse.app.homeserver \
+    --config-path=/config/homeserver.yaml \
+    --config-path=/config/main.yaml
+autorestart=unexpected
+exitcodes=0