diff --git a/contrib/docker_compose_workers/README.md b/contrib/docker_compose_workers/README.md
new file mode 100644
index 0000000000..4dbfee2853
--- /dev/null
+++ b/contrib/docker_compose_workers/README.md
@@ -0,0 +1,125 @@
+# Setting up Synapse with Workers using Docker Compose
+
+This directory describes how deploy and manage Synapse and workers via [Docker Compose](https://docs.docker.com/compose/).
+
+Example worker configuration files can be found [here](workers).
+
+All examples and snippets assume that your Synapse service is called `synapse` in your Docker Compose file.
+
+An example Docker Compose file can be found [here](docker-compose.yaml).
+
+## Worker Service Examples in Docker Compose
+
+In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`).
+
+### Generic Worker Example
+
+```yaml
+synapse-generic-worker-1:
+ image: matrixdotorg/synapse:latest
+ container_name: synapse-generic-worker-1
+ restart: unless-stopped
+ entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-generic-worker-1.yaml"]
+ healthcheck:
+ test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"]
+ start_period: "5s"
+ interval: "15s"
+ timeout: "5s"
+ volumes:
+ - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+ environment:
+ SYNAPSE_WORKER: synapse.app.generic_worker
+ # Expose port if required so your reverse proxy can send requests to this worker
+ # Port configuration will depend on how the http listener is defined in the worker configuration file
+ ports:
+ - 8081:8081
+ depends_on:
+ - synapse
+```
+
+### Federation Sender Example
+
+Please note: The federation sender does not receive REST API calls so no exposed ports are required.
+
+```yaml
+synapse-federation-sender-1:
+ image: matrixdotorg/synapse:latest
+ container_name: synapse-federation-sender-1
+ restart: unless-stopped
+ entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-federation-sender-1.yaml"]
+ healthcheck:
+ disable: true
+ volumes:
+ - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+ environment:
+ SYNAPSE_WORKER: synapse.app.federation_sender
+ depends_on:
+ - synapse
+```
+
+## `homeserver.yaml` Configuration
+
+### Enable Redis
+
+Locate the `redis` section of your `homeserver.yaml` and enable and configure it:
+
+```yaml
+redis:
+ enabled: true
+ host: redis
+ port: 6379
+ # password: <secret_password>
+```
+
+This assumes that your Redis service is called `redis` in your Docker Compose file.
+
+### Add a replication Listener
+
+Locate the `listeners` section of your `homeserver.yaml` and add the following replication listener:
+
+```yaml
+listeners:
+ # Other listeners
+
+ - port: 9093
+ type: http
+ resources:
+ - names: [replication]
+```
+
+This listener is used by the workers for replication and is referred to in worker config files using the following settings:
+
+```yaml
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+```
+
+### Add Workers to `instance_map`
+
+Locate the `instance_map` section of your `homeserver.yaml` and populate it with your workers:
+
+```yaml
+instance_map:
+ synapse-generic-worker-1: # The worker_name setting in your worker configuration file
+ host: synapse-generic-worker-1 # The name of the worker service in your Docker Compose file
+ port: 8034 # The port assigned to the replication listener in your worker config file
+ synapse-federation-sender-1:
+ host: synapse-federation-sender-1
+ port: 8034
+```
+
+### Configure Federation Senders
+
+This section is applicable if you are using Federation senders (synapse.app.federation_sender). Locate the `send_federation` and `federation_sender_instances` settings in your `homeserver.yaml` and configure them:
+
+```yaml
+# This will disable federation sending on the main Synapse instance
+send_federation: false
+
+federation_sender_instances:
+ - synapse-federation-sender-1 # The worker_name setting in your federation sender worker configuration file
+```
+
+## Other Worker types
+
+Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers.
\ No newline at end of file
diff --git a/contrib/docker_compose_workers/docker-compose.yaml b/contrib/docker_compose_workers/docker-compose.yaml
new file mode 100644
index 0000000000..eaf02c2af9
--- /dev/null
+++ b/contrib/docker_compose_workers/docker-compose.yaml
@@ -0,0 +1,77 @@
+networks:
+ backend:
+
+services:
+ postgres:
+ image: postgres:latest
+ restart: unless-stopped
+ volumes:
+ - ${VOLUME_PATH}/var/lib/postgresql/data:/var/lib/postgresql/data:rw
+ networks:
+ - backend
+ environment:
+ POSTGRES_DB: synapse
+ POSTGRES_USER: synapse_user
+ POSTGRES_PASSWORD: postgres
+ POSTGRES_INITDB_ARGS: --encoding=UTF8 --locale=C
+
+ redis:
+ image: redis:latest
+ restart: unless-stopped
+ networks:
+ - backend
+
+ synapse:
+ image: matrixdotorg/synapse:latest
+ container_name: synapse
+ restart: unless-stopped
+ volumes:
+ - ${VOLUME_PATH}/data:/data:rw
+ ports:
+ - 8008:8008
+ networks:
+ - backend
+ environment:
+ SYNAPSE_CONFIG_DIR: /data
+ SYNAPSE_CONFIG_PATH: /data/homeserver.yaml
+ depends_on:
+ - postgres
+
+ synapse-generic-worker-1:
+ image: matrixdotorg/synapse:latest
+ container_name: synapse-generic-worker-1
+ restart: unless-stopped
+ entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-generic-worker-1.yaml"]
+ healthcheck:
+ test: ["CMD-SHELL", "curl -fSs http://localhost:8081/health || exit 1"]
+ start_period: "5s"
+ interval: "15s"
+ timeout: "5s"
+ networks:
+ - backend
+ volumes:
+ - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+ environment:
+ SYNAPSE_WORKER: synapse.app.generic_worker
+ # Expose port if required so your reverse proxy can send requests to this worker
+ # Port configuration will depend on how the http listener is defined in the worker configuration file
+ ports:
+ - 8081:8081
+ depends_on:
+ - synapse
+
+ synapse-federation-sender-1:
+ image: matrixdotorg/synapse:latest
+ container_name: synapse-federation-sender-1
+ restart: unless-stopped
+ entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/synapse-federation-sender-1.yaml"]
+ healthcheck:
+ disable: true
+ networks:
+ - backend
+ volumes:
+ - ${VOLUME_PATH}/data:/data:rw # Replace VOLUME_PATH with the path to your Synapse volume
+ environment:
+ SYNAPSE_WORKER: synapse.app.federation_sender
+ depends_on:
+ - synapse
diff --git a/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml b/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml
new file mode 100644
index 0000000000..5ba42a92d2
--- /dev/null
+++ b/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml
@@ -0,0 +1,14 @@
+worker_app: synapse.app.federation_sender
+worker_name: synapse-federation-sender-1
+
+# The replication listener on the main synapse process.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_listeners:
+ - type: http
+ port: 8034
+ resources:
+ - names: [replication]
+
+worker_log_config: /data/federation_sender.log.config
diff --git a/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml b/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml
new file mode 100644
index 0000000000..694584105a
--- /dev/null
+++ b/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml
@@ -0,0 +1,19 @@
+worker_app: synapse.app.generic_worker
+worker_name: synapse-generic-worker-1
+
+# The replication listener on the main synapse process.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_listeners:
+ - type: http
+ port: 8034
+ resources:
+ - names: [replication]
+ - type: http
+ port: 8081
+ x_forwarded: true
+ resources:
+ - names: [client, federation]
+
+worker_log_config: /data/worker.log.config
diff --git a/contrib/graph/graph.py b/contrib/graph/graph.py
index fdbac087bd..3c4f47dbd2 100644
--- a/contrib/graph/graph.py
+++ b/contrib/graph/graph.py
@@ -1,11 +1,3 @@
-import argparse
-import cgi
-import datetime
-import json
-
-import pydot
-import urllib2
-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,12 +12,25 @@ import urllib2
# See the License for the specific language governing permissions and
# limitations under the License.
+import argparse
+import cgi
+import datetime
+import json
+import urllib.request
+from typing import List
+
+import pydot
+
-def make_name(pdu_id, origin):
- return "%s@%s" % (pdu_id, origin)
+def make_name(pdu_id: str, origin: str) -> str:
+ return f"{pdu_id}@{origin}"
-def make_graph(pdus, room, filename_prefix):
+def make_graph(pdus: List[dict], filename_prefix: str) -> None:
+ """
+ Generate a dot and SVG file for a graph of events in the room based on the
+ topological ordering by querying a homeserver.
+ """
pdu_map = {}
node_map = {}
@@ -111,10 +116,10 @@ def make_graph(pdus, room, filename_prefix):
graph.write_svg("%s.svg" % filename_prefix, prog="dot")
-def get_pdus(host, room):
+def get_pdus(host: str, room: str) -> List[dict]:
transaction = json.loads(
- urllib2.urlopen(
- "http://%s/_matrix/federation/v1/context/%s/" % (host, room)
+ urllib.request.urlopen(
+ f"http://{host}/_matrix/federation/v1/context/{room}/"
).read()
)
@@ -141,4 +146,4 @@ if __name__ == "__main__":
pdus = get_pdus(host, room)
- make_graph(pdus, room, prefix)
+ make_graph(pdus, prefix)
diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py
index 0980231e4a..b46094ce0a 100644
--- a/contrib/graph/graph2.py
+++ b/contrib/graph/graph2.py
@@ -14,22 +14,31 @@
import argparse
-import cgi
import datetime
+import html
import json
import sqlite3
import pydot
-from synapse.events import FrozenEvent
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
+from synapse.events import make_event_from_dict
from synapse.util.frozenutils import unfreeze
-def make_graph(db_name, room_id, file_prefix, limit):
+def make_graph(db_name: str, room_id: str, file_prefix: str, limit: int) -> None:
+ """
+ Generate a dot and SVG file for a graph of events in the room based on the
+ topological ordering by reading from a Synapse SQLite database.
+ """
conn = sqlite3.connect(db_name)
+ sql = "SELECT room_version FROM rooms WHERE room_id = ?"
+ c = conn.execute(sql, (room_id,))
+ room_version = KNOWN_ROOM_VERSIONS[c.fetchone()[0]]
+
sql = (
- "SELECT json FROM event_json as j "
+ "SELECT json, internal_metadata FROM event_json as j "
"INNER JOIN events as e ON e.event_id = j.event_id "
"WHERE j.room_id = ?"
)
@@ -43,7 +52,10 @@ def make_graph(db_name, room_id, file_prefix, limit):
c = conn.execute(sql, args)
- events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()]
+ events = [
+ make_event_from_dict(json.loads(e[0]), room_version, json.loads(e[1]))
+ for e in c.fetchall()
+ ]
events.sort(key=lambda e: e.depth)
@@ -84,7 +96,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
"name": event.event_id,
"type": event.type,
"state_key": event.get("state_key", None),
- "content": cgi.escape(content, quote=True),
+ "content": html.escape(content, quote=True),
"time": t,
"depth": event.depth,
"state_group": state_group,
@@ -96,11 +108,11 @@ def make_graph(db_name, room_id, file_prefix, limit):
graph.add_node(node)
for event in events:
- for prev_id, _ in event.prev_events:
+ for prev_id in event.prev_event_ids():
try:
end_node = node_map[prev_id]
except Exception:
- end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
+ end_node = pydot.Node(name=prev_id, label=f"<<b>{prev_id}</b>>")
node_map[prev_id] = end_node
graph.add_node(end_node)
@@ -112,7 +124,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
if len(event_ids) <= 1:
continue
- cluster = pydot.Cluster(str(group), label="<State Group: %s>" % (str(group),))
+ cluster = pydot.Cluster(str(group), label=f"<State Group: {str(group)}>")
for event_id in event_ids:
cluster.add_node(node_map[event_id])
@@ -126,7 +138,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Generate a PDU graph for a given room by talking "
- "to the given homeserver to get the list of PDUs. \n"
+ "to the given Synapse SQLite file to get the list of PDUs. \n"
"Requires pydot."
)
parser.add_argument(
diff --git a/contrib/graph/graph3.py b/contrib/graph/graph3.py
index dd0c19368b..a28a1594c7 100644
--- a/contrib/graph/graph3.py
+++ b/contrib/graph/graph3.py
@@ -1,13 +1,3 @@
-import argparse
-import cgi
-import datetime
-
-import pydot
-import simplejson as json
-
-from synapse.events import FrozenEvent
-from synapse.util.frozenutils import unfreeze
-
# Copyright 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,15 +12,35 @@ from synapse.util.frozenutils import unfreeze
# See the License for the specific language governing permissions and
# limitations under the License.
+import argparse
+import datetime
+import html
+import json
+
+import pydot
-def make_graph(file_name, room_id, file_prefix, limit):
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
+from synapse.events import make_event_from_dict
+from synapse.util.frozenutils import unfreeze
+
+
+def make_graph(file_name: str, file_prefix: str, limit: int) -> None:
+ """
+ Generate a dot and SVG file for a graph of events in the room based on the
+ topological ordering by reading line-delimited JSON from a file.
+ """
print("Reading lines")
with open(file_name) as f:
lines = f.readlines()
print("Read lines")
- events = [FrozenEvent(json.loads(line)) for line in lines]
+ # Figure out the room version, assume the first line is the create event.
+ room_version = KNOWN_ROOM_VERSIONS[
+ json.loads(lines[0]).get("content", {}).get("room_version")
+ ]
+
+ events = [make_event_from_dict(json.loads(line), room_version) for line in lines]
print("Loaded events.")
@@ -66,8 +76,8 @@ def make_graph(file_name, room_id, file_prefix, limit):
content.append(
"<b>%s</b>: %s,"
% (
- cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
- cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
+ html.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
+ html.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
)
)
@@ -101,11 +111,11 @@ def make_graph(file_name, room_id, file_prefix, limit):
print("Created Nodes")
for event in events:
- for prev_id, _ in event.prev_events:
+ for prev_id in event.prev_event_ids():
try:
end_node = node_map[prev_id]
except Exception:
- end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
+ end_node = pydot.Node(name=prev_id, label=f"<<b>{prev_id}</b>>")
node_map[prev_id] = end_node
graph.add_node(end_node)
@@ -139,8 +149,7 @@ if __name__ == "__main__":
)
parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
parser.add_argument("event_file")
- parser.add_argument("room")
args = parser.parse_args()
- make_graph(args.event_file, args.room, args.prefix, args.limit)
+ make_graph(args.event_file, args.prefix, args.limit)
|
