diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/docker/README.md | 35 | ||||
| -rw-r--r-- | contrib/docker/docker-compose.yml | 23 | ||||
| -rw-r--r-- | contrib/experiments/test_messaging.py | 6 | ||||
| -rw-r--r-- | contrib/grafana/README.md | 2 | ||||
| -rw-r--r-- | contrib/grafana/synapse.json | 822 | ||||
| -rw-r--r-- | contrib/graph/graph2.py | 4 | ||||
| -rw-r--r-- | contrib/systemd-with-workers/README.md | 152 | ||||
| -rw-r--r-- | contrib/systemd-with-workers/system/matrix-synapse-worker@.service | 19 | ||||
| -rw-r--r-- | contrib/systemd-with-workers/system/matrix-synapse.service | 18 | ||||
| -rw-r--r-- | contrib/systemd-with-workers/system/matrix.target | 7 | ||||
| -rw-r--r-- | contrib/systemd-with-workers/workers/federation_reader.yaml | 14 | ||||
| -rw-r--r-- | contrib/systemd/README.md | 17 | ||||
| -rw-r--r-- | contrib/systemd/matrix-synapse.service | 12 |
13 files changed, 783 insertions, 348 deletions
diff --git a/contrib/docker/README.md b/contrib/docker/README.md index af102f7594..89c1518bd0 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,39 +1,26 @@ -# Synapse Docker - -FIXME: this is out-of-date as of -https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up -to date would be welcome. - -### Automated configuration - -It is recommended that you use Docker Compose to run your containers, including -this image and a Postgres server. A sample ``docker-compose.yml`` is provided, -including example labels for reverse proxying and other artifacts. - -Read the section about environment variables and set at least mandatory variables, -then run the server: - -``` -docker-compose up -d -``` -If secrets are not specified in the environment variables, they will be generated -as part of the startup. Please ensure these secrets are kept between launches of the -Docker container, as their loss may require users to log in again. +# Synapse Docker -### Manual configuration +### Configuration A sample ``docker-compose.yml`` is provided, including example labels for reverse proxying and other artifacts. The docker-compose file is an example, please comment/uncomment sections that are not suitable for your usecase. Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path, -to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run: +to use manual configuration. + +To generate a fresh `homeserver.yaml`, you can use the `generate` command. +(See the [documentation](../../docker/README.md#generating-a-configuration-file) +for more information.) You will need to specify appropriate values for at least the +`SYNAPSE_SERVER_NAME` and `SYNAPSE_REPORT_STATS` environment variables. For example: ``` -docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate +docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=yes synapse generate ``` +(This will also generate necessary signing keys.) + Then, customize your configuration and run the server: ``` diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1e4ee43758..17354b6610 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,11 +15,7 @@ services: restart: unless-stopped # See the readme for a full documentation of the environment settings environment: - - SYNAPSE_SERVER_NAME=my.matrix.host - - SYNAPSE_REPORT_STATS=no - - SYNAPSE_ENABLE_REGISTRATION=yes - - SYNAPSE_LOG_LEVEL=INFO - - POSTGRES_PASSWORD=changeme + - SYNAPSE_CONFIG_PATH=/data/homeserver.yaml volumes: # You may either store all the files in a local folder - ./files:/data @@ -35,9 +31,23 @@ services: - 8448:8448/tcp # ... or use a reverse proxy, here is an example for traefik: labels: + # The following lines are valid for Traefik version 1.x: - traefik.enable=true - traefik.frontend.rule=Host:my.matrix.Host - traefik.port=8008 + # Alternatively, for Traefik version 2.0: + - traefik.enable=true + - traefik.http.routers.http-synapse.entryPoints=http + - traefik.http.routers.http-synapse.rule=Host(`my.matrix.host`) + - traefik.http.middlewares.https_redirect.redirectscheme.scheme=https + - traefik.http.middlewares.https_redirect.redirectscheme.permanent=true + - traefik.http.routers.http-synapse.middlewares=https_redirect + - traefik.http.routers.https-synapse.entryPoints=https + - traefik.http.routers.https-synapse.rule=Host(`my.matrix.host`) + - traefik.http.routers.https-synapse.service=synapse + - traefik.http.routers.https-synapse.tls=true + - traefik.http.services.synapse.loadbalancer.server.port=8008 + - traefik.http.routers.https-synapse.tls.certResolver=le-ssl db: image: docker.io/postgres:10-alpine @@ -45,6 +55,9 @@ services: environment: - POSTGRES_USER=synapse - POSTGRES_PASSWORD=changeme + # ensure the database gets created correctly + # https://github.com/matrix-org/synapse/blob/master/docs/postgres.md#set-up-database + - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C volumes: # You may store the database tables in a local folder.. - ./schemas:/var/lib/postgresql/data diff --git a/contrib/experiments/test_messaging.py b/contrib/experiments/test_messaging.py index 5ef140ae48..3bbbcfa1b4 100644 --- a/contrib/experiments/test_messaging.py +++ b/contrib/experiments/test_messaging.py @@ -78,7 +78,7 @@ class InputOutput(object): m = re.match("^join (\S+)$", line) if m: # The `sender` wants to join a room. - room_name, = m.groups() + (room_name,) = m.groups() self.print_line("%s joining %s" % (self.user, room_name)) self.server.join_room(room_name, self.user, self.user) # self.print_line("OK.") @@ -105,7 +105,7 @@ class InputOutput(object): m = re.match("^backfill (\S+)$", line) if m: # we want to backfill a room - room_name, = m.groups() + (room_name,) = m.groups() self.print_line("backfill %s" % room_name) self.server.backfill(room_name) return @@ -339,7 +339,7 @@ def main(stdscr): root_logger = logging.getLogger() formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s" + "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" ) if not os.path.exists("logs"): os.makedirs("logs") diff --git a/contrib/grafana/README.md b/contrib/grafana/README.md index 6a6cc0bed4..ca780d412e 100644 --- a/contrib/grafana/README.md +++ b/contrib/grafana/README.md @@ -1,6 +1,6 @@ # Using the Synapse Grafana dashboard 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/ -1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst +1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/ 3. Set up additional recording rules diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 5b1bfd1679..30a8681f5a 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -18,7 +18,7 @@ "gnetId": null, "graphTooltip": 0, "id": 1, - "iteration": 1561447718159, + "iteration": 1591098104645, "links": [ { "asDropdown": true, @@ -34,6 +34,7 @@ "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -52,12 +53,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 1 }, + "hiddenSeries": false, "id": 75, "legend": { "avg": false, @@ -72,7 +75,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -151,6 +156,7 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, @@ -158,6 +164,7 @@ "x": 12, "y": 1 }, + "hiddenSeries": false, "id": 33, "legend": { "avg": false, @@ -172,7 +179,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -302,12 +311,14 @@ "dashes": false, "datasource": "$datasource", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 10 }, + "hiddenSeries": false, "id": 107, "legend": { "avg": false, @@ -322,7 +333,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -425,12 +438,14 @@ "dashes": false, "datasource": "$datasource", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 19 }, + "hiddenSeries": false, "id": 118, "legend": { "avg": false, @@ -445,7 +460,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -542,6 +559,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -559,13 +577,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 29 + "y": 2 }, + "hiddenSeries": false, "id": 5, "legend": { "alignAsTable": false, @@ -584,6 +604,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -686,12 +709,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 29 + "y": 2 }, + "hiddenSeries": false, "id": 37, "legend": { "avg": false, @@ -706,6 +731,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -792,13 +820,15 @@ "editable": true, "error": false, "fill": 0, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 36 + "y": 9 }, + "hiddenSeries": false, "id": 34, "legend": { "avg": false, @@ -813,6 +843,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -880,13 +913,16 @@ "datasource": "$datasource", "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 36 + "y": 9 }, + "hiddenSeries": false, "id": 105, + "interval": "", "legend": { "avg": false, "current": false, @@ -900,6 +936,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -934,9 +973,10 @@ "refId": "C" }, { - "expr": "", + "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])", "format": "time_series", "intervalFactor": 1, + "legendFormat": "{{job}}-{{index}} mean", "refId": "D" } ], @@ -987,14 +1027,16 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, + "fill": 0, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 43 + "y": 16 }, - "id": 50, + "hiddenSeries": false, + "id": 53, "legend": { "avg": false, "current": false, @@ -1008,6 +1050,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -1019,20 +1064,18 @@ "steppedLine": false, "targets": [ { - "expr": "rate(python_twisted_reactor_tick_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_twisted_reactor_tick_time_count[$bucket_size])", + "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", - "interval": "", "intervalFactor": 2, "legendFormat": "{{job}}-{{index}}", - "refId": "A", - "step": 20 + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Avg reactor tick time", + "title": "Up", "tooltip": { "shared": true, "sort": 0, @@ -1048,7 +1091,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -1061,7 +1104,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -1076,12 +1119,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 43 + "y": 16 }, + "hiddenSeries": false, "id": 49, "legend": { "avg": false, @@ -1096,6 +1141,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -1170,14 +1218,17 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 0, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 50 + "y": 23 }, - "id": 53, + "hiddenSeries": false, + "id": 136, + "interval": "", "legend": { "avg": false, "current": false, @@ -1189,11 +1240,12 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", - "paceLength": 10, + "options": { + "dataLinks": [] + }, "percentage": false, - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1202,18 +1254,21 @@ "steppedLine": false, "targets": [ { - "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{job}}-{{index}}", + "expr": "rate(synapse_http_client_requests{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "legendFormat": "{{job}}-{{index}} {{method}}", "refId": "A" + }, + { + "expr": "rate(synapse_http_matrixfederationclient_requests{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "legendFormat": "{{job}}-{{index}} {{method}} (federation)", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Up", + "title": "Outgoing HTTP request rate", "tooltip": { "shared": true, "sort": 0, @@ -1229,7 +1284,7 @@ }, "yaxes": [ { - "format": "short", + "format": "reqps", "label": null, "logBase": 1, "max": null, @@ -1257,12 +1312,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 50 + "y": 23 }, + "hiddenSeries": false, "id": 120, "legend": { "avg": false, @@ -1277,6 +1334,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 2, "points": false, @@ -1289,6 +1349,7 @@ { "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", + "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}", @@ -1297,6 +1358,7 @@ { "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", + "hide": false, "instant": false, "interval": "", "intervalFactor": 1, @@ -1361,6 +1423,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -1732,6 +1795,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -1750,6 +1814,7 @@ "editable": true, "error": false, "fill": 2, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -1757,6 +1822,7 @@ "x": 0, "y": 31 }, + "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": true, @@ -1775,7 +1841,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -1858,6 +1926,7 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -1865,6 +1934,7 @@ "x": 12, "y": 31 }, + "hiddenSeries": false, "id": 32, "legend": { "avg": false, @@ -1879,7 +1949,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -1948,6 +2020,7 @@ "editable": true, "error": false, "fill": 2, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -1955,7 +2028,8 @@ "x": 0, "y": 39 }, - "id": 23, + "hiddenSeries": false, + "id": 139, "legend": { "alignAsTable": true, "avg": false, @@ -1973,7 +2047,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -1984,7 +2060,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -2059,6 +2135,7 @@ "editable": true, "error": false, "fill": 2, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -2066,6 +2143,7 @@ "x": 12, "y": 39 }, + "hiddenSeries": false, "id": 52, "legend": { "alignAsTable": true, @@ -2084,7 +2162,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -2095,7 +2175,7 @@ "steppedLine": false, "targets": [ { - "expr": "(rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_response_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2167,6 +2247,7 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -2174,6 +2255,7 @@ "x": 0, "y": 47 }, + "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": true, @@ -2191,7 +2273,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -2202,7 +2286,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(synapse_http_server_response_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2260,6 +2344,7 @@ "editable": true, "error": false, "fill": 2, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, @@ -2267,6 +2352,7 @@ "x": 12, "y": 47 }, + "hiddenSeries": false, "id": 47, "legend": { "alignAsTable": true, @@ -2285,7 +2371,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -2352,12 +2440,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 55 }, + "hiddenSeries": false, "id": 103, "legend": { "avg": false, @@ -2372,7 +2462,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -2439,6 +2531,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -2454,12 +2547,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 32 }, + "hiddenSeries": false, "id": 99, "legend": { "avg": false, @@ -2474,7 +2569,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -2542,12 +2639,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 32 }, + "hiddenSeries": false, "id": 101, "legend": { "avg": false, @@ -2562,7 +2661,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -2628,6 +2729,93 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "legendFormat": "{{job}}-{{index}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Background jobs in flight", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Background jobs", @@ -2635,6 +2823,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -2650,12 +2839,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 61 + "y": 33 }, + "hiddenSeries": false, "id": 79, "legend": { "avg": false, @@ -2670,6 +2861,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -2684,8 +2878,13 @@ "expr": "sum(rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "txn rate", + "legendFormat": "successful txn rate", "refId": "A" + }, + { + "expr": "sum(rate(synapse_util_metrics_block_count{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))", + "legendFormat": "failed txn rate", + "refId": "B" } ], "thresholds": [], @@ -2736,12 +2935,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 61 + "y": 33 }, + "hiddenSeries": false, "id": 83, "legend": { "avg": false, @@ -2756,6 +2957,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -2829,12 +3033,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 70 + "y": 42 }, + "hiddenSeries": false, "id": 109, "legend": { "avg": false, @@ -2849,6 +3055,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -2923,12 +3132,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 70 + "y": 42 }, + "hiddenSeries": false, "id": 111, "legend": { "avg": false, @@ -2943,6 +3154,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3002,6 +3216,144 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 51 + }, + "hiddenSeries": false, + "id": 140, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "presence changed", + "refId": "A" + }, + { + "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "presence map", + "refId": "B" + }, + { + "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "presence destinations", + "refId": "E" + }, + { + "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "keyed edus", + "refId": "C" + }, + { + "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "other edus", + "refId": "D" + }, + { + "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "stream positions", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing EDU queues on master", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "title": "Federation", @@ -3009,6 +3361,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -3024,12 +3377,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 34 }, + "hiddenSeries": false, "id": 51, "legend": { "avg": false, @@ -3044,6 +3399,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3112,6 +3470,95 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "hiddenSeries": false, + "id": 134, + "legend": { + "avg": false, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})", + "legendFormat": "{{kind}} {{app_id}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active pusher instances by app", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "repeat": null, @@ -3120,6 +3567,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -3135,12 +3583,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 35 + "y": 52 }, + "hiddenSeries": false, "id": 48, "legend": { "avg": false, @@ -3155,7 +3605,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3225,12 +3677,14 @@ "datasource": "$datasource", "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 35 + "y": 52 }, + "hiddenSeries": false, "id": 104, "legend": { "alignAsTable": true, @@ -3246,7 +3700,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3340,13 +3796,15 @@ "editable": true, "error": false, "fill": 0, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 42 + "y": 59 }, + "hiddenSeries": false, "id": 10, "legend": { "avg": false, @@ -3363,7 +3821,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3432,13 +3892,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 42 + "y": 59 }, + "hiddenSeries": false, "id": 11, "legend": { "avg": false, @@ -3455,7 +3917,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3523,6 +3987,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -3540,13 +4005,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 0, - "y": 36 + "y": 67 }, + "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, @@ -3562,6 +4029,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3630,13 +4100,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 12, - "y": 36 + "y": 67 }, + "hiddenSeries": false, "id": 26, "legend": { "alignAsTable": true, @@ -3652,6 +4124,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3720,13 +4195,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 0, - "y": 49 + "y": 80 }, + "hiddenSeries": false, "id": 13, "legend": { "alignAsTable": true, @@ -3742,6 +4219,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3753,7 +4233,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -3807,16 +4287,19 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "The time each database transaction takes to execute, on average, broken down by metrics block.", "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 12, - "y": 49 + "y": 80 }, + "hiddenSeries": false, "id": 27, "legend": { "alignAsTable": true, @@ -3832,6 +4315,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3856,7 +4342,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Average Database Time per Block", + "title": "Average Database Transaction time, by Block", "tooltip": { "shared": false, "sort": 0, @@ -3900,13 +4386,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 0, - "y": 62 + "y": 93 }, + "hiddenSeries": false, "id": 28, "legend": { "avg": false, @@ -3921,6 +4409,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -3989,13 +4480,15 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 13, "w": 12, "x": 12, - "y": 62 + "y": 93 }, + "hiddenSeries": false, "id": 25, "legend": { "avg": false, @@ -4010,6 +4503,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4076,6 +4572,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -4094,6 +4591,7 @@ "editable": true, "error": false, "fill": 0, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 10, @@ -4101,6 +4599,7 @@ "x": 0, "y": 37 }, + "hiddenSeries": false, "id": 1, "legend": { "alignAsTable": true, @@ -4118,6 +4617,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4187,6 +4689,7 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 10, @@ -4194,6 +4697,7 @@ "x": 12, "y": 37 }, + "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, @@ -4210,6 +4714,9 @@ "linewidth": 2, "links": [], "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4278,6 +4785,7 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 10, @@ -4285,6 +4793,7 @@ "x": 0, "y": 47 }, + "hiddenSeries": false, "id": 38, "legend": { "alignAsTable": true, @@ -4301,6 +4810,9 @@ "linewidth": 2, "links": [], "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4366,12 +4878,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 47 }, + "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, @@ -4387,6 +4901,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4453,12 +4970,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 57 }, + "hiddenSeries": false, "id": 65, "legend": { "alignAsTable": true, @@ -4474,6 +4993,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4540,6 +5062,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -4555,12 +5078,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 66 }, + "hiddenSeries": false, "id": 91, "legend": { "avg": false, @@ -4575,6 +5100,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4645,6 +5173,7 @@ "editable": true, "error": false, "fill": 1, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, @@ -4652,6 +5181,7 @@ "x": 12, "y": 66 }, + "hiddenSeries": false, "id": 21, "legend": { "alignAsTable": true, @@ -4667,6 +5197,9 @@ "linewidth": 2, "links": [], "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4733,12 +5266,14 @@ "datasource": "$datasource", "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 75 }, + "hiddenSeries": false, "id": 89, "legend": { "avg": false, @@ -4755,6 +5290,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4826,12 +5364,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 75 }, + "hiddenSeries": false, "id": 93, "legend": { "avg": false, @@ -4846,6 +5386,9 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -4911,12 +5454,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 84 }, + "hiddenSeries": false, "id": 95, "legend": { "avg": false, @@ -4931,6 +5476,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 5, "points": false, @@ -5019,6 +5567,7 @@ "show": true }, "links": [], + "options": {}, "reverseYBuckets": false, "targets": [ { @@ -5060,6 +5609,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -5075,12 +5625,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 67 + "y": 39 }, + "hiddenSeries": false, "id": 2, "legend": { "avg": false, @@ -5095,6 +5647,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5194,12 +5749,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 67 + "y": 39 }, + "hiddenSeries": false, "id": 41, "legend": { "avg": false, @@ -5214,6 +5771,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5282,12 +5842,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 74 + "y": 46 }, + "hiddenSeries": false, "id": 42, "legend": { "avg": false, @@ -5302,6 +5864,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5369,12 +5934,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 74 + "y": 46 }, + "hiddenSeries": false, "id": 43, "legend": { "avg": false, @@ -5389,6 +5956,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5456,12 +6026,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 81 + "y": 53 }, + "hiddenSeries": false, "id": 113, "legend": { "avg": false, @@ -5476,6 +6048,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5492,6 +6067,13 @@ "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} {{stream_name}}", "refId": "A" + }, + { + "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{job}}-{{index}}", + "refId": "B" } ], "thresholds": [], @@ -5518,7 +6100,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5542,12 +6124,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 81 + "y": 53 }, + "hiddenSeries": false, "id": 115, "legend": { "avg": false, @@ -5562,6 +6146,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5573,7 +6160,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(synapse_replication_tcp_protocol_close_reason{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_replication_tcp_protocol_close_reason{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}-{{index}} {{reason_type}}", @@ -5628,6 +6215,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -5643,12 +6231,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 13 + "y": 58 }, + "hiddenSeries": false, "id": 67, "legend": { "avg": false, @@ -5663,7 +6253,9 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5679,7 +6271,7 @@ "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{job}}-{{index}} ", + "legendFormat": "{{job}}-{{index}} {{name}}", "refId": "A" } ], @@ -5731,12 +6323,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, - "y": 13 + "y": 58 }, + "hiddenSeries": false, "id": 71, "legend": { "avg": false, @@ -5751,7 +6345,9 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5819,12 +6415,14 @@ "dashes": false, "datasource": "$datasource", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 22 + "y": 67 }, + "hiddenSeries": false, "id": 121, "interval": "", "legend": { @@ -5840,7 +6438,9 @@ "linewidth": 1, "links": [], "nullPointMode": "connected", - "options": {}, + "options": { + "dataLinks": [] + }, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5909,6 +6509,7 @@ }, { "collapsed": true, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -5938,7 +6539,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 41 }, "heatmap": {}, "hideZeroBuckets": true, @@ -5993,12 +6594,14 @@ "datasource": "$datasource", "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 41 }, + "hiddenSeries": false, "id": 124, "interval": "", "legend": { @@ -6014,7 +6617,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 2, "points": false, @@ -6095,7 +6700,7 @@ "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 49 }, "heatmap": {}, "hideZeroBuckets": true, @@ -6150,12 +6755,14 @@ "datasource": "$datasource", "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 22 + "y": 49 }, + "hiddenSeries": false, "id": 128, "legend": { "avg": false, @@ -6170,7 +6777,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 2, "points": false, @@ -6270,7 +6879,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 57 }, "heatmap": {}, "hideZeroBuckets": true, @@ -6325,12 +6934,14 @@ "datasource": "$datasource", "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 57 }, + "hiddenSeries": false, "id": 130, "legend": { "avg": false, @@ -6345,7 +6956,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 2, "points": false, @@ -6445,7 +7058,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 65 }, "heatmap": {}, "hideZeroBuckets": true, @@ -6500,12 +7113,14 @@ "datasource": "$datasource", "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 65 }, + "hiddenSeries": false, "id": 132, "interval": "", "legend": { @@ -6521,7 +7136,9 @@ "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 2, "points": false, @@ -6607,7 +7224,7 @@ } ], "refresh": "5m", - "schemaVersion": 18, + "schemaVersion": 22, "style": "dark", "tags": [ "matrix" @@ -6616,7 +7233,6 @@ "list": [ { "current": { - "tags": [], "text": "Prometheus", "value": "Prometheus" }, @@ -6638,6 +7254,7 @@ "auto_count": 100, "auto_min": "30s", "current": { + "selected": false, "text": "auto", "value": "$__auto_interval_bucket_size" }, @@ -6719,9 +7336,9 @@ "allFormat": "regex wildcard", "allValue": "", "current": { - "text": "All", + "text": "synapse", "value": [ - "$__all" + "synapse" ] }, "datasource": "$datasource", @@ -6750,6 +7367,7 @@ "allFormat": "regex wildcard", "allValue": ".*", "current": { + "selected": false, "text": "All", "value": "$__all" }, @@ -6810,5 +7428,5 @@ "timezone": "", "title": "Synapse", "uid": "000000012", - "version": 10 + "version": 29 } \ No newline at end of file diff --git a/contrib/graph/graph2.py b/contrib/graph/graph2.py index 9db8725eee..4619f0e3c1 100644 --- a/contrib/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -36,7 +36,7 @@ def make_graph(db_name, room_id, file_prefix, limit): args = [room_id] if limit: - sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?" + sql += " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?" args.append(limit) @@ -53,7 +53,7 @@ def make_graph(db_name, room_id, file_prefix, limit): for event in events: c = conn.execute( - "SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?", + "SELECT state_group FROM event_to_state_groups WHERE event_id = ?", (event.event_id,), ) diff --git a/contrib/systemd-with-workers/README.md b/contrib/systemd-with-workers/README.md index 74b261e9fb..8d21d532bd 100644 --- a/contrib/systemd-with-workers/README.md +++ b/contrib/systemd-with-workers/README.md @@ -1,150 +1,2 @@ -# Setup Synapse with Workers and Systemd - -This is a setup for managing synapse with systemd including support for -managing workers. It provides a `matrix-synapse`, as well as a -`matrix-synapse-worker@` service for any workers you require. Additionally to -group the required services it sets up a `matrix.target`. You can use this to -automatically start any bot- or bridge-services. More on this in -[Bots and Bridges](#bots-and-bridges). - -See the folder [system](system) for any service and target files. - -The folder [workers](workers) contains an example configuration for the -`federation_reader` worker. Pay special attention to the name of the -configuration file. In order to work with the `matrix-synapse-worker@.service` -service, it needs to have the exact same name as the worker app. - -This setup expects neither the homeserver nor any workers to fork. Forking is -handled by systemd. - -## Setup - -1. Adjust your matrix configs. Make sure that the worker config files have the -exact same name as the worker app. Compare `matrix-synapse-worker@.service` for -why. You can find an example worker config in the [workers](workers) folder. See -below for relevant settings in the `homeserver.yaml`. -2. Copy the `*.service` and `*.target` files in [system](system) to -`/etc/systemd/system`. -3. `systemctl enable matrix-synapse.service` this adds the homeserver -app to the `matrix.target` -4. *Optional.* `systemctl enable -matrix-synapse-worker@federation_reader.service` this adds the federation_reader -app to the `matrix-synapse.service` -5. *Optional.* Repeat step 4 for any additional workers you require. -6. *Optional.* Add any bots or bridges by enabling them. -7. Start all matrix related services via `systemctl start matrix.target` -8. *Optional.* Enable autostart of all matrix related services on system boot -via `systemctl enable matrix.target` - -## Usage - -After you have setup you can use the following commands to manage your synapse -installation: - -``` -# Start matrix-synapse, all workers and any enabled bots or bridges. -systemctl start matrix.target - -# Restart matrix-synapse and all workers (not necessarily restarting bots -# or bridges, see "Bots and Bridges") -systemctl restart matrix-synapse.service - -# Stop matrix-synapse and all workers (not necessarily restarting bots -# or bridges, see "Bots and Bridges") -systemctl stop matrix-synapse.service - -# Restart a specific worker (i. e. federation_reader), the homeserver is -# unaffected by this. -systemctl restart matrix-synapse-worker@federation_reader.service - -# Add a new worker (assuming all configs are setup already) -systemctl enable matrix-synapse-worker@federation_writer.service -systemctl restart matrix-synapse.service -``` - -## The Configs - -Make sure the `worker_app` is set in the `homeserver.yaml` and it does not fork. - -``` -worker_app: synapse.app.homeserver -daemonize: false -``` - -None of the workers should fork, as forking is handled by systemd. Hence make -sure this is present in all worker config files. - -``` -worker_daemonize: false -``` - -The config files of all workers are expected to be located in -`/etc/matrix-synapse/workers`. If you want to use a different location you have -to edit the provided `*.service` files accordingly. - -## Bots and Bridges - -Most bots and bridges do not care if the homeserver goes down or is restarted. -Depending on the implementation this may crash them though. So look up the docs -or ask the community of the specific bridge or bot you want to run to make sure -you choose the correct setup. - -Whichever configuration you choose, after the setup the following will enable -automatically starting (and potentially restarting) your bot/bridge with the -`matrix.target`. - -``` -systemctl enable <yourBotOrBridgeName>.service -``` - -**Note** that from an inactive synapse the bots/bridges will only be started with -synapse if you start the `matrix.target`, not if you start the -`matrix-synapse.service`. This is on purpose. Think of `matrix-synapse.service` -as *just* synapse, but `matrix.target` being anything matrix related, including -synapse and any and all enabled bots and bridges. - -### Start with synapse but ignore synapse going down - -If the bridge can handle shutdowns of the homeserver you'll want to install the -service in the `matrix.target` and optionally add a -`After=matrix-synapse.service` dependency to have the bot/bridge start after -synapse on starting everything. - -In this case the service file should look like this. - -``` -[Unit] -# ... -# Optional, this will only ensure that if you start everything, synapse will -# be started before the bot/bridge will be started. -After=matrix-synapse.service - -[Service] -# ... - -[Install] -WantedBy=matrix.target -``` - -### Stop/restart when synapse stops/restarts - -If the bridge can't handle shutdowns of the homeserver you'll still want to -install the service in the `matrix.target` but also have to specify the -`After=matrix-synapse.service` *and* `BindsTo=matrix-synapse.service` -dependencies to have the bot/bridge stop/restart with synapse. - -In this case the service file should look like this. - -``` -[Unit] -# ... -# Mandatory -After=matrix-synapse.service -BindsTo=matrix-synapse.service - -[Service] -# ... - -[Install] -WantedBy=matrix.target -``` +The documentation for using systemd to manage synapse workers is now part of +the main synapse distribution. See [docs/systemd-with-workers](../../docs/systemd-with-workers). diff --git a/contrib/systemd-with-workers/system/matrix-synapse-worker@.service b/contrib/systemd-with-workers/system/matrix-synapse-worker@.service deleted file mode 100644 index 3507e2e989..0000000000 --- a/contrib/systemd-with-workers/system/matrix-synapse-worker@.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Synapse Matrix Worker -After=matrix-synapse.service -BindsTo=matrix-synapse.service - -[Service] -Type=notify -NotifyAccess=main -User=matrix-synapse -WorkingDirectory=/var/lib/matrix-synapse -EnvironmentFile=/etc/default/matrix-synapse -ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.%i --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --config-path=/etc/matrix-synapse/workers/%i.yaml -ExecReload=/bin/kill -HUP $MAINPID -Restart=always -RestartSec=3 -SyslogIdentifier=matrix-synapse-%i - -[Install] -WantedBy=matrix-synapse.service diff --git a/contrib/systemd-with-workers/system/matrix-synapse.service b/contrib/systemd-with-workers/system/matrix-synapse.service deleted file mode 100644 index 68e8991f18..0000000000 --- a/contrib/systemd-with-workers/system/matrix-synapse.service +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=Synapse Matrix Homeserver - -[Service] -Type=notify -NotifyAccess=main -User=matrix-synapse -WorkingDirectory=/var/lib/matrix-synapse -EnvironmentFile=/etc/default/matrix-synapse -ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys -ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ -ExecReload=/bin/kill -HUP $MAINPID -Restart=always -RestartSec=3 -SyslogIdentifier=matrix-synapse - -[Install] -WantedBy=matrix.target diff --git a/contrib/systemd-with-workers/system/matrix.target b/contrib/systemd-with-workers/system/matrix.target deleted file mode 100644 index aff97d03ef..0000000000 --- a/contrib/systemd-with-workers/system/matrix.target +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Contains matrix services like synapse, bridges and bots -After=network.target -AllowIsolate=no - -[Install] -WantedBy=multi-user.target diff --git a/contrib/systemd-with-workers/workers/federation_reader.yaml b/contrib/systemd-with-workers/workers/federation_reader.yaml deleted file mode 100644 index 47c54ec0d4..0000000000 --- a/contrib/systemd-with-workers/workers/federation_reader.yaml +++ /dev/null @@ -1,14 +0,0 @@ -worker_app: synapse.app.federation_reader - -worker_replication_host: 127.0.0.1 -worker_replication_port: 9092 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8011 - resources: - - names: [federation] - -worker_daemonize: false -worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml diff --git a/contrib/systemd/README.md b/contrib/systemd/README.md new file mode 100644 index 0000000000..5d42b3464f --- /dev/null +++ b/contrib/systemd/README.md @@ -0,0 +1,17 @@ +# Setup Synapse with Systemd +This is a setup for managing synapse with a user contributed systemd unit +file. It provides a `matrix-synapse` systemd unit file that should be tailored +to accommodate your installation in accordance with the installation +instructions provided in [installation instructions](../../INSTALL.md). + +## Setup +1. Under the service section, ensure the `User` variable matches which user +you installed synapse under and wish to run it as. +2. Under the service section, ensure the `WorkingDirectory` variable matches +where you have installed synapse. +3. Under the service section, ensure the `ExecStart` variable matches the +appropriate locations of your installation. +4. Copy the `matrix-synapse.service` to `/etc/systemd/system/` +5. Start Synapse: `sudo systemctl start matrix-synapse` +6. Verify Synapse is running: `sudo systemctl status matrix-synapse` +7. *optional* Enable Synapse to start at system boot: `sudo systemctl enable matrix-synapse` diff --git a/contrib/systemd/matrix-synapse.service b/contrib/systemd/matrix-synapse.service index 38d369ea3d..a754078410 100644 --- a/contrib/systemd/matrix-synapse.service +++ b/contrib/systemd/matrix-synapse.service @@ -4,14 +4,20 @@ # systemctl enable matrix-synapse # systemctl start matrix-synapse # +# This assumes that Synapse has been installed by a user named +# synapse. +# # This assumes that Synapse has been installed in a virtualenv in -# /opt/synapse/env. +# the user's home directory: `/home/synapse/synapse/env`. # # **NOTE:** This is an example service file that may change in the future. If you # wish to use this please copy rather than symlink it. [Unit] Description=Synapse Matrix homeserver +# If you are using postgresql to persist data, uncomment this line to make sure +# synapse starts after the postgresql service. +# After=postgresql.service [Service] Type=notify @@ -22,8 +28,8 @@ Restart=on-abort User=synapse Group=nogroup -WorkingDirectory=/opt/synapse -ExecStart=/opt/synapse/env/bin/python -m synapse.app.homeserver --config-path=/opt/synapse/homeserver.yaml +WorkingDirectory=/home/synapse/synapse +ExecStart=/home/synapse/synapse/env/bin/python -m synapse.app.homeserver --config-path=/home/synapse/synapse/homeserver.yaml SyslogIdentifier=matrix-synapse # adjust the cache factor if necessary |