diff options
| author | David Robertson <davidr@element.io> | 2023-10-24 14:23:19 +0100 |
|---|---|---|
| committer | David Robertson <davidr@element.io> | 2023-10-24 14:23:19 +0100 |
| commit | c0d2f7649e2951ed8a91532adf1f6acceeb0f06f (patch) | |
| tree | 92dfd202e2316fa3bf3f33c47433de651dd0e880 | |
| parent | Add test case to detect dodgy b64 encoding (diff) | |
| parent | Rework alias and public room list rules docs (#16541) (diff) | |
| download | synapse-c0d2f7649e2951ed8a91532adf1f6acceeb0f06f.tar.xz | |
Merge branch 'develop' of github.com:matrix-org/synapse into develop
33 files changed, 786 insertions, 460 deletions
diff --git a/changelog.d/16471.bugfix b/changelog.d/16471.bugfix new file mode 100644 index 0000000000..c94cd5b78f --- /dev/null +++ b/changelog.d/16471.bugfix @@ -0,0 +1 @@ +Fixed a bug that prevents Grafana from finding the correct datasource. Contributed by @MichaelSasser. diff --git a/changelog.d/16473.bugfix b/changelog.d/16473.bugfix new file mode 100644 index 0000000000..4f4a0380cd --- /dev/null +++ b/changelog.d/16473.bugfix @@ -0,0 +1 @@ +Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`. diff --git a/changelog.d/16515.misc b/changelog.d/16515.misc new file mode 100644 index 0000000000..d54dd730e1 --- /dev/null +++ b/changelog.d/16515.misc @@ -0,0 +1 @@ +Remove duplicate call to mark remote server 'awake' when using a federation sending worker. diff --git a/changelog.d/16521.misc b/changelog.d/16521.misc new file mode 100644 index 0000000000..c6a8ddcf9c --- /dev/null +++ b/changelog.d/16521.misc @@ -0,0 +1 @@ +Stop deleting from an unused table. diff --git a/changelog.d/16526.misc b/changelog.d/16526.misc new file mode 100644 index 0000000000..93ceaeafc9 --- /dev/null +++ b/changelog.d/16526.misc @@ -0,0 +1 @@ +Improve type hints. diff --git a/changelog.d/16530.bugfix b/changelog.d/16530.bugfix new file mode 100644 index 0000000000..503ea0af20 --- /dev/null +++ b/changelog.d/16530.bugfix @@ -0,0 +1 @@ +Force TLS certificate verification in user registration script. diff --git a/changelog.d/16531.doc b/changelog.d/16531.doc new file mode 100644 index 0000000000..0932d1abf1 --- /dev/null +++ b/changelog.d/16531.doc @@ -0,0 +1 @@ +Add a sentence to the opentracing docs on how you can have jaeger in a different place than synapse. diff --git a/changelog.d/16539.misc b/changelog.d/16539.misc new file mode 100644 index 0000000000..cd21bdb26d --- /dev/null +++ b/changelog.d/16539.misc @@ -0,0 +1 @@ +Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0. diff --git a/changelog.d/16540.bugfix b/changelog.d/16540.bugfix new file mode 100644 index 0000000000..34ee9facf9 --- /dev/null +++ b/changelog.d/16540.bugfix @@ -0,0 +1 @@ +Fix long-standing bug where `/sync` could tightloop after restart when using SQLite. diff --git a/changelog.d/16541.doc b/changelog.d/16541.doc new file mode 100644 index 0000000000..39aeecada6 --- /dev/null +++ b/changelog.d/16541.doc @@ -0,0 +1 @@ +Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally. diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 90f449aa76..188597c8dd 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -1,14 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], "__elements": {}, "__requires": [ { @@ -47,7 +37,7 @@ { "builtIn": 1, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "enable": false, "hide": true, @@ -93,7 +83,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -107,7 +97,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -129,7 +119,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -203,7 +193,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)", "format": "heatmap", @@ -235,7 +225,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -333,7 +323,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", @@ -343,7 +333,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", @@ -354,7 +344,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", @@ -364,7 +354,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", "format": "time_series", @@ -374,7 +364,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", "legendFormat": "25%", @@ -382,7 +372,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))", "legendFormat": "5%", @@ -390,7 +380,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))", "legendFormat": "Average", @@ -398,7 +388,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))", "hide": false, @@ -468,7 +458,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -515,7 +505,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -575,7 +565,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -625,7 +615,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -638,7 +628,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", "hide": true, @@ -776,7 +766,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -831,7 +821,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -844,7 +834,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -893,7 +883,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -910,7 +900,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -973,7 +963,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", @@ -987,7 +977,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -1217,7 +1207,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -1267,7 +1257,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -1280,7 +1270,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", "interval": "", @@ -1326,7 +1316,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1379,7 +1369,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -1432,7 +1422,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1487,7 +1477,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", @@ -1500,7 +1490,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1", @@ -1546,7 +1536,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1592,7 +1582,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -1604,7 +1594,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -1664,7 +1654,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1710,7 +1700,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", @@ -1720,7 +1710,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", @@ -1857,7 +1847,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -1869,7 +1859,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -1893,7 +1883,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1967,7 +1957,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)", "format": "heatmap", @@ -1998,7 +1988,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "", "editable": true, @@ -2049,7 +2039,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)", "format": "time_series", @@ -2099,7 +2089,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "decimals": 1, "fill": 1, @@ -2140,7 +2130,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -2187,7 +2177,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "decimals": 1, "fill": 1, @@ -2228,7 +2218,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", @@ -2278,7 +2268,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "decimals": 1, "fill": 1, @@ -2322,7 +2312,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", @@ -2370,7 +2360,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "decimals": 1, "fill": 1, @@ -2414,7 +2404,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)", @@ -2614,7 +2604,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n", "fieldConfig": { @@ -2692,7 +2682,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", @@ -2706,7 +2696,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", @@ -2726,7 +2716,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -2738,7 +2728,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -2755,7 +2745,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -2808,7 +2798,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -2877,7 +2867,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -2926,7 +2916,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})", "format": "time_series", @@ -2976,7 +2966,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -3029,7 +3019,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -3098,7 +3088,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -3151,7 +3141,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -3220,7 +3210,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -3272,7 +3262,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -3321,7 +3311,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -3374,7 +3364,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))", "format": "time_series", @@ -3422,7 +3412,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3475,7 +3465,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})", "format": "time_series", @@ -3486,7 +3476,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "interval": "", @@ -3529,7 +3519,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -3541,7 +3531,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -3557,7 +3547,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3604,7 +3594,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -3650,7 +3640,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3697,7 +3687,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -3743,7 +3733,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3788,7 +3778,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", "legendFormat": "{{job}}-{{index}} {{name}}", @@ -3830,7 +3820,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -3842,7 +3832,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -3858,7 +3848,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -3905,7 +3895,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", "format": "time_series", @@ -3915,7 +3905,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", "legendFormat": "failed txn rate", @@ -3958,7 +3948,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4005,7 +3995,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))", "format": "time_series", @@ -4015,7 +4005,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))", "format": "time_series", @@ -4061,7 +4051,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4108,7 +4098,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations_count_total{instance=\"$instance\"}[$bucket_size]))", @@ -4121,7 +4111,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))", "format": "time_series", @@ -4167,7 +4157,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4214,7 +4204,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])", @@ -4509,7 +4499,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "The number of events in the in-memory queues ", "fieldConfig": { @@ -4556,7 +4546,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", @@ -4568,7 +4558,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "interval": "", @@ -4617,7 +4607,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Number of events queued up on the master process for processing by the federation sender", "fieldConfig": { @@ -4665,7 +4655,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -4676,7 +4666,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -4688,7 +4678,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -4700,7 +4690,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -4712,7 +4702,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -4724,7 +4714,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -4780,7 +4770,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4857,7 +4847,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)", "format": "heatmap", @@ -4892,7 +4882,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -4981,7 +4971,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -4992,7 +4982,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -5003,7 +4993,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -5014,7 +5004,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -5025,7 +5015,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "interval": "", @@ -5034,7 +5024,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "interval": "", @@ -5043,7 +5033,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "interval": "", @@ -5116,7 +5106,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -5193,7 +5183,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)", "format": "heatmap", @@ -5229,7 +5219,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -5279,7 +5269,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", @@ -5333,7 +5323,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -5383,7 +5373,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", @@ -5437,7 +5427,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -5477,7 +5467,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))", "interval": "", @@ -5522,7 +5512,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -5903,7 +5893,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -6008,7 +5998,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", @@ -6021,7 +6011,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", @@ -6033,7 +6023,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -6044,7 +6034,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -6054,7 +6044,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "format": "time_series", @@ -6064,7 +6054,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "legendFormat": "25%", @@ -6072,7 +6062,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))", "legendFormat": "5%", @@ -6080,7 +6070,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "legendFormat": "Average", @@ -6267,7 +6257,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -6280,7 +6270,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -6359,7 +6349,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", @@ -6373,7 +6363,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", @@ -6394,7 +6384,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -6441,7 +6431,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})", "legendFormat": "{{kind}} {{app_id}}", @@ -6483,7 +6473,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -6495,7 +6485,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -6662,7 +6652,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", @@ -7077,7 +7067,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -7089,7 +7079,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -7101,7 +7091,7 @@ "panels": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -7179,7 +7169,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])", "format": "time_series", @@ -7198,7 +7188,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan", "fieldConfig": { @@ -7247,7 +7237,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -7259,7 +7249,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -7269,7 +7259,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -7279,7 +7269,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -7327,7 +7317,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -7379,7 +7369,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -7427,7 +7417,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -7479,7 +7469,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -7527,7 +7517,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -7579,7 +7569,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -7627,7 +7617,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -7673,7 +7663,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", "format": "time_series", @@ -7683,7 +7673,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", "format": "time_series", @@ -7693,7 +7683,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", "format": "time_series", @@ -7703,7 +7693,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))", "format": "time_series", @@ -7751,7 +7741,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -7763,7 +7753,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -7779,7 +7769,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -7830,7 +7820,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])", "format": "time_series", @@ -7877,7 +7867,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -7928,7 +7918,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])", "format": "time_series", @@ -8079,7 +8069,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "The time each database transaction takes to execute, on average, broken down by metrics block.", "editable": true, @@ -8131,7 +8121,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -8178,7 +8168,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -8228,7 +8218,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -8275,7 +8265,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -8325,7 +8315,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -8374,7 +8364,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -8414,7 +8404,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "interval": "", @@ -8457,7 +8447,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -8469,7 +8459,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -8485,7 +8475,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "decimals": 2, "editable": true, @@ -8538,7 +8528,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", @@ -8588,7 +8578,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -8639,7 +8629,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -8688,7 +8678,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editable": true, "error": false, @@ -8739,7 +8729,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", @@ -8787,7 +8777,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -8839,7 +8829,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", @@ -8888,7 +8878,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -8935,7 +8925,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -8981,7 +8971,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -8993,7 +8983,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -9009,7 +8999,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9055,7 +9045,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "interval": "", @@ -9099,7 +9089,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9145,7 +9135,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])", "interval": "", @@ -9154,7 +9144,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "", "interval": "", @@ -9199,7 +9189,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -9211,7 +9201,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -9227,7 +9217,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9274,7 +9264,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])", "format": "time_series", @@ -9321,7 +9311,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "decimals": 3, "editable": true, @@ -9373,7 +9363,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])", "format": "time_series", @@ -9420,7 +9410,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.", "fieldConfig": { @@ -9475,7 +9465,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", "format": "time_series", @@ -9522,7 +9512,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9569,7 +9559,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -9614,7 +9604,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9661,7 +9651,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", @@ -9772,7 +9762,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -9784,7 +9774,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -9801,7 +9791,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9848,7 +9838,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)", "format": "time_series", @@ -9893,7 +9883,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -9991,7 +9981,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10090,7 +10080,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10288,7 +10278,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10335,7 +10325,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", @@ -10382,7 +10372,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10429,7 +10419,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", "format": "time_series", @@ -10439,7 +10429,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}", "format": "time_series", @@ -10484,7 +10474,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -10496,7 +10486,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -10512,7 +10502,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10559,7 +10549,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -10607,7 +10597,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10654,7 +10644,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", @@ -10702,7 +10692,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -10750,7 +10740,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1", "format": "time_series", @@ -10797,7 +10787,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -10809,7 +10799,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -10833,7 +10823,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.", "fieldConfig": { @@ -10909,7 +10899,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", @@ -10941,7 +10931,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.", "fieldConfig": { @@ -10989,7 +10979,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0", "format": "heatmap", @@ -11044,7 +11034,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.", "fieldConfig": { @@ -11120,7 +11110,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", @@ -11152,7 +11142,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.", "fieldConfig": { @@ -11199,7 +11189,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11209,7 +11199,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11219,7 +11209,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11229,7 +11219,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11284,7 +11274,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", "fieldConfig": { @@ -11360,7 +11350,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", @@ -11392,7 +11382,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.", "fieldConfig": { @@ -11439,7 +11429,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11449,7 +11439,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11459,7 +11449,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11469,7 +11459,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", @@ -11524,7 +11514,7 @@ }, "dataFormat": "tsbuckets", "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.", "fieldConfig": { @@ -11600,7 +11590,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "heatmap", @@ -11634,7 +11624,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.", "fieldConfig": { @@ -11682,7 +11672,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", @@ -11695,7 +11685,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -11706,7 +11696,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -11717,7 +11707,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", @@ -11765,7 +11755,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.", "fill": 1, @@ -11805,7 +11795,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", @@ -11814,7 +11804,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", @@ -11823,7 +11813,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", @@ -11866,7 +11856,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -11878,7 +11868,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -11895,7 +11885,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -11949,7 +11939,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})", @@ -11963,7 +11953,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})", @@ -12012,7 +12002,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -12051,7 +12041,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}", "interval": "", @@ -12094,7 +12084,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -12106,7 +12096,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -12123,7 +12113,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -12169,7 +12159,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", @@ -12222,7 +12212,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -12268,7 +12258,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])", @@ -12319,7 +12309,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -12331,7 +12321,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -12348,7 +12338,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -12387,7 +12377,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])", @@ -12436,7 +12426,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -12475,7 +12465,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])", @@ -12522,7 +12512,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -12534,7 +12524,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -12550,7 +12540,7 @@ "dashLength": 10, "dashes": false, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -12589,7 +12579,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", @@ -12598,7 +12588,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", @@ -12607,7 +12597,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", @@ -12616,7 +12606,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", @@ -12625,7 +12615,7 @@ }, { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", @@ -12670,7 +12660,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -12709,7 +12699,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", @@ -12758,7 +12748,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fill": 1, "fillGradient": 0, @@ -12797,7 +12787,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", @@ -12844,7 +12834,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -12856,7 +12846,7 @@ "collapsed": true, "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -12869,7 +12859,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -12946,7 +12936,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])", @@ -12966,7 +12956,7 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fill": 1, @@ -13006,7 +12996,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))", @@ -13063,7 +13053,7 @@ "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -13140,7 +13130,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)", "format": "heatmap", @@ -13172,7 +13162,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -13246,7 +13236,7 @@ "targets": [ { "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])", @@ -13264,7 +13254,7 @@ { "datasource": { "type": "prometheus", - "uid": "000000001" + "uid": "${DS_PROMETHEUS}" }, "refId": "A" } @@ -13290,7 +13280,8 @@ "hide": 0, "includeAll": false, "multi": false, - "name": "datasource", + "name": "DS_PROMETHEUS", + "label": "Datasource", "options": [], "query": "prometheus", "queryValue": "", @@ -13361,7 +13352,7 @@ { "current": {}, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "definition": "", "hide": 0, @@ -13387,7 +13378,7 @@ "allValue": "", "current": {}, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "definition": "", "hide": 0, @@ -13417,7 +13408,7 @@ "allValue": ".*", "current": {}, "datasource": { - "uid": "$datasource" + "uid": "${DS_PROMETHEUS}" }, "definition": "", "hide": 0, diff --git a/docs/development/synapse_architecture/streams.md b/docs/development/synapse_architecture/streams.md index bee0b8a8c0..67d92acfa1 100644 --- a/docs/development/synapse_architecture/streams.md +++ b/docs/development/synapse_architecture/streams.md @@ -51,17 +51,24 @@ will be inserted with that ID. For any given stream reader (including writers themselves), we may define a per-writer current stream ID: -> The current stream ID _for a writer W_ is the largest stream ID such that +> A current stream ID _for a writer W_ is the largest stream ID such that > all transactions added by W with equal or smaller ID have completed. Similarly, there is a "linear" notion of current stream ID: -> The "linear" current stream ID is the largest stream ID such that +> A "linear" current stream ID is the largest stream ID such that > all facts (added by any writer) with equal or smaller ID have completed. Because different stream readers A and B learn about new facts at different times, A and B may disagree about current stream IDs. Put differently: we should think of stream readers as being independent of each other, proceeding through a stream of facts at different rates. +The above definition does not give a unique current stream ID, in fact there can +be a range of current stream IDs. Synapse uses both the minimum and maximum IDs +for different purposes. Most often the maximum is used, as its generally +beneficial for workers to advance their IDs as soon as possible. However, the +minimum is used in situations where e.g. another worker is going to wait until +the stream advances past a position. + **NB.** For both senses of "current", that if a writer opens a transaction that never completes, the current stream ID will never advance beyond that writer's last written stream ID. For single-writer streams, the per-writer current ID and the linear current ID are the same. @@ -114,7 +121,7 @@ Writers need to track: - track their current position (i.e. its own per-writer stream ID). - their facts currently awaiting completion. -At startup, +At startup, - the current position of that writer can be found by querying the database (which suggests that facts need to be written to the database atomically, in a transaction); and - there are no facts awaiting completion. diff --git a/docs/opentracing.md b/docs/opentracing.md index abb94b565f..bf48874160 100644 --- a/docs/opentracing.md +++ b/docs/opentracing.md @@ -51,6 +51,11 @@ docker run -d --name jaeger \ jaegertracing/all-in-one:1 ``` +By default, Synapse will publish traces to Jaeger on localhost. +If Jaeger is hosted elsewhere, point Synapse to the correct host by setting +`opentracing.jaeger_config.local_agent.reporting_host` [in the Synapse configuration](usage/configuration/config_documentation.md#opentracing-1) +or by setting the `JAEGER_AGENT_HOST` environment variable to the desired address. + Latest documentation is probably at https://www.jaegertracing.io/docs/latest/getting-started. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 92e00c1380..6cc83c1cd0 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3797,62 +3797,160 @@ enable_room_list_search: false --- ### `alias_creation_rules` -The `alias_creation_rules` option controls who is allowed to create aliases -on this server. +The `alias_creation_rules` option allows server admins to prevent unwanted +alias creation on this server. -The format of this option is a list of rules that contain globs that -match against user_id, room_id and the new alias (fully qualified with -server name). The action in the first rule that matches is taken, -which can currently either be "allow" or "deny". +This setting is an optional list of 0 or more rules. By default, no list is +provided, meaning that all alias creations are permitted. -Missing user_id/room_id/alias fields default to "*". +Otherwise, requests to create aliases are matched against each rule in order. +The first rule that matches decides if the request is allowed or denied. If no +rule matches, the request is denied. In particular, this means that configuring +an empty list of rules will deny every alias creation request. -If no rules match the request is denied. An empty list means no one -can create aliases. +Each rule is a YAML object containing four fields, each of which is an optional string: -Options for the rules include: -* `user_id`: Matches against the creator of the alias. Defaults to "*". -* `alias`: Matches against the alias being created. Defaults to "*". -* `room_id`: Matches against the room ID the alias is being pointed at. Defaults to "*" -* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. +* `user_id`: a glob pattern that matches against the creator of the alias. +* `alias`: a glob pattern that matches against the alias being created. +* `room_id`: a glob pattern that matches against the room ID the alias is being pointed at. +* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. + +Each of the glob patterns is optional, defaulting to `*` ("match anything"). +Note that the patterns match against fully qualified IDs, e.g. against +`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead +of `alice`, `room` and `abcedgghijk`. Example configuration: + +```yaml +# No rule list specified. All alias creations are allowed. +# This is the default behaviour. +alias_creation_rules: +``` + +```yaml +# A list of one rule which allows everything. +# This has the same effect as the previous example. +alias_creation_rules: + - "action": "allow" +``` + +```yaml +# An empty list of rules. All alias creations are denied. +alias_creation_rules: [] +``` + +```yaml +# A list of one rule which denies everything. +# This has the same effect as the previous example. +alias_creation_rules: + - "action": "deny" +``` + +```yaml +# Prevent a specific user from creating aliases. +# Allow other users to create any alias +alias_creation_rules: + - user_id: "@bad_user:example.com" + action: deny + + - action: allow +``` + ```yaml +# Prevent aliases being created which point to a specific room. alias_creation_rules: - - user_id: "bad_user" - alias: "spammy_alias" - room_id: "*" + - room_id: "!forbiddenRoom:example.com" action: deny + + - action: allow ``` + --- ### `room_list_publication_rules` -The `room_list_publication_rules` option controls who can publish and -which rooms can be published in the public room list. +The `room_list_publication_rules` option allows server admins to prevent +unwanted entries from being published in the public room list. The format of this option is the same as that for -`alias_creation_rules`. +[`alias_creation_rules`](#alias_creation_rules): an optional list of 0 or more +rules. By default, no list is provided, meaning that all rooms may be +published to the room list. + +Otherwise, requests to publish a room are matched against each rule in order. +The first rule that matches decides if the request is allowed or denied. If no +rule matches, the request is denied. In particular, this means that configuring +an empty list of rules will deny every alias creation request. -If the room has one or more aliases associated with it, only one of -the aliases needs to match the alias rule. If there are no aliases -then only rules with `alias: *` match. +Each rule is a YAML object containing four fields, each of which is an optional string: -If no rules match the request is denied. An empty list means no one -can publish rooms. +* `user_id`: a glob pattern that matches against the user publishing the room. +* `alias`: a glob pattern that matches against one of published room's aliases. + - If the room has no aliases, the alias match fails unless `alias` is unspecified or `*`. + - If the room has exactly one alias, the alias match succeeds if the `alias` pattern matches that alias. + - If the room has two or more aliases, the alias match succeeds if the pattern matches at least one of the aliases. +* `room_id`: a glob pattern that matches against the room ID of the room being published. +* `action`: either `allow` or `deny`. What to do with the request if the rule matches. Defaults to `allow`. + +Each of the glob patterns is optional, defaulting to `*` ("match anything"). +Note that the patterns match against fully qualified IDs, e.g. against +`@alice:example.com`, `#room:example.com` and `!abcdefghijk:example.com` instead +of `alice`, `room` and `abcedgghijk`. -Options for the rules include: -* `user_id`: Matches against the creator of the alias. Defaults to "*". -* `alias`: Matches against any current local or canonical aliases associated with the room. Defaults to "*". -* `room_id`: Matches against the room ID being published. Defaults to "*". -* `action`: Whether to "allow" or "deny" the request if the rule matches. Defaults to allow. Example configuration: + ```yaml +# No rule list specified. Anyone may publish any room to the public list. +# This is the default behaviour. room_list_publication_rules: - - user_id: "*" - alias: "*" - room_id: "*" - action: allow +``` + +```yaml +# A list of one rule which allows everything. +# This has the same effect as the previous example. +room_list_publication_rules: + - "action": "allow" +``` + +```yaml +# An empty list of rules. No-one may publish to the room list. +room_list_publication_rules: [] +``` + +```yaml +# A list of one rule which denies everything. +# This has the same effect as the previous example. +room_list_publication_rules: + - "action": "deny" +``` + +```yaml +# Prevent a specific user from publishing rooms. +# Allow other users to publish anything. +room_list_publication_rules: + - user_id: "@bad_user:example.com" + action: deny + + - action: allow +``` + +```yaml +# Prevent publication of a specific room. +room_list_publication_rules: + - room_id: "!forbiddenRoom:example.com" + action: deny + + - action: allow +``` + +```yaml +# Prevent publication of rooms with at least one alias containing the word "potato". +room_list_publication_rules: + - alias: "#*potato*:example.com" + action: deny + + - action: allow ``` --- diff --git a/poetry.lock b/poetry.lock index 8585d9855d..67620f8efa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -1337,13 +1337,13 @@ test = ["aiounittest", "tox", "twisted"] [[package]] name = "matrix-synapse-ldap3" -version = "0.2.2" +version = "0.3.0" description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" files = [ - {file = "matrix-synapse-ldap3-0.2.2.tar.gz", hash = "sha256:b388d95693486eef69adaefd0fd9e84463d52fe17b0214a00efcaa669b73cb74"}, - {file = "matrix_synapse_ldap3-0.2.2-py3-none-any.whl", hash = "sha256:66ee4c85d7952c6c27fd04c09cdfdf4847b8e8b7d6a7ada6ba1100013bda060f"}, + {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, + {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, ] [package.dependencies] diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index 19ca399d44..9293808640 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -50,7 +50,7 @@ def request_registration( url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),) # Get the nonce - r = requests.get(url, verify=False) + r = requests.get(url) if r.status_code != 200: _print("ERROR! Received %d %s" % (r.status_code, r.reason)) @@ -88,7 +88,7 @@ def request_registration( } _print("Sending registration request...") - r = requests.post(url, json=data, verify=False) + r = requests.post(url, json=data) if r.status_code != 200: _print("ERROR! Received %d %s" % (r.status_code, r.reason)) diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 63cf24a14d..7476839db5 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -238,7 +238,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): data[_STREAM_POSITION_KEY] = { "streams": { - stream.NAME: stream.current_token(local_instance_name) + stream.NAME: stream.minimal_local_current_token() for stream in streams }, "instance_name": local_instance_name, diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index d5337fe588..384355698d 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -279,14 +279,6 @@ class ReplicationDataHandler: # may be streaming. self.notifier.notify_replication() - def on_remote_server_up(self, server: str) -> None: - """Called when get a new REMOTE_SERVER_UP command.""" - - # Let's wake up the transaction queue for the server in case we have - # pending stuff to send to it. - if self.send_handler: - self.send_handler.wake_destination(server) - async def wait_for_stream_position( self, instance_name: str, @@ -405,9 +397,6 @@ class FederationSenderHandler: self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer") - def wake_destination(self, server: str) -> None: - self.federation_sender.wake_destination(server) - async def process_replication_rows( self, stream_name: str, token: int, rows: list ) -> None: diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index b668bb5da1..1d586fb180 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -657,8 +657,6 @@ class ReplicationCommandHandler: self, conn: IReplicationConnection, cmd: RemoteServerUpCommand ) -> None: """Called when get a new REMOTE_SERVER_UP command.""" - self._replication_data_handler.on_remote_server_up(cmd.data) - self._notifier.notify_remote_server_up(cmd.data) def on_LOCK_RELEASED( diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index c6088a0f99..5c4d228f3d 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -33,6 +33,7 @@ from synapse.replication.http.streams import ReplicationGetStreamUpdates if TYPE_CHECKING: from synapse.server import HomeServer + from synapse.storage.util.id_generators import AbstractStreamIdGenerator logger = logging.getLogger(__name__) @@ -107,22 +108,10 @@ class Stream: def __init__( self, local_instance_name: str, - current_token_function: Callable[[str], Token], update_function: UpdateFunction, ): """Instantiate a Stream - `current_token_function` and `update_function` are callbacks which - should be implemented by subclasses. - - `current_token_function` takes an instance name, which is a writer to - the stream, and returns the position in the stream of the writer (as - viewed from the current process). On the writer process this is where - the writer has successfully written up to, whereas on other processes - this is the position which we have received updates up to over - replication. (Note that most streams have a single writer and so their - implementations ignore the instance name passed in). - `update_function` is called to get updates for this stream between a pair of stream tokens. See the `UpdateFunction` type definition for more info. @@ -133,12 +122,28 @@ class Stream: update_function: callback go get stream updates, as above """ self.local_instance_name = local_instance_name - self.current_token = current_token_function self.update_function = update_function # The token from which we last asked for updates self.last_token = self.current_token(self.local_instance_name) + def current_token(self, instance_name: str) -> Token: + """This takes an instance name, which is a writer to + the stream, and returns the position in the stream of the writer (as + viewed from the current process). + """ + # We can't make this an abstract class as it makes mypy unhappy. + raise NotImplementedError() + + def minimal_local_current_token(self) -> Token: + """Tries to return a minimal current token for the local instance, + i.e. for writers this would be the last successful write. + + If local instance is not a writer (or has written yet) then falls back + to returning the normal "current token". + """ + raise NotImplementedError() + def discard_updates_and_advance(self) -> None: """Called when the stream should advance but the updates would be discarded, e.g. when there are no currently connected workers. @@ -190,6 +195,25 @@ class Stream: return updates, upto_token, limited +class _StreamFromIdGen(Stream): + """Helper class for simple streams that use a stream ID generator""" + + def __init__( + self, + local_instance_name: str, + update_function: UpdateFunction, + stream_id_gen: "AbstractStreamIdGenerator", + ): + self._stream_id_gen = stream_id_gen + super().__init__(local_instance_name, update_function) + + def current_token(self, instance_name: str) -> Token: + return self._stream_id_gen.get_current_token_for_writer(instance_name) + + def minimal_local_current_token(self) -> Token: + return self._stream_id_gen.get_minimal_local_current_token() + + def current_token_without_instance( current_token: Callable[[], int] ) -> Callable[[str], int]: @@ -242,17 +266,21 @@ class BackfillStream(Stream): self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - self._current_token, self.store.get_all_new_backfill_event_rows, ) - def _current_token(self, instance_name: str) -> int: + def current_token(self, instance_name: str) -> Token: # The backfill stream over replication operates on *positive* numbers, # which means we need to negate it. return -self.store._backfill_id_gen.get_current_token_for_writer(instance_name) + def minimal_local_current_token(self) -> Token: + # The backfill stream over replication operates on *positive* numbers, + # which means we need to negate it. + return -self.store._backfill_id_gen.get_minimal_local_current_token() -class PresenceStream(Stream): + +class PresenceStream(_StreamFromIdGen): @attr.s(slots=True, frozen=True, auto_attribs=True) class PresenceStreamRow: user_id: str @@ -283,9 +311,7 @@ class PresenceStream(Stream): update_function = make_http_update_function(hs, self.NAME) super().__init__( - hs.get_instance_name(), - current_token_without_instance(store.get_current_presence_token), - update_function, + hs.get_instance_name(), update_function, store._presence_id_gen ) @@ -305,13 +331,18 @@ class PresenceFederationStream(Stream): ROW_TYPE = PresenceFederationStreamRow def __init__(self, hs: "HomeServer"): - federation_queue = hs.get_presence_handler().get_federation_queue() + self._federation_queue = hs.get_presence_handler().get_federation_queue() super().__init__( hs.get_instance_name(), - federation_queue.get_current_token, - federation_queue.get_replication_rows, + self._federation_queue.get_replication_rows, ) + def current_token(self, instance_name: str) -> Token: + return self._federation_queue.get_current_token(instance_name) + + def minimal_local_current_token(self) -> Token: + return self._federation_queue.get_current_token(self.local_instance_name) + class TypingStream(Stream): @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -341,20 +372,25 @@ class TypingStream(Stream): update_function: Callable[ [str, int, int, int], Awaitable[Tuple[List[Tuple[int, Any]], int, bool]] ] = typing_writer_handler.get_all_typing_updates - current_token_function = typing_writer_handler.get_current_token + self.current_token_function = typing_writer_handler.get_current_token else: # Query the typing writer process update_function = make_http_update_function(hs, self.NAME) - current_token_function = hs.get_typing_handler().get_current_token + self.current_token_function = hs.get_typing_handler().get_current_token super().__init__( hs.get_instance_name(), - current_token_without_instance(current_token_function), update_function, ) + def current_token(self, instance_name: str) -> Token: + return self.current_token_function() + + def minimal_local_current_token(self) -> Token: + return self.current_token_function() -class ReceiptsStream(Stream): + +class ReceiptsStream(_StreamFromIdGen): @attr.s(slots=True, frozen=True, auto_attribs=True) class ReceiptsStreamRow: room_id: str @@ -371,12 +407,12 @@ class ReceiptsStream(Stream): store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - current_token_without_instance(store.get_max_receipt_stream_id), store.get_all_updated_receipts, + store._receipts_id_gen, ) -class PushRulesStream(Stream): +class PushRulesStream(_StreamFromIdGen): """A user has changed their push rules""" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -387,20 +423,16 @@ class PushRulesStream(Stream): ROW_TYPE = PushRulesStreamRow def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main + store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - self._current_token, - self.store.get_all_push_rule_updates, + store.get_all_push_rule_updates, + store._push_rules_stream_id_gen, ) - def _current_token(self, instance_name: str) -> int: - push_rules_token = self.store.get_max_push_rules_stream_id() - return push_rules_token - -class PushersStream(Stream): +class PushersStream(_StreamFromIdGen): """A user has added/changed/removed a pusher""" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -418,8 +450,8 @@ class PushersStream(Stream): super().__init__( hs.get_instance_name(), - current_token_without_instance(store.get_pushers_stream_token), store.get_all_updated_pushers_rows, + store._pushers_id_gen, ) @@ -447,15 +479,20 @@ class CachesStream(Stream): ROW_TYPE = CachesStreamRow def __init__(self, hs: "HomeServer"): - store = hs.get_datastores().main + self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - store.get_cache_stream_token_for_writer, - store.get_all_updated_caches, + self.store.get_all_updated_caches, ) + def current_token(self, instance_name: str) -> Token: + return self.store.get_cache_stream_token_for_writer(instance_name) + + def minimal_local_current_token(self) -> Token: + return self.current_token(self.local_instance_name) + -class DeviceListsStream(Stream): +class DeviceListsStream(_StreamFromIdGen): """Either a user has updated their devices or a remote server needs to be told about a device update. """ @@ -473,8 +510,8 @@ class DeviceListsStream(Stream): self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - current_token_without_instance(self.store.get_device_stream_token), self._update_function, + self.store._device_list_id_gen, ) async def _update_function( @@ -525,7 +562,7 @@ class DeviceListsStream(Stream): return updates, upper_limit_token, devices_limited or signatures_limited -class ToDeviceStream(Stream): +class ToDeviceStream(_StreamFromIdGen): """New to_device messages for a client""" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -539,12 +576,12 @@ class ToDeviceStream(Stream): store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - current_token_without_instance(store.get_to_device_stream_token), store.get_all_new_device_messages, + store._device_inbox_id_gen, ) -class AccountDataStream(Stream): +class AccountDataStream(_StreamFromIdGen): """Global or per room account data was changed""" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -560,8 +597,8 @@ class AccountDataStream(Stream): self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - current_token_without_instance(self.store.get_max_account_data_stream_id), self._update_function, + self.store._account_data_id_gen, ) async def _update_function( diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index da6d948e1b..38823113d8 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -19,10 +19,10 @@ from typing import TYPE_CHECKING, Iterable, Optional, Tuple, Type, TypeVar, cast import attr from synapse.replication.tcp.streams._base import ( - Stream, StreamRow, StreamUpdateResult, Token, + _StreamFromIdGen, ) if TYPE_CHECKING: @@ -139,7 +139,7 @@ _EventRows: Tuple[Type[BaseEventsStreamRow], ...] = ( TypeToRow = {Row.TypeId: Row for Row in _EventRows} -class EventsStream(Stream): +class EventsStream(_StreamFromIdGen): """We received a new event, or an event went from being an outlier to not""" NAME = "events" @@ -147,9 +147,7 @@ class EventsStream(Stream): def __init__(self, hs: "HomeServer"): self._store = hs.get_datastores().main super().__init__( - hs.get_instance_name(), - self._store._stream_id_gen.get_current_token_for_writer, - self._update_function, + hs.get_instance_name(), self._update_function, self._store._stream_id_gen ) async def _update_function( diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py index 4046bdec69..7f5af5852c 100644 --- a/synapse/replication/tcp/streams/federation.py +++ b/synapse/replication/tcp/streams/federation.py @@ -18,6 +18,7 @@ import attr from synapse.replication.tcp.streams._base import ( Stream, + Token, current_token_without_instance, make_http_update_function, ) @@ -47,7 +48,7 @@ class FederationStream(Stream): # will be a real FederationSender, which has stubs for current_token and # get_replication_rows.) federation_sender = hs.get_federation_sender() - current_token = current_token_without_instance( + self.current_token_func = current_token_without_instance( federation_sender.get_current_token ) update_function: Callable[ @@ -57,15 +58,21 @@ class FederationStream(Stream): elif hs.should_send_federation(): # federation sender: Query master process update_function = make_http_update_function(hs, self.NAME) - current_token = self._stub_current_token + self.current_token_func = self._stub_current_token else: # other worker: stub out the update function (we're not interested in # any updates so when we get a POSITION we do nothing) update_function = self._stub_update_function - current_token = self._stub_current_token + self.current_token_func = self._stub_current_token - super().__init__(hs.get_instance_name(), current_token, update_function) + super().__init__(hs.get_instance_name(), update_function) + + def current_token(self, instance_name: str) -> Token: + return self.current_token_func(instance_name) + + def minimal_local_current_token(self) -> Token: + return self.current_token(self.local_instance_name) @staticmethod def _stub_current_token(instance_name: str) -> int: diff --git a/synapse/replication/tcp/streams/partial_state.py b/synapse/replication/tcp/streams/partial_state.py index a8ce5ffd72..ad181d7e93 100644 --- a/synapse/replication/tcp/streams/partial_state.py +++ b/synapse/replication/tcp/streams/partial_state.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING import attr -from synapse.replication.tcp.streams import Stream +from synapse.replication.tcp.streams._base import _StreamFromIdGen if TYPE_CHECKING: from synapse.server import HomeServer @@ -27,7 +27,7 @@ class UnPartialStatedRoomStreamRow: room_id: str -class UnPartialStatedRoomStream(Stream): +class UnPartialStatedRoomStream(_StreamFromIdGen): """ Stream to notify about rooms becoming un-partial-stated; that is, when the background sync finishes such that we now have full state for @@ -41,8 +41,8 @@ class UnPartialStatedRoomStream(Stream): store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - store.get_un_partial_stated_rooms_token, store.get_un_partial_stated_rooms_from_stream, + store._un_partial_stated_rooms_stream_id_gen, ) @@ -56,7 +56,7 @@ class UnPartialStatedEventStreamRow: rejection_status_changed: bool -class UnPartialStatedEventStream(Stream): +class UnPartialStatedEventStream(_StreamFromIdGen): """ Stream to notify about events becoming un-partial-stated. """ @@ -68,6 +68,6 @@ class UnPartialStatedEventStream(Stream): store = hs.get_datastores().main super().__init__( hs.get_instance_name(), - store.get_un_partial_stated_events_token, store.get_un_partial_stated_events_from_stream, + store._un_partial_stated_events_stream_id_gen, ) diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index 39498d52c6..84ef8136c2 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -94,7 +94,10 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) hs.get_replication_notifier(), "room_account_data", "stream_id", - extra_tables=[("room_tags_revisions", "stream_id")], + extra_tables=[ + ("account_data", "stream_id"), + ("room_tags_revisions", "stream_id"), + ], is_writer=self._instance_name in hs.config.worker.writers.account_data, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 8af638d60f..5bf864c1fb 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -2096,12 +2096,6 @@ class EventsWorkerStore(SQLBaseStore): def _cleanup_old_transaction_ids_txn(txn: LoggingTransaction) -> None: one_day_ago = self._clock.time_msec() - 24 * 60 * 60 * 1000 sql = """ - DELETE FROM event_txn_id - WHERE inserted_ts < ? - """ - txn.execute(sql, (one_day_ago,)) - - sql = """ DELETE FROM event_txn_id_device_id WHERE inserted_ts < ? """ diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 5b50bd66bc..158b528dce 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCHEMA_VERSION = 82 # remember to update the list below when updating +SCHEMA_VERSION = 83 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -121,6 +121,9 @@ Changes in SCHEMA_VERSION = 81 Changes in SCHEMA_VERSION = 82 - The insertion_events, insertion_event_extremities, insertion_event_edges, and batch_events tables are no longer purged in preparation for their removal. + +Changes in SCHEMA_VERSION = 83 + - The event_txn_id is no longer used. """ diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index d2c874b9a8..9c3eafb562 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -134,6 +134,15 @@ class AbstractStreamIdGenerator(metaclass=abc.ABCMeta): raise NotImplementedError() @abc.abstractmethod + def get_minimal_local_current_token(self) -> int: + """Tries to return a minimal current token for the local instance, + i.e. for writers this would be the last successful write. + + If local instance is not a writer (or has written yet) then falls back + to returning the normal "current token". + """ + + @abc.abstractmethod def get_next(self) -> AsyncContextManager[int]: """ Usage: @@ -312,6 +321,9 @@ class StreamIdGenerator(AbstractStreamIdGenerator): def get_current_token_for_writer(self, instance_name: str) -> int: return self.get_current_token() + def get_minimal_local_current_token(self) -> int: + return self.get_current_token() + class MultiWriterIdGenerator(AbstractStreamIdGenerator): """Generates and tracks stream IDs for a stream with multiple writers. @@ -408,6 +420,11 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # The maximum stream ID that we have seen been allocated across any writer. self._max_seen_allocated_stream_id = 1 + # The maximum position of the local instance. This can be higher than + # the corresponding position in `current_positions` table when there are + # no active writes in progress. + self._max_position_of_local_instance = self._max_seen_allocated_stream_id + self._sequence_gen = PostgresSequenceGenerator(sequence_name) # We check that the table and sequence haven't diverged. @@ -427,6 +444,16 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._current_positions.values(), default=1 ) + # For the case where `stream_positions` is not up to date, + # `_persisted_upto_position` may be higher. + self._max_seen_allocated_stream_id = max( + self._max_seen_allocated_stream_id, self._persisted_upto_position + ) + + # Bump our local maximum position now that we've loaded things from the + # DB. + self._max_position_of_local_instance = self._max_seen_allocated_stream_id + if not writers: # If there have been no explicit writers given then any instance can # write to the stream. In which case, let's pre-seed our own @@ -545,6 +572,14 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): if instance == self._instance_name: self._current_positions[instance] = stream_id + if self._writers: + # If we have explicit writers then make sure that each instance has + # a position. + for writer in self._writers: + self._current_positions.setdefault( + writer, self._persisted_upto_position + ) + cur.close() def _load_next_id_txn(self, txn: Cursor) -> int: @@ -688,6 +723,9 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): if new_cur: curr = self._current_positions.get(self._instance_name, 0) self._current_positions[self._instance_name] = max(curr, new_cur) + self._max_position_of_local_instance = max( + curr, new_cur, self._max_position_of_local_instance + ) self._add_persisted_position(next_id) @@ -702,10 +740,26 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): # persisted up to position. This stops Synapse from doing a full table # scan when a new writer announces itself over replication. with self._lock: - return self._return_factor * self._current_positions.get( + if self._instance_name == instance_name: + return self._return_factor * self._max_position_of_local_instance + + pos = self._current_positions.get( instance_name, self._persisted_upto_position ) + # We want to return the maximum "current token" that we can for a + # writer, this helps ensure that streams progress as fast as + # possible. + pos = max(pos, self._persisted_upto_position) + + return self._return_factor * pos + + def get_minimal_local_current_token(self) -> int: + with self._lock: + return self._return_factor * self._current_positions.get( + self._instance_name, self._persisted_upto_position + ) + def get_positions(self) -> Dict[str, int]: """Get a copy of the current positon map. @@ -774,6 +828,18 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._persisted_upto_position = max(min_curr, self._persisted_upto_position) + # Advance our local max position. + self._max_position_of_local_instance = max( + self._max_position_of_local_instance, self._persisted_upto_position + ) + + if not self._unfinished_ids and not self._in_flight_fetches: + # If we don't have anything in flight, it's safe to advance to the + # max seen stream ID. + self._max_position_of_local_instance = max( + self._max_seen_allocated_stream_id, self._max_position_of_local_instance + ) + # We now iterate through the seen positions, discarding those that are # less than the current min positions, and incrementing the min position # if its exactly one greater. diff --git a/synapse/util/file_consumer.py b/synapse/util/file_consumer.py index 46771a401b..26b46be5e1 100644 --- a/synapse/util/file_consumer.py +++ b/synapse/util/file_consumer.py @@ -13,7 +13,7 @@ # limitations under the License. import queue -from typing import BinaryIO, Optional, Union, cast +from typing import Any, BinaryIO, Optional, Union, cast from twisted.internet import threads from twisted.internet.defer import Deferred @@ -58,7 +58,9 @@ class BackgroundFileConsumer: self._bytes_queue: queue.Queue[Optional[bytes]] = queue.Queue() # Deferred that is resolved when finished writing - self._finished_deferred: Optional[Deferred[None]] = None + # + # This is really Deferred[None], but mypy doesn't seem to like that. + self._finished_deferred: Optional[Deferred[Any]] = None # If the _writer thread throws an exception it gets stored here. self._write_exception: Optional[Exception] = None @@ -80,9 +82,13 @@ class BackgroundFileConsumer: self.streaming = streaming self._finished_deferred = run_in_background( threads.deferToThreadPool, - self._reactor, - self._reactor.getThreadPool(), - self._writer, + # mypy seems to get confused with the chaining of ParamSpec from + # run_in_background to deferToThreadPool. + # + # For Twisted trunk, ignore arg-type; for Twisted release ignore unused-ignore. + self._reactor, # type: ignore[arg-type,unused-ignore] + self._reactor.getThreadPool(), # type: ignore[arg-type,unused-ignore] + self._writer, # type: ignore[arg-type,unused-ignore] ) if not streaming: self._producer.resumeProducing() diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 867dbd6001..c888d1ff01 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -156,6 +156,7 @@ class AppServiceHandlerTestCase(unittest.TestCase): result = self.successResultOf( defer.ensureDeferred(self.handler.query_room_alias_exists(room_alias)) ) + assert result is not None self.mock_as_api.query_alias.assert_called_once_with( interested_service, room_alias_str diff --git a/tests/http/server/_base.py b/tests/http/server/_base.py index 36472e57a8..d524c183f8 100644 --- a/tests/http/server/_base.py +++ b/tests/http/server/_base.py @@ -335,7 +335,7 @@ class Deferred__next__Patch: self._request_number = request_number self._seen_awaits = seen_awaits - self._original_Deferred___next__ = Deferred.__next__ + self._original_Deferred___next__ = Deferred.__next__ # type: ignore[misc,unused-ignore] # The number of `await`s on `Deferred`s we have seen so far. self.awaits_seen = 0 diff --git a/tests/http/test_matrixfederationclient.py b/tests/http/test_matrixfederationclient.py index ab94f3f67a..bf1d287699 100644 --- a/tests/http/test_matrixfederationclient.py +++ b/tests/http/test_matrixfederationclient.py @@ -70,7 +70,7 @@ class FederationClientTests(HomeserverTestCase): """ @defer.inlineCallbacks - def do_request() -> Generator["Deferred[object]", object, object]: + def do_request() -> Generator["Deferred[Any]", object, object]: with LoggingContext("one") as context: fetch_d = defer.ensureDeferred( self.cl.get_json("testserv:8008", "foo/bar") diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index 9174fb0964..fd53b0644c 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -259,8 +259,9 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase): id_gen = self._create_id_generator() - # The table is empty so we expect an empty map for positions - self.assertEqual(id_gen.get_positions(), {}) + # The table is empty so we expect the map for positions to have a dummy + # minimum value. + self.assertEqual(id_gen.get_positions(), {"master": 1}) def test_single_instance(self) -> None: """Test that reads and writes from a single process are handled @@ -349,15 +350,12 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase): first_id_gen = self._create_id_generator("first", writers=["first", "second"]) second_id_gen = self._create_id_generator("second", writers=["first", "second"]) - # The first ID gen will notice that it can advance its token to 7 as it - # has no in progress writes... self.assertEqual(first_id_gen.get_positions(), {"first": 3, "second": 7}) - self.assertEqual(first_id_gen.get_current_token_for_writer("first"), 3) + self.assertEqual(first_id_gen.get_current_token_for_writer("first"), 7) self.assertEqual(first_id_gen.get_current_token_for_writer("second"), 7) - # ... but the second ID gen doesn't know that. self.assertEqual(second_id_gen.get_positions(), {"first": 3, "second": 7}) - self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 3) + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 7) self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 7) # Try allocating a new ID gen and check that we only see position @@ -398,6 +396,56 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase): second_id_gen.advance("first", 8) self.assertEqual(second_id_gen.get_positions(), {"first": 8, "second": 9}) + def test_multi_instance_empty_row(self) -> None: + """Test that reads and writes from multiple processes are handled + correctly, when one of the writers starts without any rows. + """ + # Insert some rows for two out of three of the ID gens. + self._insert_rows("first", 3) + self._insert_rows("second", 4) + + first_id_gen = self._create_id_generator( + "first", writers=["first", "second", "third"] + ) + second_id_gen = self._create_id_generator( + "second", writers=["first", "second", "third"] + ) + third_id_gen = self._create_id_generator( + "third", writers=["first", "second", "third"] + ) + + self.assertEqual( + first_id_gen.get_positions(), {"first": 3, "second": 7, "third": 7} + ) + self.assertEqual(first_id_gen.get_current_token_for_writer("first"), 7) + self.assertEqual(first_id_gen.get_current_token_for_writer("second"), 7) + self.assertEqual(first_id_gen.get_current_token_for_writer("third"), 7) + + self.assertEqual( + second_id_gen.get_positions(), {"first": 3, "second": 7, "third": 7} + ) + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 7) + self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 7) + self.assertEqual(second_id_gen.get_current_token_for_writer("third"), 7) + + # Try allocating a new ID gen and check that we only see position + # advanced after we leave the context manager. + + async def _get_next_async() -> None: + async with third_id_gen.get_next() as stream_id: + self.assertEqual(stream_id, 8) + + self.assertEqual( + third_id_gen.get_positions(), {"first": 3, "second": 7, "third": 7} + ) + self.assertEqual(third_id_gen.get_persisted_upto_position(), 7) + + self.get_success(_get_next_async()) + + self.assertEqual( + third_id_gen.get_positions(), {"first": 3, "second": 7, "third": 8} + ) + def test_get_next_txn(self) -> None: """Test that the `get_next_txn` function works correctly.""" @@ -600,6 +648,70 @@ class MultiWriterIdGeneratorTestCase(HomeserverTestCase): with self.assertRaises(IncorrectDatabaseSetup): self._create_id_generator("first") + def test_minimal_local_token(self) -> None: + self._insert_rows("first", 3) + self._insert_rows("second", 4) + + first_id_gen = self._create_id_generator("first", writers=["first", "second"]) + second_id_gen = self._create_id_generator("second", writers=["first", "second"]) + + self.assertEqual(first_id_gen.get_positions(), {"first": 3, "second": 7}) + self.assertEqual(first_id_gen.get_minimal_local_current_token(), 3) + + self.assertEqual(second_id_gen.get_positions(), {"first": 3, "second": 7}) + self.assertEqual(second_id_gen.get_minimal_local_current_token(), 7) + + def test_current_token_gap(self) -> None: + """Test that getting the current token for a writer returns the maximal + token when there are no writes. + """ + self._insert_rows("first", 3) + self._insert_rows("second", 4) + + first_id_gen = self._create_id_generator( + "first", writers=["first", "second", "third"] + ) + second_id_gen = self._create_id_generator( + "second", writers=["first", "second", "third"] + ) + + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 7) + self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 7) + self.assertEqual(second_id_gen.get_current_token(), 7) + + # Check that the first ID gen advancing causes the second ID gen to + # advance (as the second ID gen has nothing in flight). + + async def _get_next_async() -> None: + async with first_id_gen.get_next_mult(2): + pass + + self.get_success(_get_next_async()) + second_id_gen.advance("first", 9) + + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 9) + self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 9) + self.assertEqual(second_id_gen.get_current_token(), 7) + + # Check that the first ID gen advancing doesn't advance the second ID + # gen when the second ID gen has stuff in flight. + self.get_success(_get_next_async()) + + ctxmgr = second_id_gen.get_next() + self.get_success(ctxmgr.__aenter__()) + + second_id_gen.advance("first", 11) + + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 11) + self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 9) + self.assertEqual(second_id_gen.get_current_token(), 7) + + self.get_success(ctxmgr.__aexit__(None, None, None)) + + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 11) + self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 12) + self.assertEqual(second_id_gen.get_current_token(), 7) + class BackwardsMultiWriterIdGeneratorTestCase(HomeserverTestCase): """Tests MultiWriterIdGenerator that produce *negative* stream IDs.""" @@ -712,8 +824,8 @@ class BackwardsMultiWriterIdGeneratorTestCase(HomeserverTestCase): self.get_success(_get_next_async()) - self.assertEqual(id_gen_1.get_positions(), {"first": -1}) - self.assertEqual(id_gen_2.get_positions(), {"first": -1}) + self.assertEqual(id_gen_1.get_positions(), {"first": -1, "second": -1}) + self.assertEqual(id_gen_2.get_positions(), {"first": -1, "second": -1}) self.assertEqual(id_gen_1.get_persisted_upto_position(), -1) self.assertEqual(id_gen_2.get_persisted_upto_position(), -1) @@ -822,11 +934,11 @@ class MultiTableMultiWriterIdGeneratorTestCase(HomeserverTestCase): second_id_gen = self._create_id_generator("second", writers=["first", "second"]) self.assertEqual(first_id_gen.get_positions(), {"first": 3, "second": 6}) - self.assertEqual(first_id_gen.get_current_token_for_writer("first"), 3) - self.assertEqual(first_id_gen.get_current_token_for_writer("second"), 6) + self.assertEqual(first_id_gen.get_current_token_for_writer("first"), 7) + self.assertEqual(first_id_gen.get_current_token_for_writer("second"), 7) self.assertEqual(first_id_gen.get_persisted_upto_position(), 7) self.assertEqual(second_id_gen.get_positions(), {"first": 3, "second": 7}) - self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 3) + self.assertEqual(second_id_gen.get_current_token_for_writer("first"), 7) self.assertEqual(second_id_gen.get_current_token_for_writer("second"), 7) self.assertEqual(second_id_gen.get_persisted_upto_position(), 7) diff --git a/tests/unittest.py b/tests/unittest.py index 99ad02eb06..79c47fc3cc 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -30,6 +30,7 @@ from typing import ( Generic, Iterable, List, + Mapping, NoReturn, Optional, Tuple, @@ -251,7 +252,7 @@ class TestCase(unittest.TestCase): except AssertionError as e: raise (type(e))(f"Assert error for '.{key}':") from e - def assert_dict(self, required: dict, actual: dict) -> None: + def assert_dict(self, required: Mapping, actual: Mapping) -> None: """Does a partial assert of a dict. Args: |