diff --git a/contrib/docker_compose_workers/README.md b/contrib/docker_compose_workers/README.md
index 4dbfee2853..bdd3dd32e0 100644
--- a/contrib/docker_compose_workers/README.md
+++ b/contrib/docker_compose_workers/README.md
@@ -94,20 +94,6 @@ worker_replication_host: synapse
worker_replication_http_port: 9093
```
-### Add Workers to `instance_map`
-
-Locate the `instance_map` section of your `homeserver.yaml` and populate it with your workers:
-
-```yaml
-instance_map:
- synapse-generic-worker-1: # The worker_name setting in your worker configuration file
- host: synapse-generic-worker-1 # The name of the worker service in your Docker Compose file
- port: 8034 # The port assigned to the replication listener in your worker config file
- synapse-federation-sender-1:
- host: synapse-federation-sender-1
- port: 8034
-```
-
### Configure Federation Senders
This section is applicable if you are using Federation senders (synapse.app.federation_sender). Locate the `send_federation` and `federation_sender_instances` settings in your `homeserver.yaml` and configure them:
@@ -122,4 +108,4 @@ federation_sender_instances:
## Other Worker types
-Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers.
\ No newline at end of file
+Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers.
diff --git a/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml b/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml
index 5ba42a92d2..5b40664d67 100644
--- a/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml
+++ b/contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml
@@ -5,10 +5,4 @@ worker_name: synapse-federation-sender-1
worker_replication_host: synapse
worker_replication_http_port: 9093
-worker_listeners:
- - type: http
- port: 8034
- resources:
- - names: [replication]
-
worker_log_config: /data/federation_sender.log.config
diff --git a/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml b/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml
index 694584105a..09e55df9f3 100644
--- a/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml
+++ b/contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml
@@ -7,10 +7,6 @@ worker_replication_http_port: 9093
worker_listeners:
- type: http
- port: 8034
- resources:
- - names: [replication]
- - type: http
port: 8081
x_forwarded: true
resources:
diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index 819426b8ea..68705b6e6d 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -9,17 +9,18 @@
"pluginName": "Prometheus"
}
],
+ "__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
- "version": "7.3.7"
+ "version": "9.2.2"
},
{
"type": "panel",
"id": "graph",
- "name": "Graph",
+ "name": "Graph (old)",
"version": ""
},
{
@@ -33,13 +34,21 @@
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"enable": false,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
@@ -51,10 +60,9 @@
]
},
"editable": true,
- "gnetId": null,
+ "fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
- "iteration": 1628606819564,
"links": [
{
"asDropdown": false,
@@ -66,24 +74,16 @@
],
"title": "Dashboards",
"type": "dashboards"
- },
- {
- "asDropdown": false,
- "icon": "external link",
- "includeVars": false,
- "keepTime": false,
- "tags": [],
- "targetBlank": true,
- "title": "Synapse Documentation",
- "tooltip": "Open Documentation",
- "type": "link",
- "url": "https://matrix-org.github.io/synapse/latest/"
}
],
+ "liveNow": false,
"panels": [
{
"collapsed": false,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
@@ -92,6 +92,15 @@
},
"id": 73,
"panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Overview",
"type": "row"
},
@@ -108,10 +117,21 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {}
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
},
"overrides": []
},
@@ -129,9 +149,51 @@
"show": false
},
"links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellRadius": 0,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
"reverseYBuckets": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
"format": "heatmap",
"interval": "",
@@ -149,31 +211,24 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
- "decimals": null,
"format": "s",
"logBase": 2,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -207,7 +262,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -266,6 +321,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -273,6 +331,9 @@
"refId": "D"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
@@ -281,6 +342,9 @@
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -288,6 +352,9 @@
"refId": "C"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -295,22 +362,34 @@
"refId": "B"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"legendFormat": "25%",
"refId": "F"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"legendFormat": "5%",
"refId": "G"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
"legendFormat": "Average",
"refId": "H"
},
{
- "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))",
"hide": false,
"instant": false,
"legendFormat": "Events",
@@ -319,6 +398,7 @@
],
"thresholds": [
{
+ "$$hashKey": "object:283",
"colorMode": "warning",
"fill": false,
"line": true,
@@ -327,6 +407,7 @@
"yaxis": "left"
},
{
+ "$$hashKey": "object:284",
"colorMode": "critical",
"fill": false,
"line": true,
@@ -335,9 +416,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Event Send Time Quantiles (excluding errors, all workers)",
"tooltip": {
"shared": true,
@@ -346,34 +425,30 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
+ "$$hashKey": "object:255",
"format": "s",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:256",
"format": "hertz",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -381,10 +456,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -417,7 +493,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -427,6 +503,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
@@ -437,6 +516,7 @@
],
"thresholds": [
{
+ "$$hashKey": "object:566",
"colorMode": "critical",
"fill": true,
"line": true,
@@ -445,9 +525,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "CPU usage",
"tooltip": {
"shared": false,
@@ -456,34 +534,28 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
+ "$$hashKey": "object:538",
"format": "percentunit",
- "label": null,
"logBase": 1,
"max": "1.5",
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:539",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -491,12 +563,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -530,7 +603,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -540,6 +613,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
@@ -550,6 +626,9 @@
"target": ""
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
"hide": true,
"interval": "",
@@ -558,9 +637,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Memory",
"tooltip": {
"shared": false,
@@ -570,31 +647,27 @@
"transformations": [],
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:1560",
"format": "bytes",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:1561",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -602,10 +675,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -638,12 +712,13 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
+ "$$hashKey": "object:639",
"alias": "/max$/",
"color": "#890F02",
"fill": 0,
@@ -655,6 +730,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -665,6 +743,9 @@
"step": 20
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": true,
@@ -676,9 +757,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Open FDs",
"tooltip": {
"shared": false,
@@ -687,40 +766,35 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
+ "$$hashKey": "object:650",
"format": "none",
"label": "",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
- "decimals": null,
+ "$$hashKey": "object:651",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"gridPos": {
"h": 1,
"w": 24,
@@ -734,12 +808,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -751,7 +826,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 25
+ "y": 27
},
"hiddenSeries": false,
"id": 5,
@@ -777,15 +852,17 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
+ "$$hashKey": "object:1240",
"alias": "/user/"
},
{
+ "$$hashKey": "object:1241",
"alias": "/system/"
}
],
@@ -794,6 +871,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 1,
@@ -803,6 +883,9 @@
"step": 20
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
@@ -815,6 +898,7 @@
],
"thresholds": [
{
+ "$$hashKey": "object:1278",
"colorMode": "custom",
"fillColor": "rgba(255, 255, 255, 1)",
"line": true,
@@ -824,6 +908,7 @@
"yaxis": "left"
},
{
+ "$$hashKey": "object:1279",
"colorMode": "custom",
"fillColor": "rgba(255, 255, 255, 1)",
"line": true,
@@ -833,6 +918,7 @@
"yaxis": "left"
},
{
+ "$$hashKey": "object:1498",
"colorMode": "critical",
"fill": true,
"line": true,
@@ -841,9 +927,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "CPU",
"tooltip": {
"shared": false,
@@ -852,15 +936,13 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
+ "$$hashKey": "object:1250",
"format": "percentunit",
"label": "",
"logBase": 1,
@@ -869,71 +951,117 @@
"show": true
},
{
+ "$$hashKey": "object:1251",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
"fieldConfig": {
"defaults": {
- "custom": {},
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 25
+ "y": 27
},
- "hiddenSeries": false,
"id": 105,
"interval": "",
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [],
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
},
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.3.2",
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "histogram_quantile(0.999, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} 99.9%",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
"expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
@@ -943,13 +1071,23 @@
"step": 20
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
"expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
"format": "time_series",
+ "interval": "",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} 95%",
"refId": "B"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
@@ -957,6 +1095,10 @@
"refId": "C"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 1,
@@ -964,58 +1106,21 @@
"refId": "D"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "Reactor tick quantiles",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -1027,7 +1132,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 32
+ "y": 34
},
"hiddenSeries": false,
"id": 34,
@@ -1049,7 +1154,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1059,6 +1164,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
@@ -1069,6 +1177,9 @@
"target": ""
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
"interval": "",
"legendFormat": "total",
@@ -1076,9 +1187,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Memory",
"tooltip": {
"shared": false,
@@ -1088,9 +1197,7 @@
"transformations": [],
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -1098,21 +1205,17 @@
{
"format": "bytes",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1120,10 +1223,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -1134,7 +1238,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 32
+ "y": 34
},
"hiddenSeries": false,
"id": 49,
@@ -1156,7 +1260,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1172,6 +1276,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
@@ -1182,9 +1289,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Prometheus scrape time",
"tooltip": {
"shared": false,
@@ -1193,18 +1298,14 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
@@ -1219,8 +1320,7 @@
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1228,10 +1328,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -1242,7 +1343,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 39
+ "y": 41
},
"hiddenSeries": false,
"id": 53,
@@ -1264,7 +1365,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1274,6 +1375,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
@@ -1282,9 +1386,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Up",
"tooltip": {
"shared": false,
@@ -1293,33 +1395,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1327,10 +1420,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -1341,7 +1435,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 39
+ "y": 41
},
"hiddenSeries": false,
"id": 120,
@@ -1362,7 +1456,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -1372,6 +1466,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
@@ -1381,7 +1478,10 @@
"refId": "A"
},
{
- "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"instant": false,
@@ -1401,9 +1501,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Stacked CPU usage",
"tooltip": {
"shared": false,
@@ -1412,33 +1510,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:572",
"format": "percentunit",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:573",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1446,10 +1537,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -1460,7 +1553,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 46
+ "y": 48
},
"hiddenSeries": false,
"id": 136,
@@ -1481,7 +1574,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -1491,20 +1584,28 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_client_requests{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_client_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"legendFormat": "{{job}}-{{index}} {{method}}",
+ "range": true,
"refId": "A"
},
{
- "expr": "rate(synapse_http_matrixfederationclient_requests{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_http_matrixfederationclient_requests_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"legendFormat": "{{job}}-{{index}} {{method}} (federation)",
+ "range": true,
"refId": "B"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Outgoing HTTP request rate",
"tooltip": {
"shared": false,
@@ -1513,43 +1614,139 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:123",
"format": "reqps",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:124",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "active threads",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 48
+ },
+ "id": 207,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "synapse_threadpool_working_threads{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Threadpool activity",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "refId": "A"
}
],
- "repeat": null,
"title": "Process info",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"gridPos": {
"h": 1,
"w": 24,
@@ -1571,10 +1768,21 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {}
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
},
"overrides": []
},
@@ -1582,7 +1790,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 21
+ "y": 56
},
"heatmap": {},
"hideZeroBuckets": false,
@@ -1592,9 +1800,51 @@
"show": false
},
"links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellRadius": 0,
+ "cellValues": {},
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
"reverseYBuckets": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
"format": "heatmap",
"intervalFactor": 1,
@@ -1611,33 +1861,26 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
- "decimals": null,
"format": "s",
"logBase": 2,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -1649,7 +1892,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 21
+ "y": 56
},
"hiddenSeries": false,
"id": 33,
@@ -1671,7 +1914,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1681,7 +1924,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1692,9 +1938,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Events Persisted (all workers)",
"tooltip": {
"shared": true,
@@ -1703,31 +1947,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:102",
"format": "hertz",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:103",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1735,21 +1974,17 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": 1,
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
+ "decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 30
+ "y": 65
},
"hiddenSeries": false,
"id": 40,
@@ -1770,7 +2005,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1780,6 +2015,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
@@ -1788,9 +2026,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Events/s Local vs Remote",
"tooltip": {
"shared": true,
@@ -1799,9 +2035,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -1810,22 +2044,17 @@
"format": "hertz",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1833,21 +2062,17 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": 1,
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
+ "decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 30
+ "y": 65
},
"hiddenSeries": false,
"id": 46,
@@ -1868,7 +2093,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1878,6 +2103,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"instant": false,
@@ -1888,9 +2116,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Events/s by Type",
"tooltip": {
"shared": false,
@@ -1899,33 +2125,25 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -1935,21 +2153,17 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": 1,
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
+ "decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 37
+ "y": 72
},
"hiddenSeries": false,
"id": 44,
@@ -1973,7 +2187,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1983,6 +2197,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
@@ -1992,9 +2209,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Events/s by Origin",
"tooltip": {
"shared": false,
@@ -2003,33 +2218,25 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2037,21 +2244,18 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": 1,
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
+ "decimals": 1,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 37
+ "y": 72
},
"hiddenSeries": false,
"id": 45,
@@ -2075,7 +2279,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2085,18 +2289,21 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_storage_events_persisted_events_sep{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(synapse_storage_events_persisted_events_sep_total{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\", origin_type=\"local\"}[$bucket_size])) by (origin_type, origin_entity)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{origin_entity}} ({{origin_type}})",
+ "range": true,
"refId": "A",
"step": 20
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Memberships/s by Origin",
"tooltip": {
"shared": true,
@@ -2105,33 +2312,27 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:232",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:233",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2139,10 +2340,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2153,7 +2356,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 44
+ "y": 79
},
"hiddenSeries": false,
"id": 118,
@@ -2175,13 +2378,14 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeatDirection": "h",
"seriesOverrides": [
{
+ "$$hashKey": "object:316",
"alias": "mean",
"linewidth": 2
}
@@ -2191,6 +2395,11 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"interval": "",
@@ -2199,6 +2408,10 @@
"refId": "A"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"interval": "",
@@ -2207,6 +2420,10 @@
"refId": "B"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"intervalFactor": 1,
@@ -2214,6 +2431,10 @@
"refId": "C"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
"format": "time_series",
"intervalFactor": 1,
@@ -2221,17 +2442,21 @@
"refId": "D"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
"expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} mean",
+ "range": true,
"refId": "E"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Event send time quantiles by worker",
"tooltip": {
"shared": true,
@@ -2240,43 +2465,158 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:263",
"format": "s",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:264",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "CPU and DB time spent on most expensive state resolution in a room, summed over all workers. This is a very rough proxy for \"how fast is state res\", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 30,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s/s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 79
+ },
+ "id": 222,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "DB time",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": false,
+ "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "interval": "",
+ "legendFormat": "CPU time",
+ "refId": "C"
+ }
+ ],
+ "title": "Stateres worst-case",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "refId": "A"
}
],
- "repeat": null,
"title": "Event persistence",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"gridPos": {
"h": 1,
"w": 24,
@@ -2290,13 +2630,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": null,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2308,7 +2649,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 31
+ "y": 57
},
"hiddenSeries": false,
"id": 4,
@@ -2333,7 +2674,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2343,7 +2684,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -2354,6 +2698,7 @@
],
"thresholds": [
{
+ "$$hashKey": "object:234",
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
@@ -2362,6 +2707,7 @@
"yaxis": "left"
},
{
+ "$$hashKey": "object:235",
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
@@ -2370,9 +2716,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Request Count by arrival time",
"tooltip": {
"shared": false,
@@ -2381,31 +2725,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:206",
"format": "hertz",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:207",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2413,12 +2752,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2430,7 +2771,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 31
+ "y": 57
},
"hiddenSeries": false,
"id": 32,
@@ -2451,7 +2792,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2461,7 +2802,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
@@ -2471,9 +2815,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Top 10 Request Counts",
"tooltip": {
"shared": false,
@@ -2482,31 +2824,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:305",
"format": "hertz",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:306",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2514,13 +2851,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": null,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2532,7 +2870,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 39
+ "y": 65
},
"hiddenSeries": false,
"id": 139,
@@ -2557,7 +2895,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2567,7 +2905,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -2578,6 +2919,7 @@
],
"thresholds": [
{
+ "$$hashKey": "object:135",
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
@@ -2586,6 +2928,7 @@
"yaxis": "left"
},
{
+ "$$hashKey": "object:136",
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
@@ -2594,9 +2937,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Total CPU Usage by Endpoint",
"tooltip": {
"shared": false,
@@ -2605,31 +2946,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:107",
"format": "percentunit",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:108",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2637,13 +2973,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": null,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2655,7 +2992,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 39
+ "y": 65
},
"hiddenSeries": false,
"id": 52,
@@ -2680,7 +3017,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2690,7 +3027,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -2701,6 +3041,7 @@
],
"thresholds": [
{
+ "$$hashKey": "object:417",
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
@@ -2709,6 +3050,7 @@
"yaxis": "left"
},
{
+ "$$hashKey": "object:418",
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
@@ -2717,9 +3059,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average CPU Usage by Endpoint",
"tooltip": {
"shared": false,
@@ -2728,31 +3068,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:389",
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:390",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2760,12 +3095,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2777,7 +3114,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 47
+ "y": 73
},
"hiddenSeries": false,
"id": 7,
@@ -2801,7 +3138,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2811,7 +3148,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -2821,9 +3161,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "DB Usage by endpoint",
"tooltip": {
"shared": false,
@@ -2832,31 +3170,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:488",
"format": "percentunit",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:489",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2864,13 +3197,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "decimals": null,
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2882,7 +3215,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 47
+ "y": 73
},
"hiddenSeries": false,
"id": 47,
@@ -2907,7 +3240,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2917,6 +3250,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "(sum(rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))/(sum(rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])) without (code))",
"format": "time_series",
"hide": false,
@@ -2928,9 +3264,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Non-sync avg response time",
"tooltip": {
"shared": false,
@@ -2939,9 +3273,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -2949,21 +3281,16 @@
{
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": false
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -2971,10 +3298,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -2985,7 +3313,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 55
+ "y": 81
},
"hiddenSeries": false,
"id": 103,
@@ -3006,7 +3334,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3023,6 +3351,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
"format": "time_series",
"interval": "",
@@ -3031,6 +3362,9 @@
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Total",
@@ -3038,9 +3372,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Requests in flight",
"tooltip": {
"shared": false,
@@ -3049,43 +3381,45 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
- "repeat": null,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Requests",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"gridPos": {
"h": 1,
"w": 24,
@@ -3099,10 +3433,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3113,7 +3448,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 32
+ "y": 5
},
"hiddenSeries": false,
"id": 99,
@@ -3130,9 +3465,12 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "8.4.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3142,7 +3480,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -3151,9 +3492,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "CPU usage by background jobs",
"tooltip": {
"shared": false,
@@ -3162,33 +3501,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "percentunit",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3196,10 +3526,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3210,7 +3541,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 32
+ "y": 5
},
"hiddenSeries": false,
"id": 101,
@@ -3227,9 +3558,12 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "8.4.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3239,7 +3573,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
@@ -3248,9 +3585,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "DB usage by background jobs (including scheduling time)",
"tooltip": {
"shared": false,
@@ -3259,33 +3594,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "percentunit",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3293,10 +3619,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3307,7 +3634,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 41
+ "y": 14
},
"hiddenSeries": false,
"id": 138,
@@ -3323,8 +3650,11 @@
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "8.4.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -3334,15 +3664,16 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_background_process_in_flight_count{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Background jobs in flight",
"tooltip": {
"shared": false,
@@ -3351,42 +3682,45 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Background jobs",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"gridPos": {
"h": 1,
"w": 24,
@@ -3400,10 +3734,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3414,7 +3749,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 6
+ "y": 59
},
"hiddenSeries": false,
"id": 79,
@@ -3436,7 +3771,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3446,22 +3781,26 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "successful txn rate",
"refId": "A"
},
{
- "expr": "sum(rate(synapse_util_metrics_block_count{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))",
"legendFormat": "failed txn rate",
"refId": "B"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Outgoing federation transaction rate",
"tooltip": {
"shared": true,
@@ -3470,33 +3809,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3504,10 +3834,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3518,7 +3849,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 6
+ "y": 59
},
"hiddenSeries": false,
"id": 83,
@@ -3540,7 +3871,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3550,14 +3881,20 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_federation_server_received_pdus{instance=~\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "pdus",
"refId": "A"
},
{
- "expr": "sum(rate(synapse_federation_server_received_edus{instance=~\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "edus",
@@ -3565,9 +3902,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Incoming PDU/EDU rate",
"tooltip": {
"shared": true,
@@ -3576,33 +3911,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3610,10 +3936,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3624,7 +3951,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 15
+ "y": 68
},
"hiddenSeries": false,
"id": 109,
@@ -3646,7 +3973,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3656,7 +3983,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations:total{instance=\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations:total_total{instance=\"$instance\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -3664,7 +3994,10 @@
"refId": "A"
},
{
- "expr": "sum(rate(synapse_federation_client_sent_edus{instance=\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "edus",
@@ -3672,9 +4005,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Outgoing PDU/EDU rate",
"tooltip": {
"shared": true,
@@ -3683,33 +4014,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3717,10 +4039,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3731,7 +4055,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 15
+ "y": 68
},
"hiddenSeries": false,
"id": 111,
@@ -3753,7 +4077,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3763,18 +4087,21 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_federation_client_sent_edus_by_type{instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_federation_client_sent_edus_by_type_total{instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{type}}",
+ "range": true,
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Outgoing EDUs by type",
"tooltip": {
"shared": true,
@@ -3783,33 +4110,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:462",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:463",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3817,11 +4137,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"description": "The number of events in the in-memory queues ",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3832,7 +4154,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 24
+ "y": 77
},
"hiddenSeries": false,
"id": 142,
@@ -3852,7 +4174,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -3862,12 +4184,22 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
"expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "pending PDUs {{job}}-{{index}}",
+ "range": true,
"refId": "A"
},
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "pending EDUs {{job}}-{{index}}",
@@ -3875,9 +4207,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "In-memory federation transmission queues",
"tooltip": {
"shared": true,
@@ -3886,33 +4216,29 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:547",
"format": "short",
"label": "events",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:548",
"format": "short",
"label": "",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -3920,11 +4246,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Number of events queued up on the master process for processing by the federation sender",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -3935,7 +4262,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 24
+ "y": 77
},
"hiddenSeries": false,
"id": 140,
@@ -3957,7 +4284,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3967,6 +4294,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_send_queue_presence_changed_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
@@ -3975,6 +4305,9 @@
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_send_queue_presence_map_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -3984,6 +4317,9 @@
"refId": "B"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_send_queue_presence_destinations_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -3993,6 +4329,9 @@
"refId": "E"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_send_queue_keyed_edu_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -4002,6 +4341,9 @@
"refId": "C"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_send_queue_edus_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -4011,6 +4353,9 @@
"refId": "D"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_send_queue_pos_time_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -4021,9 +4366,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Outgoing EDU queues on master",
"tooltip": {
"shared": true,
@@ -4032,39 +4375,30 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "none",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": -1,
- "cardRound": null
+ "cardPadding": -1
},
"color": {
"cardColor": "#b4ff00",
@@ -4075,10 +4409,21 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {}
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "scaleDistribution": {
+ "type": "linear"
+ }
+ }
},
"overrides": []
},
@@ -4086,7 +4431,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 32
+ "y": 85
},
"heatmap": {},
"hideZeroBuckets": false,
@@ -4096,9 +4441,54 @@
"show": false
},
"links": [],
+ "options": {
+ "calculate": false,
+ "calculation": {},
+ "cellGap": -1,
+ "cellValues": {
+ "decimals": 2
+ },
+ "color": {
+ "exponent": 0.5,
+ "fill": "#b4ff00",
+ "min": 0,
+ "mode": "scheme",
+ "reverse": false,
+ "scale": "exponential",
+ "scheme": "Inferno",
+ "steps": 128
+ },
+ "exemplars": {
+ "color": "rgba(255,0,255,0.7)"
+ },
+ "filterValues": {
+ "le": 1e-9
+ },
+ "legend": {
+ "show": false
+ },
+ "rowsFrame": {
+ "layout": "auto"
+ },
+ "showValue": "never",
+ "tooltip": {
+ "show": true,
+ "yHistogram": true
+ },
+ "yAxis": {
+ "axisPlacement": "left",
+ "decimals": 0,
+ "reverse": false,
+ "unit": "s"
+ }
+ },
+ "pluginVersion": "9.2.2",
"reverseYBuckets": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
"format": "heatmap",
"instant": false,
@@ -4118,30 +4508,24 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -4152,7 +4536,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 33
+ "y": 86
},
"hiddenSeries": false,
"id": 162,
@@ -4175,7 +4559,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4226,6 +4610,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
@@ -4234,6 +4621,9 @@
"refId": "D"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
@@ -4242,6 +4632,9 @@
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
@@ -4250,6 +4643,9 @@
"refId": "C"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
"format": "time_series",
"interval": "",
@@ -4258,18 +4654,27 @@
"refId": "B"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
"interval": "",
"legendFormat": "25%",
"refId": "F"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
"interval": "",
"legendFormat": "5%",
"refId": "G"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Average",
@@ -4294,9 +4699,7 @@
"yaxis": "left"
}
],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Federation send PDU lag quantiles",
"tooltip": {
"shared": true,
@@ -4305,19 +4708,15 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "s",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
@@ -4325,20 +4724,17 @@
"format": "hertz",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": -1,
- "cardRound": null
+ "cardPadding": -1
},
"color": {
"cardColor": "#b4ff00",
@@ -4349,18 +4745,14 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 41
+ "y": 94
},
"heatmap": {},
"hideZeroBuckets": false,
@@ -4373,6 +4765,9 @@
"reverseYBuckets": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
"format": "heatmap",
"instant": false,
@@ -4392,32 +4787,26 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -4429,7 +4818,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 42
+ "y": 95
},
"hiddenSeries": false,
"id": 203,
@@ -4451,7 +4840,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4461,6 +4850,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
"format": "time_series",
"interval": "",
@@ -4471,9 +4863,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Age of oldest event in staging area",
"tooltip": {
"msResolution": false,
@@ -4483,33 +4873,27 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:92",
"format": "ms",
- "label": null,
"logBase": 1,
- "max": null,
"min": 0,
"show": true
},
{
+ "$$hashKey": "object:93",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -4517,12 +4901,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -4534,7 +4919,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 50
+ "y": 103
},
"hiddenSeries": false,
"id": 202,
@@ -4556,7 +4941,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4566,6 +4951,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
"format": "time_series",
"interval": "",
@@ -4576,9 +4964,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Number of events in federation staging area",
"tooltip": {
"msResolution": false,
@@ -4588,33 +4974,27 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:92",
"format": "none",
- "label": null,
"logBase": 1,
- "max": null,
"min": 0,
"show": true
},
{
+ "$$hashKey": "object:93",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -4622,12 +5002,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -4635,7 +5012,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 51
+ "y": 104
},
"hiddenSeries": false,
"id": 205,
@@ -4655,7 +5032,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -4665,6 +5042,10 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
"interval": "",
"legendFormat": "soft-failed events",
@@ -4672,9 +5053,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Soft-failed event rate",
"tooltip": {
"shared": true,
@@ -4683,48 +5062,776 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:131",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:132",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": false
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Federation",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 31
},
+ "id": 227,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "id": 239,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_reject_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of rate limit rejected requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "id": 235,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of requests being slept by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Why is the data zero (0)? https://github.com/matrix-org/synapse/pull/13541#discussion_r951926322",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "id": 237,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_reject_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of hosts being rejected by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "https://github.com/matrix-org/synapse/pull/13541",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "id": 233,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(increase(synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ }
+ ],
+ "title": "Number of hosts being slept by the rate limiter",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 17
+ },
+ "hiddenSeries": false,
+ "id": 229,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "9.0.4",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:276",
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "$$hashKey": "object:277",
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "$$hashKey": "object:278",
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "$$hashKey": "object:279",
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "$$hashKey": "object:280",
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "$$hashKey": "object:281",
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "$$hashKey": "object:282",
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "$$hashKey": "object:283",
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ },
+ {
+ "$$hashKey": "object:284",
+ "alias": ">99%",
+ "color": "#B877D9",
+ "fill": 3,
+ "lines": true
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9995, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 1,
+ "legendFormat": ">99%",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_rate_limit_queue_wait_time_seconds_bucket{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_rate_limit_queue_wait_time_seconds_sum{index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_rate_limit_queue_wait_time_seconds_count{index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "legendFormat": "Average",
+ "refId": "H"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:283",
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:284",
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 2,
+ "yaxis": "left"
+ }
+ ],
+ "timeRegions": [],
+ "title": "Rate limit queue wait time Quantiles (all workers)",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:255",
+ "format": "s",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "$$hashKey": "object:256",
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "line"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Default reject threshold (50 requests within a second)"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "red",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 2
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 17
+ },
+ "id": 231,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "sum(rate(synapse_rate_limit_sleep_total{instance=\"$instance\"}[$bucket_size]))",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "code",
+ "expr": "50",
+ "hide": false,
+ "legendFormat": "Default reject threshold (50 requests within a second)",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Rate of requests being slept by the rate limiter",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Federation rate limiter",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 32
+ },
"id": 60,
"panels": [
{
@@ -4732,10 +5839,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -4746,7 +5854,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 34
+ "y": 32
},
"hiddenSeries": false,
"id": 51,
@@ -4763,9 +5871,12 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "8.4.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4775,7 +5886,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -4784,7 +5898,10 @@
"step": 20
},
{
- "expr": "rate(synapse_http_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "failed {{job}}",
@@ -4793,9 +5910,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "HTTP Push rate",
"tooltip": {
"shared": true,
@@ -4804,33 +5919,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -4838,11 +5944,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -4853,7 +5960,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 34
+ "y": 32
},
"hiddenSeries": false,
"id": 134,
@@ -4870,8 +5977,11 @@
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "8.4.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -4881,15 +5991,16 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "topk(10,synapse_pushers{job=~\"$job\",index=~\"$index\", instance=\"$instance\"})",
"legendFormat": "{{kind}} {{app_id}}",
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Active pusher instances by app",
"tooltip": {
"shared": false,
@@ -4898,60 +6009,65 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
- "repeat": null,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Pushes",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 32
+ "y": 33
},
- "id": 58,
+ "id": 219,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How many entries in current state that we are iterating over while calculating push rules.",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -4965,13 +6081,114 @@
"y": 33
},
"hiddenSeries": false,
- "id": 48,
+ "id": 209,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{index}}",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Iterations over State",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
"show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "Rate that the cached push rules for a room get invalidated due to underlying push rules being changed. ",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 33
+ },
+ "hiddenSeries": false,
+ "id": 211,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
"total": false,
"values": false
},
@@ -4984,7 +6201,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "8.4.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4994,19 +6211,24 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
"format": "time_series",
+ "interval": "",
"intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
+ "legendFormat": "{{index}}",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
"refId": "A",
- "step": 20
+ "step": 2
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
- "title": "Avg time waiting for db conn",
+ "title": "Push Rule Invalidations",
"tooltip": {
"shared": true,
"sort": 0,
@@ -5014,34 +6236,149 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
- "format": "s",
+ "format": "hertz",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
- "show": false
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often the \"delta optimisation\" happens.\n\nThe delta optimisation is when we update the push rules for a room incrementally after a state change where we know the delta between the old state and the new state.\n\nThis can't happen if we don't the delta or we're calculating push rules from scratch.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 213,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Delta Optimisation",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5049,11 +6386,374 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often we have the correct cached push rules for a room.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 40
+ },
+ "hiddenSeries": false,
+ "id": 215,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "How often we reuse existing calculated push rules",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "How often we have existing cached push rules for the room. \n\nNote that these might be outdated and need to be recalculated if the state has changed.",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "hiddenSeries": false,
+ "id": 217,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "8.4.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Number of calls",
+ "yaxis": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Hit Rate",
+ "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total",
+ "refId": "A",
+ "step": 2
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "Number of calls",
+ "refId": "B",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "How often we have the RulesForRoom cached",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "hertz",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Push Rule Cache",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 58,
+ "panels": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "id": 48,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "title": "Avg time waiting for db conn",
+ "type": "timeseries"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5064,7 +6764,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 33
+ "y": 35
},
"hiddenSeries": false,
"id": 104,
@@ -5087,7 +6787,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5097,6 +6797,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"hide": false,
@@ -5106,6 +6809,9 @@
"step": 20
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
@@ -5113,6 +6819,9 @@
"refId": "B"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"intervalFactor": 1,
@@ -5120,6 +6829,9 @@
"refId": "C"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
@@ -5129,9 +6841,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Db scheduling time quantiles",
"tooltip": {
"shared": false,
@@ -5140,34 +6850,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "s",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": false
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5175,12 +6877,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5192,7 +6895,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 40
+ "y": 42
},
"hiddenSeries": false,
"id": 10,
@@ -5216,7 +6919,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5226,7 +6929,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5236,9 +6942,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Top DB transactions by txn rate",
"tooltip": {
"shared": false,
@@ -5247,9 +6951,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -5257,21 +6959,17 @@
{
"format": "hertz",
"logBase": 1,
- "max": null,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5279,12 +6977,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5296,7 +6995,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 40
+ "y": 42
},
"hiddenSeries": false,
"id": 11,
@@ -5320,7 +7019,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5330,7 +7029,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"instant": false,
"interval": "",
@@ -5341,9 +7043,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "DB transactions by total txn time",
"tooltip": {
"shared": false,
@@ -5352,9 +7052,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -5362,21 +7060,16 @@
{
"format": "percentunit",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5384,12 +7077,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5401,7 +7095,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 47
+ "y": 49
},
"hiddenSeries": false,
"id": 180,
@@ -5425,7 +7119,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5435,7 +7129,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"instant": false,
"interval": "",
@@ -5446,9 +7143,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average DB txn time",
"tooltip": {
"shared": false,
@@ -5457,9 +7152,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -5467,21 +7160,16 @@
{
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5489,10 +7177,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5503,7 +7192,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 47
+ "y": 49
},
"hiddenSeries": false,
"id": 200,
@@ -5524,7 +7213,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5534,6 +7223,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -5541,6 +7233,9 @@
"refId": "D"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -5548,6 +7243,9 @@
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -5555,6 +7253,9 @@
"refId": "C"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
@@ -5563,9 +7264,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Time waiting for DB connection quantiles",
"tooltip": {
"shared": true,
@@ -5574,49 +7273,54 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
+ "$$hashKey": "object:203",
"format": "s",
"label": "",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:204",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": false
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
- "repeat": null,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Database",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 33
+ "y": 35
},
"id": 59,
"panels": [
@@ -5625,12 +7329,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5642,7 +7347,7 @@
"h": 13,
"w": 12,
"x": 0,
- "y": 9
+ "y": 35
},
"hiddenSeries": false,
"id": 12,
@@ -5660,9 +7365,12 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5672,7 +7380,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5682,9 +7393,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Total CPU Usage by Block",
"tooltip": {
"shared": true,
@@ -5693,9 +7402,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -5703,21 +7410,16 @@
{
"format": "percentunit",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5725,12 +7427,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5742,7 +7445,7 @@
"h": 13,
"w": 12,
"x": 12,
- "y": 9
+ "y": 35
},
"hiddenSeries": false,
"id": 26,
@@ -5760,9 +7463,12 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5772,7 +7478,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5782,9 +7491,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average CPU Time per Block",
"tooltip": {
"shared": true,
@@ -5793,9 +7500,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -5803,21 +7508,16 @@
{
"format": "ms",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5825,12 +7525,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5842,7 +7544,7 @@
"h": 13,
"w": 12,
"x": 0,
- "y": 22
+ "y": 48
},
"hiddenSeries": false,
"id": 13,
@@ -5860,9 +7562,12 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5872,19 +7577,22 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
- "legendFormat": "{{job}} {{block_name}}",
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A",
"step": 20
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Total DB Usage by Block",
"tooltip": {
"shared": true,
@@ -5893,31 +7601,27 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:196",
"format": "percentunit",
"logBase": 1,
- "max": null,
"min": 0,
"show": true
},
{
+ "$$hashKey": "object:197",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -5925,13 +7629,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -5943,7 +7648,7 @@
"h": 13,
"w": 12,
"x": 12,
- "y": 22
+ "y": 48
},
"hiddenSeries": false,
"id": 27,
@@ -5961,9 +7666,12 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5973,7 +7681,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -5983,9 +7694,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average Database Transaction time, by Block",
"tooltip": {
"shared": true,
@@ -5994,9 +7703,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -6004,21 +7711,16 @@
{
"format": "ms",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6026,12 +7728,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6043,7 +7746,7 @@
"h": 13,
"w": 12,
"x": 0,
- "y": 35
+ "y": 61
},
"hiddenSeries": false,
"id": 28,
@@ -6060,9 +7763,12 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6072,7 +7778,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -6082,9 +7791,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average Transactions per Block",
"tooltip": {
"shared": false,
@@ -6093,9 +7800,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -6103,21 +7808,16 @@
{
"format": "none",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6125,12 +7825,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6142,7 +7843,7 @@
"h": 13,
"w": 12,
"x": 12,
- "y": 35
+ "y": 61
},
"hiddenSeries": false,
"id": 25,
@@ -6159,9 +7860,12 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6171,7 +7875,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -6181,9 +7888,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average Wallclock Time per Block",
"tooltip": {
"shared": false,
@@ -6192,31 +7897,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "format": "ms",
+ "$$hashKey": "object:180",
+ "format": "s",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:181",
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6224,12 +7924,8 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -6237,7 +7933,7 @@
"h": 15,
"w": 12,
"x": 0,
- "y": 48
+ "y": 74
},
"hiddenSeries": false,
"id": 154,
@@ -6254,8 +7950,11 @@
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.0.4",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -6265,16 +7964,17 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{job}}-{{index}} {{block_name}}",
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Block count",
"tooltip": {
"shared": true,
@@ -6283,48 +7983,50 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
- "repeat": null,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Per-block metrics",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 34
+ "y": 36
},
"id": 61,
"panels": [
@@ -6333,13 +8035,14 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"decimals": 2,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6351,7 +8054,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 35
+ "y": 36
},
"hiddenSeries": false,
"id": 1,
@@ -6375,7 +8078,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6385,7 +8088,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{name}} {{job}}-{{index}}",
@@ -6394,9 +8100,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Cache Hit Ratio",
"tooltip": {
"msResolution": true,
@@ -6406,15 +8110,12 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "percentunit",
"label": "",
"logBase": 1,
@@ -6425,14 +8126,11 @@
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": false
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6440,12 +8138,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6457,7 +8156,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 35
+ "y": 36
},
"hiddenSeries": false,
"id": 8,
@@ -6480,7 +8179,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6490,7 +8189,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
"interval": "",
@@ -6501,9 +8203,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Cache Size",
"tooltip": {
"shared": false,
@@ -6512,9 +8212,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -6522,21 +8220,17 @@
{
"format": "short",
"logBase": 1,
- "max": null,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6544,12 +8238,13 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6561,7 +8256,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 45
+ "y": 46
},
"hiddenSeries": false,
"id": 38,
@@ -6584,7 +8279,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6594,7 +8289,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -6604,9 +8302,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Cache request rate",
"tooltip": {
"shared": false,
@@ -6615,9 +8311,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -6625,21 +8319,17 @@
{
"format": "rps",
"logBase": 1,
- "max": null,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6647,10 +8337,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6661,7 +8352,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 45
+ "y": 46
},
"hiddenSeries": false,
"id": 39,
@@ -6669,11 +8360,16 @@
"alignAsTable": true,
"avg": false,
"current": false,
- "max": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
"min": false,
+ "rightSide": false,
"show": true,
+ "sort": "max",
+ "sortDesc": true,
"total": false,
- "values": false
+ "values": true
},
"lines": true,
"linewidth": 1,
@@ -6683,7 +8379,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6693,7 +8389,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "topk(10, rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -6703,9 +8402,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Top 10 cache misses",
"tooltip": {
"shared": false,
@@ -6714,33 +8411,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:101",
"format": "rps",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:102",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6748,10 +8438,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -6762,7 +8453,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 55
+ "y": 56
},
"hiddenSeries": false,
"id": 65,
@@ -6784,7 +8475,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6794,17 +8485,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
+ "interval": "",
"intervalFactor": 1,
"legendFormat": "{{name}} ({{reason}}) {{job}}-{{index}}",
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Cache eviction rate",
"tooltip": {
"shared": false,
@@ -6813,49 +8506,51 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "hertz",
"label": "entries / second",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
- "repeat": null,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Caches",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 35
+ "y": 37
},
"id": 148,
"panels": [
@@ -6864,7 +8559,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -6908,16 +8605,17 @@
"steppedLine": false,
"targets": [
{
- "expr": "synapse_util_caches_response_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Response cache size",
"tooltip": {
"shared": false,
@@ -6926,33 +8624,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -6960,7 +8649,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -7004,12 +8695,18 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_util_caches_response_cache:hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache:total{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{name}} {{job}}-{{index}}",
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "",
"interval": "",
"legendFormat": "",
@@ -7017,9 +8714,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Response cache hit rate",
"tooltip": {
"shared": false,
@@ -7028,17 +8723,13 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "percentunit",
- "label": null,
"logBase": 1,
"max": "1",
"min": "0",
@@ -7046,30 +8737,38 @@
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Response caches",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 36
+ "y": 38
},
"id": 62,
"panels": [
@@ -7078,7 +8777,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -7123,6 +8824,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
"format": "time_series",
"instant": false,
@@ -7132,9 +8836,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Total GC time by bucket (10m smoothing)",
"tooltip": {
"shared": true,
@@ -7143,34 +8845,25 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "percentunit",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -7178,7 +8871,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"decimals": 3,
"editable": true,
"error": false,
@@ -7228,6 +8923,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
"format": "time_series",
"intervalFactor": 2,
@@ -7238,9 +8936,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Average GC Time Per Collection",
"tooltip": {
"shared": false,
@@ -7249,9 +8945,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -7259,21 +8953,16 @@
{
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -7281,7 +8970,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
"fieldConfig": {
"defaults": {
@@ -7334,6 +9025,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
"format": "time_series",
"intervalFactor": 1,
@@ -7342,9 +9036,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Allocation counts",
"tooltip": {
"shared": false,
@@ -7353,9 +9045,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -7364,23 +9054,17 @@
"format": "short",
"label": "Gen N-1 GCs since last Gen N GC",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
- "decimals": null,
"format": "short",
"label": "Objects since last Gen 0 GC",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -7388,7 +9072,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -7433,6 +9119,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 1,
@@ -7441,9 +9130,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Object counts per collection",
"tooltip": {
"shared": true,
@@ -7452,33 +9139,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -7486,7 +9164,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -7531,6 +9211,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 1,
@@ -7539,9 +9222,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "GC frequency",
"tooltip": {
"shared": true,
@@ -7550,51 +9231,43 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": 0,
- "cardRound": null
+ "cardPadding": 0
},
"color": {
"cardColor": "#b4ff00",
"colorScale": "sqrt",
"colorScheme": "interpolateSpectral",
"exponent": 0.5,
- "max": null,
"min": 0,
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"fieldConfig": {
"defaults": {
"custom": {}
@@ -7618,6 +9291,10 @@
"reverseYBuckets": false,
"targets": [
{
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
"expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
"format": "heatmap",
"intervalFactor": 1,
@@ -7634,34 +9311,37 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
- "decimals": null,
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
+ },
+ "yBucketBound": "auto"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "refId": "A"
}
],
- "repeat": null,
"title": "GC",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 37
+ "y": 39
},
"id": 63,
"panels": [
@@ -7670,10 +9350,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -7684,10 +9366,10 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 13
+ "y": 40
},
"hiddenSeries": false,
- "id": 42,
+ "id": 43,
"legend": {
"avg": false,
"current": false,
@@ -7706,7 +9388,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -7716,7 +9398,10 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{command}}",
@@ -7725,10 +9410,8 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
- "title": "Rate of incoming commands",
+ "title": "Rate of outgoing commands",
"tooltip": {
"shared": false,
"sort": 0,
@@ -7736,241 +9419,331 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:89",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:90",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "description": "",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 13
- },
- "hiddenSeries": false,
- "id": 144,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "y": 40
},
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
+ "id": 41,
+ "links": [],
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
},
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.4.3",
"targets": [
{
- "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_replication_tcp_resource_stream_updates_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
"interval": "",
- "legendFormat": "{{stream_name}} {{job}}-{{index}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
- "title": "Queued incoming RDATA commands, by stream",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
+ "intervalFactor": 2,
+ "legendFormat": "{{stream_name}}",
+ "refId": "A",
+ "step": 20
}
],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "title": "Rate of outgoing RDATA commands, by stream",
+ "type": "timeseries"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
- "links": []
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 20
- },
- "hiddenSeries": false,
- "id": 43,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "y": 47
},
- "lines": true,
- "linewidth": 1,
+ "id": 42,
"links": [],
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
},
- "paceLength": 10,
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "8.4.3",
"targets": [
{
- "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
"format": "time_series",
+ "interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{command}}",
"refId": "A",
"step": 20
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
- "title": "Rate of outgoing commands",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
+ "title": "Rate of incoming commands (including echoes)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "axisSoftMin": 1,
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "links": [],
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
},
- "yaxes": [
- {
- "format": "hertz",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "id": 220,
+ "links": [],
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "8.4.3",
+ "targets": [
{
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "exemplar": true,
+ "expr": "rate(synapse_replication_tcp_protocol_inbound_rdata_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+ "refId": "A",
+ "step": 20
}
],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "title": "Rate of incoming RDATA commands (excluding echoes), by stream",
+ "type": "timeseries"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "description": "",
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -7980,11 +9753,11 @@
"gridPos": {
"h": 7,
"w": 12,
- "x": 12,
- "y": 20
+ "x": 0,
+ "y": 54
},
"hiddenSeries": false,
- "id": 41,
+ "id": 144,
"legend": {
"avg": false,
"current": false,
@@ -7996,15 +9769,13 @@
},
"lines": true,
"linewidth": 1,
- "links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
- "paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 5,
+ "pluginVersion": "9.0.4",
+ "pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
@@ -8013,20 +9784,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{stream_name}}",
- "refId": "A",
- "step": 20
+ "legendFormat": "{{stream_name}} {{job}}-{{index}}",
+ "refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
- "title": "Outgoing stream updates",
+ "title": "Queued incoming RDATA commands, by stream",
"tooltip": {
"shared": false,
"sort": 0,
@@ -8034,33 +9804,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "format": "hertz",
- "label": null,
+ "$$hashKey": "object:218",
+ "format": "short",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:219",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -8068,10 +9831,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -8081,11 +9846,11 @@
"gridPos": {
"h": 7,
"w": 12,
- "x": 0,
- "y": 27
+ "x": 12,
+ "y": 54
},
"hiddenSeries": false,
- "id": 113,
+ "id": 115,
"legend": {
"avg": false,
"current": false,
@@ -8104,7 +9869,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -8114,25 +9879,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_replication_tcp_protocol_close_reason_total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{stream_name}}",
+ "legendFormat": "{{job}}-{{index}} {{reason_type}}",
"refId": "A"
- },
- {
- "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}}",
- "refId": "B"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
- "title": "Replication connections",
+ "title": "Replication connection close reasons",
"tooltip": {
"shared": true,
"sort": 0,
@@ -8140,33 +9899,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "format": "short",
- "label": null,
+ "$$hashKey": "object:260",
+ "format": "hertz",
"logBase": 1,
- "max": null,
- "min": "0",
"show": true
},
{
+ "$$hashKey": "object:261",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -8174,10 +9926,11 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -8187,11 +9940,11 @@
"gridPos": {
"h": 7,
"w": 12,
- "x": 12,
- "y": 27
+ "x": 0,
+ "y": 61
},
"hiddenSeries": false,
- "id": 115,
+ "id": 113,
"legend": {
"avg": false,
"current": false,
@@ -8210,7 +9963,7 @@
},
"paceLength": 10,
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -8220,18 +9973,29 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_replication_tcp_protocol_close_reason{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "synapse_replication_tcp_resource_connections_per_stream{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{reason_type}}",
+ "legendFormat": "{{job}}-{{index}} {{stream_name}}",
"refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "synapse_replication_tcp_resource_total_connections{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "B"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
- "title": "Replication connection close reasons",
+ "title": "Replication connections",
"tooltip": {
"shared": true,
"sort": 0,
@@ -8239,48 +10003,51 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "format": "hertz",
- "label": null,
+ "format": "short",
"logBase": 1,
- "max": null,
- "min": null,
+ "min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
- "repeat": null,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Replication",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 38
+ "y": 40
},
"id": 69,
"panels": [
@@ -8289,7 +10056,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -8335,6 +10104,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
@@ -8344,9 +10116,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Event processing lag",
"tooltip": {
"shared": true,
@@ -8355,9 +10125,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -8366,22 +10134,17 @@
"format": "short",
"label": "events",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -8389,7 +10152,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -8435,6 +10200,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"hide": false,
@@ -8445,9 +10213,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Age of last processed event",
"tooltip": {
"shared": true,
@@ -8456,33 +10222,25 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "ms",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -8490,7 +10248,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
"custom": {},
@@ -8537,6 +10297,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "deriv(synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/1000 - 1",
"format": "time_series",
"hide": false,
@@ -8547,9 +10310,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Event processing catchup rate",
"tooltip": {
"shared": true,
@@ -8558,67 +10319,70 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "none",
"label": "fallbehind(-) / catchup(+): s/sec",
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Event processing loop positions",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 39
+ "y": 41
},
"id": 126,
"panels": [
{
"cards": {
- "cardPadding": 0,
- "cardRound": null
+ "cardPadding": 0
},
"color": {
"cardColor": "#B877D9",
"colorScale": "sqrt",
"colorScheme": "interpolateInferno",
"exponent": 0.5,
- "max": null,
"min": 0,
"mode": "opacity"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
"fieldConfig": {
"defaults": {
@@ -8643,15 +10407,16 @@
"reverseYBuckets": false,
"targets": [
{
- "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
- "timeFrom": null,
- "timeShift": null,
"title": "Number of rooms, by number of forward extremities in room",
"tooltip": {
"show": true,
@@ -8661,27 +10426,22 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
"fieldConfig": {
"defaults": {
@@ -8725,6 +10485,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
"format": "heatmap",
"interval": "",
@@ -8734,9 +10497,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Room counts, by number of extremities",
"tooltip": {
"shared": true,
@@ -8745,40 +10506,30 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
- "decimals": null,
"format": "none",
"label": "Number of rooms",
"logBase": 10,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": false
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": 0,
- "cardRound": null
+ "cardPadding": 0
},
"color": {
"cardColor": "#5794F2",
@@ -8789,7 +10540,9 @@
"mode": "opacity"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
"fieldConfig": {
"defaults": {
@@ -8814,15 +10567,16 @@
"reverseYBuckets": false,
"targets": [
{
- "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
- "timeFrom": null,
- "timeShift": null,
"title": "Events persisted, by number of forward extremities in room (heatmap)",
"tooltip": {
"show": true,
@@ -8832,27 +10586,22 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
"fieldConfig": {
"defaults": {
@@ -8895,28 +10644,40 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "A"
},
{
- "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "B"
},
{
- "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "C"
},
{
- "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
@@ -8924,9 +10685,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Events persisted, by number of forward extremities in room (quantiles)",
"tooltip": {
"shared": true,
@@ -8935,9 +10694,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -8946,28 +10703,22 @@
"format": "short",
"label": "Number of extremities in room",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": 0,
- "cardRound": null
+ "cardPadding": 0
},
"color": {
"cardColor": "#FF9830",
@@ -8978,7 +10729,9 @@
"mode": "opacity"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
"fieldConfig": {
"defaults": {
@@ -9003,15 +10756,16 @@
"reverseYBuckets": false,
"targets": [
{
- "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)",
"format": "heatmap",
"intervalFactor": 1,
"legendFormat": "{{le}}",
"refId": "A"
}
],
- "timeFrom": null,
- "timeShift": null,
"title": "Events persisted, by number of stale forward extremities in room (heatmap)",
"tooltip": {
"show": true,
@@ -9021,27 +10775,22 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
"fieldConfig": {
"defaults": {
@@ -9084,28 +10833,40 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "50%",
"refId": "A"
},
{
- "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "75%",
"refId": "B"
},
{
- "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "90%",
"refId": "C"
},
{
- "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "99%",
@@ -9113,9 +10874,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Events persisted, by number of stale forward extremities in room (quantiles)",
"tooltip": {
"shared": true,
@@ -9124,9 +10883,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -9135,28 +10892,22 @@
"format": "short",
"label": "Number of stale forward extremities in room",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": 0,
- "cardRound": null
+ "cardPadding": 0
},
"color": {
"cardColor": "#73BF69",
@@ -9167,7 +10918,9 @@
"mode": "opacity"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
"fieldConfig": {
"defaults": {
@@ -9192,6 +10945,9 @@
"reverseYBuckets": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "heatmap",
"interval": "",
@@ -9200,8 +10956,6 @@
"refId": "A"
}
],
- "timeFrom": null,
- "timeShift": null,
"title": "Number of state resolution performed, by number of state groups involved (heatmap)",
"tooltip": {
"show": true,
@@ -9211,27 +10965,22 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "short",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "yBucketBound": "auto"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
"fieldConfig": {
"defaults": {
@@ -9275,6 +11024,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.5, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
@@ -9283,6 +11035,9 @@
"refId": "A"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.75, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
@@ -9291,6 +11046,9 @@
"refId": "B"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.90, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
@@ -9299,6 +11057,9 @@
"refId": "C"
},
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "histogram_quantile(0.99, rate(synapse_state_number_state_groups_in_resolution_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"format": "time_series",
"interval": "",
@@ -9308,9 +11069,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Number of state resolutions performed, by number of state groups involved (quantiles)",
"tooltip": {
"shared": true,
@@ -9319,9 +11078,7 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
@@ -9330,22 +11087,17 @@
"format": "short",
"label": "Number of state groups",
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -9353,7 +11105,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
"fieldConfig": {
"defaults": {
@@ -9394,28 +11148,35 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "State res ",
"refId": "A"
},
{
- "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Potential to prune",
"refId": "B"
},
{
- "expr": "sum(rate(synapse_storage_events_times_pruned_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
"interval": "",
"legendFormat": "Pruned",
"refId": "C"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Stale extremity dropping",
"tooltip": {
"shared": true,
@@ -9424,47 +11185,50 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Extremities",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 40
+ "y": 42
},
"id": 158,
"panels": [
@@ -9473,10 +11237,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -9487,7 +11253,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 41
+ "y": 43
},
"hiddenSeries": false,
"id": 156,
@@ -9508,12 +11274,13 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
+ "$$hashKey": "object:632",
"alias": "Max",
"color": "#bf1b00",
"fill": 0,
@@ -9525,26 +11292,33 @@
"steppedLine": false,
"targets": [
{
- "expr": "synapse_admin_mau:current{instance=\"$instance\", job=~\"$job\"}",
- "format": "time_series",
- "interval": "",
- "intervalFactor": 1,
- "legendFormat": "Current",
- "refId": "A"
- },
- {
- "expr": "synapse_admin_mau:max{instance=\"$instance\", job=~\"$job\"}",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "synapse_admin_mau_max{instance=\"$instance\", job=~\"(hhs_)?synapse\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Max",
+ "range": true,
"refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "synapse_admin_mau_current{instance=\"$instance\", job=~\"(hhs_)?synapse\"}",
+ "hide": false,
+ "legendFormat": "Current",
+ "range": true,
+ "refId": "C"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "MAU Limits",
"tooltip": {
"shared": true,
@@ -9553,33 +11327,27 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:176",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:177",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -9587,12 +11355,8 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -9600,7 +11364,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 41
+ "y": 43
},
"hiddenSeries": false,
"id": 160,
@@ -9620,7 +11384,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -9630,6 +11394,9 @@
"steppedLine": false,
"targets": [
{
+ "datasource": {
+ "uid": "$datasource"
+ },
"expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
"interval": "",
"legendFormat": "{{ app_service }}",
@@ -9637,9 +11404,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "MAU by Appservice",
"tooltip": {
"shared": true,
@@ -9648,47 +11413,50 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "MAU",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 41
+ "y": 43
},
"id": 177,
"panels": [
@@ -9697,10 +11465,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -9711,7 +11481,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1
+ "y": 44
},
"hiddenSeries": false,
"id": 173,
@@ -9728,8 +11498,11 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -9739,20 +11512,23 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_notifier_users_woken_by_stream{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_notifier_users_woken_by_stream_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "{{stream}} {{index}}",
"metric": "synapse_notifier",
+ "range": true,
"refId": "A",
"step": 2
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Notifier Streams Woken",
"tooltip": {
"shared": true,
@@ -9761,33 +11537,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:734",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:735",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -9795,10 +11564,12 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {},
"links": []
},
"overrides": []
@@ -9809,7 +11580,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1
+ "y": 44
},
"hiddenSeries": false,
"id": 175,
@@ -9826,8 +11597,11 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
- "pluginVersion": "7.1.3",
+ "pluginVersion": "9.2.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -9837,19 +11611,22 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_handler_presence_get_updates{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_get_updates_total{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{type}} {{index}}",
+ "range": true,
"refId": "A",
"step": 2
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Presence Stream Fetch Type Rates",
"tooltip": {
"shared": true,
@@ -9858,47 +11635,53 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:819",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
"min": "0",
"show": true
},
{
+ "$$hashKey": "object:820",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Notifier",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 42
+ "y": 44
},
"id": 170,
"panels": [
@@ -9907,12 +11690,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -9920,7 +11700,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 43
+ "y": 45
},
"hiddenSeries": false,
"id": 168,
@@ -9940,7 +11720,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -9950,16 +11730,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_appservice_api_sent_events{instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_appservice_api_sent_events_total{instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{service}}",
+ "range": true,
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Sent Events rate",
"tooltip": {
"shared": true,
@@ -9968,33 +11751,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:177",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:178",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -10002,12 +11778,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -10015,7 +11788,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 43
+ "y": 45
},
"hiddenSeries": false,
"id": 171,
@@ -10035,7 +11808,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -10045,16 +11818,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_appservice_api_sent_transactions{instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_appservice_api_sent_transactions_total{instance=\"$instance\"}[$bucket_size])",
"interval": "",
- "legendFormat": "{{service}}",
+ "legendFormat": "{{exported_service }} {{ service }}",
+ "range": true,
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Transactions rate",
"tooltip": {
"shared": true,
@@ -10063,47 +11839,52 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:260",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:261",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Appservices",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 43
+ "y": 45
},
"id": 188,
"panels": [
@@ -10112,12 +11893,8 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -10125,7 +11902,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 44
+ "y": 46
},
"hiddenSeries": false,
"id": 182,
@@ -10145,7 +11922,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -10155,40 +11932,53 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_handler_presence_notified_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "Notified",
"refId": "A"
},
{
- "expr": "rate(synapse_handler_presence_federation_presence_out{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "Remote ping",
"refId": "B"
},
{
- "expr": "rate(synapse_handler_presence_presence_updates{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "Total updates",
"refId": "C"
},
{
- "expr": "rate(synapse_handler_presence_federation_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "Remote updates",
"refId": "D"
},
{
- "expr": "rate(synapse_handler_presence_bump_active_time{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "Bump active time",
"refId": "E"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Presence",
"tooltip": {
"shared": true,
@@ -10197,33 +11987,24 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -10231,12 +12012,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -10244,7 +12022,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 44
+ "y": 46
},
"hiddenSeries": false,
"id": 184,
@@ -10264,7 +12042,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -10274,16 +12052,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_handler_presence_state_transition{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_state_transition_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{from}} -> {{to}}",
+ "range": true,
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Presence state transitions",
"tooltip": {
"shared": true,
@@ -10292,33 +12073,26 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:1090",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:1091",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
@@ -10326,12 +12100,9 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
"fill": 1,
"fillGradient": 0,
@@ -10339,7 +12110,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 52
+ "y": 54
},
"hiddenSeries": false,
"id": 186,
@@ -10359,7 +12130,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.2.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -10369,16 +12140,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_handler_presence_notify_reason{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_handler_presence_notify_reason_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"interval": "",
"legendFormat": "{{reason}}",
+ "range": true,
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "Presence notify reason",
"tooltip": {
"shared": true,
@@ -10387,165 +12161,163 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:165",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:166",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
}
],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
+ }
+ ],
"title": "Presence",
"type": "row"
},
{
"collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 44
+ "y": 46
},
"id": 197,
"panels": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
"fieldConfig": {
"defaults": {
- "custom": {}
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "hertz"
},
"overrides": []
},
- "fill": 1,
- "fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 1
+ "y": 46
},
- "hiddenSeries": false,
"id": 191,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
"options": {
- "alertThreshold": true
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
},
- "percentage": false,
- "pluginVersion": "7.3.7",
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
+ "pluginVersion": "9.0.4",
"targets": [
{
- "expr": "rate(synapse_external_cache_set{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_external_cache_set{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
"interval": "",
- "legendFormat": "{{ cache_name }} {{ index }}",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
"refId": "A"
}
],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
"title": "External Cache Set Rate",
- "tooltip": {
- "shared": true,
- "sort": 2,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "hertz",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "type": "timeseries"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
- "description": "",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
+ "description": "",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 1
+ "y": 46
},
"hiddenSeries": false,
"id": 193,
@@ -10565,7 +12337,7 @@
"alertThreshold": true
},
"percentage": false,
- "pluginVersion": "7.3.7",
+ "pluginVersion": "9.0.4",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -10575,16 +12347,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_external_cache_get{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "sum without (hit) (rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size]))",
"interval": "",
- "legendFormat": "{{ cache_name }} {{ index }}",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
"refId": "A"
}
],
"thresholds": [],
- "timeFrom": null,
"timeRegions": [],
- "timeShift": null,
"title": "External Cache Get Rate",
"tooltip": {
"shared": true,
@@ -10593,39 +12368,31 @@
},
"type": "graph",
"xaxis": {
- "buckets": null,
"mode": "time",
- "name": null,
"show": true,
"values": []
},
"yaxes": [
{
+ "$$hashKey": "object:390",
"format": "hertz",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
},
{
+ "$$hashKey": "object:391",
"format": "short",
- "label": null,
"logBase": 1,
- "max": null,
- "min": null,
"show": true
}
],
"yaxis": {
- "align": false,
- "alignLevel": null
+ "align": false
}
},
{
"cards": {
- "cardPadding": -1,
- "cardRound": null
+ "cardPadding": -1
},
"color": {
"cardColor": "#b4ff00",
@@ -10636,18 +12403,15 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {}
- },
- "overrides": []
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
},
"gridPos": {
- "h": 9,
+ "h": 8,
"w": 12,
"x": 0,
- "y": 9
+ "y": 54
},
"heatmap": {},
"hideZeroBuckets": false,
@@ -10660,7 +12424,10 @@
"reverseYBuckets": false,
"targets": [
{
- "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le)",
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])) by (le)",
"format": "heatmap",
"instant": false,
"interval": "",
@@ -10679,20 +12446,110 @@
"xAxis": {
"show": true
},
- "xBucketNumber": null,
- "xBucketSize": null,
"yAxis": {
"decimals": 0,
"format": "s",
"logBase": 1,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
+ "show": true
+ },
+ "yBucketBound": "auto"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "$datasource"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "hertz"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 54
},
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
+ "id": 223,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "9.0.4",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$datasource"
+ },
+ "editorMode": "code",
+ "expr": "rate(synapse_external_cache_get{job=~\"$job\", instance=\"$instance\", index=~\"$index\", hit=\"False\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{job}}-{{ index }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Miss Rate",
+ "type": "timeseries"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "000000001"
+ },
+ "refId": "A"
}
],
"title": "External Cache",
@@ -10700,7 +12557,7 @@
}
],
"refresh": false,
- "schemaVersion": 26,
+ "schemaVersion": 37,
"style": "dark",
"tags": [
"matrix"
@@ -10713,10 +12570,8 @@
"text": "default",
"value": "default"
},
- "error": null,
"hide": 0,
"includeAll": false,
- "label": null,
"multi": false,
"name": "datasource",
"options": [],
@@ -10731,14 +12586,12 @@
"allFormat": "glob",
"auto": true,
"auto_count": 100,
- "auto_min": "60s",
+ "auto_min": "30s",
"current": {
"selected": false,
"text": "auto",
"value": "$__auto_interval_bucket_size"
},
- "datasource": null,
- "error": null,
"hide": 0,
"includeAll": false,
"label": "Bucket Size",
@@ -10789,24 +12642,25 @@
"type": "interval"
},
{
- "allValue": null,
"current": {},
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"definition": "",
- "error": null,
"hide": 0,
"includeAll": false,
- "label": null,
"multi": false,
"name": "instance",
"options": [],
- "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, instance)",
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)",
+ "refId": "Prometheus-instance-Variable-Query"
+ },
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
- "tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
@@ -10815,9 +12669,10 @@
"allFormat": "regex wildcard",
"allValue": "",
"current": {},
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"definition": "",
- "error": null,
"hide": 0,
"hideLabel": false,
"includeAll": true,
@@ -10826,14 +12681,16 @@
"multiFormat": "regex values",
"name": "job",
"options": [],
- "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, job)",
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)",
+ "refId": "Prometheus-job-Variable-Query"
+ },
"refresh": 2,
"refresh_on_load": false,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
- "tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
@@ -10842,9 +12699,10 @@
"allFormat": "regex wildcard",
"allValue": ".*",
"current": {},
- "datasource": "$datasource",
+ "datasource": {
+ "uid": "$datasource"
+ },
"definition": "",
- "error": null,
"hide": 0,
"hideLabel": false,
"includeAll": true,
@@ -10853,14 +12711,16 @@
"multiFormat": "regex values",
"name": "index",
"options": [],
- "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, index)",
+ "query": {
+ "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)",
+ "refId": "Prometheus-index-Variable-Query"
+ },
"refresh": 2,
"refresh_on_load": false,
"regex": "",
"skipUrlSync": false,
"sort": 3,
"tagValuesQuery": "",
- "tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
@@ -10900,5 +12760,6 @@
"timezone": "",
"title": "Synapse",
"uid": "000000012",
- "version": 100
-}
+ "version": 149,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/contrib/prometheus/synapse-v1.rules b/contrib/prometheus/synapse-v1.rules
deleted file mode 100644
index 4c900ba537..0000000000
--- a/contrib/prometheus/synapse-v1.rules
+++ /dev/null
@@ -1,21 +0,0 @@
-synapse_federation_transaction_queue_pendingEdus:total = sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)
-synapse_federation_transaction_queue_pendingPdus:total = sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)
-
-synapse_http_server_request_count:method{servlet=""} = sum(synapse_http_server_request_count) by (method)
-synapse_http_server_request_count:servlet{method=""} = sum(synapse_http_server_request_count) by (servlet)
-
-synapse_http_server_request_count:total{servlet=""} = sum(synapse_http_server_request_count:by_method) by (servlet)
-
-synapse_cache:hit_ratio_5m = rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])
-synapse_cache:hit_ratio_30s = rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s])
-
-synapse_federation_client_sent{type="EDU"} = synapse_federation_client_sent_edus + 0
-synapse_federation_client_sent{type="PDU"} = synapse_federation_client_sent_pdu_destinations:count + 0
-synapse_federation_client_sent{type="Query"} = sum(synapse_federation_client_sent_queries) by (job)
-
-synapse_federation_server_received{type="EDU"} = synapse_federation_server_received_edus + 0
-synapse_federation_server_received{type="PDU"} = synapse_federation_server_received_pdus + 0
-synapse_federation_server_received{type="Query"} = sum(synapse_federation_server_received_queries) by (job)
-
-synapse_federation_transaction_queue_pending{type="EDU"} = synapse_federation_transaction_queue_pending_edus + 0
-synapse_federation_transaction_queue_pending{type="PDU"} = synapse_federation_transaction_queue_pending_pdus + 0
diff --git a/contrib/prometheus/synapse-v2.rules b/contrib/prometheus/synapse-v2.rules
index 7e405bf7f0..dde311322f 100644
--- a/contrib/prometheus/synapse-v2.rules
+++ b/contrib/prometheus/synapse-v2.rules
@@ -1,37 +1,20 @@
groups:
- name: synapse
rules:
- - record: "synapse_federation_transaction_queue_pendingEdus:total"
- expr: "sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)"
- - record: "synapse_federation_transaction_queue_pendingPdus:total"
- expr: "sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)"
- - record: 'synapse_http_server_request_count:method'
- labels:
- servlet: ""
- expr: "sum(synapse_http_server_request_count) by (method)"
- - record: 'synapse_http_server_request_count:servlet'
- labels:
- method: ""
- expr: 'sum(synapse_http_server_request_count) by (servlet)'
-
- - record: 'synapse_http_server_request_count:total'
- labels:
- servlet: ""
- expr: 'sum(synapse_http_server_request_count:by_method) by (servlet)'
-
- - record: 'synapse_cache:hit_ratio_5m'
- expr: 'rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])'
- - record: 'synapse_cache:hit_ratio_30s'
- expr: 'rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s])'
+ ###
+ ### Prometheus Console Only
+ ### The following rules are only needed if you use the Prometheus Console
+ ### in contrib/prometheus/consoles/synapse.html
+ ###
- record: 'synapse_federation_client_sent'
labels:
type: "EDU"
- expr: 'synapse_federation_client_sent_edus + 0'
+ expr: 'synapse_federation_client_sent_edus_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "PDU"
- expr: 'synapse_federation_client_sent_pdu_destinations:count + 0'
+ expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "Query"
@@ -40,11 +23,11 @@ groups:
- record: 'synapse_federation_server_received'
labels:
type: "EDU"
- expr: 'synapse_federation_server_received_edus + 0'
+ expr: 'synapse_federation_server_received_edus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "PDU"
- expr: 'synapse_federation_server_received_pdus + 0'
+ expr: 'synapse_federation_server_received_pdus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "Query"
@@ -58,21 +41,34 @@ groups:
labels:
type: "PDU"
expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+ ###
+ ### End of 'Prometheus Console Only' rules block
+ ###
+
+ ###
+ ### Grafana Only
+ ### The following rules are only needed if you use the Grafana dashboard
+ ### in contrib/grafana/synapse.json
+ ###
- record: synapse_storage_events_persisted_by_source_type
- expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"})
+ expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
labels:
type: remote
- record: synapse_storage_events_persisted_by_source_type
- expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"})
+ expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
labels:
type: local
- record: synapse_storage_events_persisted_by_source_type
- expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"})
+ expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
labels:
type: bridges
+
- record: synapse_storage_events_persisted_by_event_type
- expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep)
- - record: synapse_storage_events_persisted_by_origin
- expr: sum without(type) (synapse_storage_events_persisted_events_sep)
+ expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
+ - record: synapse_storage_events_persisted_by_origin
+ expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
+ ###
+ ### End of 'Grafana Only' rules block
+ ###
diff --git a/contrib/workers-bash-scripts/create-multiple-generic-workers.md b/contrib/workers-bash-scripts/create-multiple-generic-workers.md
index d303101429..c9be707b3c 100644
--- a/contrib/workers-bash-scripts/create-multiple-generic-workers.md
+++ b/contrib/workers-bash-scripts/create-multiple-generic-workers.md
@@ -7,7 +7,7 @@ You can alternatively create multiple worker configuration files with a simple `
#!/bin/bash
for i in {1..5}
do
-cat << EOF >> generic_worker$i.yaml
+cat << EOF > generic_worker$i.yaml
worker_app: synapse.app.generic_worker
worker_name: generic_worker$i
@@ -15,6 +15,8 @@ worker_name: generic_worker$i
worker_replication_host: 127.0.0.1
worker_replication_http_port: 9093
+worker_main_http_uri: http://localhost:8008/
+
worker_listeners:
- type: http
port: 808$i
|
