diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json
index 539569b5b1..0c4816b7cd 100644
--- a/contrib/grafana/synapse.json
+++ b/contrib/grafana/synapse.json
@@ -14,7 +14,7 @@
"type": "grafana",
"id": "grafana",
"name": "Grafana",
- "version": "6.7.4"
+ "version": "7.3.7"
},
{
"type": "panel",
@@ -38,7 +38,6 @@
"annotations": {
"list": [
{
- "$$hashKey": "object:76",
"builtIn": 1,
"datasource": "$datasource",
"enable": false,
@@ -55,11 +54,12 @@
"gnetId": null,
"graphTooltip": 0,
"id": null,
- "iteration": 1594646317221,
+ "iteration": 1621258266004,
"links": [
{
- "asDropdown": true,
+ "asDropdown": false,
"icon": "external link",
+ "includeVars": true,
"keepTime": true,
"tags": [
"matrix"
@@ -84,72 +84,254 @@
"type": "row"
},
{
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 189,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Event Send Time (excluding errors, all workers)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 2,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
- "x": 0,
+ "x": 12,
"y": 1
},
"hiddenSeries": false,
- "id": 75,
+ "id": 152,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
+ "rightSide": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
- "linewidth": 1,
+ "linewidth": 0,
"links": [],
- "nullPointMode": "null",
+ "nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ },
+ {
+ "alias": "Events",
+ "color": "#B877D9",
+ "hideTooltip": true,
+ "points": true,
+ "yaxis": 2,
+ "zindex": -3
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
"format": "time_series",
"intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} ",
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
"refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) by (le))",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',index=~\"$index\",instance=\"$instance\",code=~\"2..\"}[$bucket_size]))",
+ "legendFormat": "Average",
+ "refId": "H"
+ },
+ {
+ "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "Events",
+ "refId": "E"
}
],
"thresholds": [
{
- "colorMode": "critical",
- "fill": true,
+ "$$hashKey": "object:283",
+ "colorMode": "warning",
+ "fill": false,
"line": true,
"op": "gt",
"value": 1,
"yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:284",
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 2,
+ "yaxis": "left"
}
],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "CPU usage",
+ "title": "Event Send Time Quantiles (excluding errors, all workers)",
"tooltip": {
"shared": true,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -162,20 +344,22 @@
},
"yaxes": [
{
+ "$$hashKey": "object:255",
"decimals": null,
- "format": "percentunit",
- "label": null,
+ "format": "s",
+ "label": "",
"logBase": 1,
- "max": "1.5",
+ "max": null,
"min": "0",
"show": true
},
{
- "format": "short",
- "label": null,
+ "$$hashKey": "object:256",
+ "format": "hertz",
+ "label": "",
"logBase": 1,
"max": null,
- "min": null,
+ "min": "0",
"show": true
}
],
@@ -190,37 +374,42 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
- "editable": true,
- "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
- "grid": {},
"gridPos": {
"h": 9,
"w": 12,
- "x": 12,
- "y": 1
+ "x": 0,
+ "y": 10
},
"hiddenSeries": false,
- "id": 33,
+ "id": 75,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
- "show": false,
+ "show": true,
"total": false,
"values": false
},
"lines": true,
- "linewidth": 2,
+ "linewidth": 3,
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -230,24 +419,33 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+ "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "",
- "refId": "A",
- "step": 20,
- "target": ""
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} ",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [
+ {
+ "$$hashKey": "object:566",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
}
],
- "thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Events Persisted",
+ "title": "CPU usage",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
- "value_type": "cumulative"
+ "value_type": "individual"
},
"type": "graph",
"xaxis": {
@@ -259,14 +457,19 @@
},
"yaxes": [
{
- "format": "hertz",
+ "$$hashKey": "object:538",
+ "decimals": null,
+ "format": "percentunit",
+ "label": null,
"logBase": 1,
- "max": null,
- "min": null,
+ "max": "1.5",
+ "min": "0",
"show": true
},
{
+ "$$hashKey": "object:539",
"format": "short",
+ "label": null,
"logBase": 1,
"max": null,
"min": null,
@@ -279,75 +482,23 @@
}
},
{
- "cards": {
- "cardPadding": 0,
- "cardRound": null
- },
- "color": {
- "cardColor": "#b4ff00",
- "colorScale": "sqrt",
- "colorScheme": "interpolateSpectral",
- "exponent": 0.5,
- "mode": "spectrum"
- },
- "dataFormat": "tsbuckets",
- "datasource": "$datasource",
- "gridPos": {
- "h": 9,
- "w": 12,
- "x": 0,
- "y": 10
- },
- "heatmap": {},
- "hideZeroBuckets": true,
- "highlightCards": true,
- "id": 85,
- "legend": {
- "show": false
- },
- "links": [],
- "reverseYBuckets": false,
- "targets": [
- {
- "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
- "format": "heatmap",
- "intervalFactor": 1,
- "legendFormat": "{{le}}",
- "refId": "A"
- }
- ],
- "title": "Event Send Time",
- "tooltip": {
- "show": true,
- "showHistogram": false
- },
- "type": "heatmap",
- "xAxis": {
- "show": true
- },
- "xBucketNumber": null,
- "xBucketSize": null,
- "yAxis": {
- "decimals": null,
- "format": "s",
- "logBase": 2,
- "max": null,
- "min": null,
- "show": true,
- "splitFactor": null
- },
- "yBucketBound": "auto",
- "yBucketNumber": null,
- "yBucketSize": null
- },
- {
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
- "fill": 0,
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
"fillGradient": 0,
+ "grid": {},
"gridPos": {
"h": 9,
"w": 12,
@@ -355,7 +506,7 @@
"y": 10
},
"hiddenSeries": false,
- "id": 107,
+ "id": 198,
"legend": {
"avg": false,
"current": false,
@@ -366,76 +517,52 @@
"values": false
},
"lines": true,
- "linewidth": 1,
+ "linewidth": 3,
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "repeat": null,
- "repeatDirection": "h",
- "seriesOverrides": [
- {
- "alias": "mean",
- "linewidth": 2
- }
- ],
+ "seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
+ "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "99%",
- "refId": "A"
+ "intervalFactor": 2,
+ "legendFormat": "{{job}} {{index}}",
+ "refId": "A",
+ "step": 20,
+ "target": ""
},
{
- "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "95%",
+ "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "hide": true,
+ "interval": "",
+ "legendFormat": "total",
"refId": "B"
- },
- {
- "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "90%",
- "refId": "C"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "50%",
- "refId": "D"
- },
- {
- "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\"}[$bucket_size])) without (job, index, method)",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "mean",
- "refId": "E"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Event send time quantiles",
+ "title": "Memory",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
- "value_type": "individual"
+ "value_type": "cumulative"
},
+ "transformations": [],
"type": "graph",
"xaxis": {
"buckets": null,
@@ -446,16 +573,16 @@
},
"yaxes": [
{
- "format": "s",
- "label": null,
+ "$$hashKey": "object:1560",
+ "format": "bytes",
"logBase": 1,
"max": null,
- "min": null,
+ "min": "0",
"show": true
},
{
+ "$$hashKey": "object:1561",
"format": "short",
- "label": null,
"logBase": 1,
"max": null,
"min": null,
@@ -473,16 +600,23 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
- "fill": 0,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
"fillGradient": 0,
"gridPos": {
- "h": 9,
+ "h": 7,
"w": 12,
- "x": 0,
+ "x": 12,
"y": 19
},
"hiddenSeries": false,
- "id": 118,
+ "id": 37,
"legend": {
"avg": false,
"current": false,
@@ -497,18 +631,21 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "repeatDirection": "h",
"seriesOverrides": [
{
- "alias": "mean",
- "linewidth": 2
+ "$$hashKey": "object:639",
+ "alias": "/max$/",
+ "color": "#890F02",
+ "fill": 0,
+ "legend": false
}
],
"spaceLength": 10,
@@ -516,49 +653,33 @@
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
+ "hide": false,
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 99%",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 95%",
- "refId": "B"
- },
- {
- "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 90%",
- "refId": "C"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 50%",
- "refId": "D"
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}}",
+ "refId": "A",
+ "step": 20
},
{
- "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
+ "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} mean",
- "refId": "E"
+ "hide": true,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{job}}-{{index}} max",
+ "refId": "B",
+ "step": 20
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Event send time quantiles by worker",
+ "title": "Open FDs",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -572,14 +693,18 @@
},
"yaxes": [
{
- "format": "s",
- "label": null,
+ "$$hashKey": "object:650",
+ "decimals": null,
+ "format": "none",
+ "label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
+ "$$hashKey": "object:651",
+ "decimals": null,
"format": "short",
"label": null,
"logBase": 1,
@@ -600,7 +725,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 28
+ "y": 26
},
"id": 54,
"panels": [
@@ -612,6 +737,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -619,7 +751,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 2
+ "y": 25
},
"hiddenSeries": false,
"id": 5,
@@ -637,22 +769,25 @@
"values": false
},
"lines": true,
- "linewidth": 1,
+ "linewidth": 3,
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
+ "$$hashKey": "object:1240",
"alias": "/user/"
},
{
+ "$$hashKey": "object:1241",
"alias": "/system/"
}
],
@@ -682,20 +817,33 @@
],
"thresholds": [
{
+ "$$hashKey": "object:1278",
"colorMode": "custom",
"fillColor": "rgba(255, 255, 255, 1)",
"line": true,
"lineColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
- "value": 0.5
+ "value": 0.5,
+ "yaxis": "left"
},
{
+ "$$hashKey": "object:1279",
"colorMode": "custom",
"fillColor": "rgba(255, 255, 255, 1)",
"line": true,
- "lineColor": "rgba(234, 112, 112, 0.22)",
+ "lineColor": "rgb(87, 6, 16)",
+ "op": "gt",
+ "value": 0.8,
+ "yaxis": "left"
+ },
+ {
+ "$$hashKey": "object:1498",
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
"op": "gt",
- "value": 0.8
+ "value": 1,
+ "yaxis": "left"
}
],
"timeFrom": null,
@@ -703,7 +851,7 @@
"timeShift": null,
"title": "CPU",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -717,6 +865,7 @@
},
"yaxes": [
{
+ "$$hashKey": "object:1250",
"decimals": null,
"format": "percentunit",
"label": "",
@@ -726,6 +875,7 @@
"show": true
},
{
+ "$$hashKey": "object:1251",
"format": "short",
"logBase": 1,
"max": null,
@@ -744,16 +894,25 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 2
+ "y": 25
},
"hiddenSeries": false,
- "id": 37,
+ "id": 105,
+ "interval": "",
"legend": {
"avg": false,
"current": false,
@@ -768,51 +927,57 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/max$/",
- "color": "#890F02",
- "fill": 0,
- "legend": false
- }
- ],
+ "seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
"format": "time_series",
- "hide": false,
+ "interval": "",
"intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
+ "legendFormat": "{{job}}-{{index}} 99%",
"refId": "A",
"step": 20
},
{
- "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
"format": "time_series",
- "hide": true,
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} max",
- "refId": "B",
- "step": 20
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} mean",
+ "refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Open FDs",
+ "title": "Reactor tick quantiles",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -826,7 +991,7 @@
},
"yaxes": [
{
- "format": "none",
+ "format": "s",
"label": null,
"logBase": 1,
"max": null,
@@ -839,7 +1004,7 @@
"logBase": 1,
"max": null,
"min": null,
- "show": true
+ "show": false
}
],
"yaxis": {
@@ -855,6 +1020,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 0,
"fillGradient": 0,
"grid": {},
@@ -862,7 +1034,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 9
+ "y": 32
},
"hiddenSeries": false,
"id": 34,
@@ -880,10 +1052,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -895,11 +1068,18 @@
{
"expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
+ "interval": "",
"intervalFactor": 2,
"legendFormat": "{{job}} {{index}}",
"refId": "A",
"step": 20,
"target": ""
+ },
+ {
+ "expr": "sum(process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+ "interval": "",
+ "legendFormat": "total",
+ "refId": "B"
}
],
"thresholds": [],
@@ -908,10 +1088,11 @@
"timeShift": null,
"title": "Memory",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "cumulative"
},
+ "transformations": [],
"type": "graph",
"xaxis": {
"buckets": null,
@@ -947,18 +1128,23 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
- "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 9
+ "y": 32
},
"hiddenSeries": false,
- "id": 105,
- "interval": "",
+ "id": 49,
"legend": {
"avg": false,
"current": false,
@@ -973,54 +1159,40 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "/^up/",
+ "legend": false,
+ "yaxis": 2
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+ "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} 99%",
+ "legendFormat": "{{job}}-{{index}}",
"refId": "A",
"step": 20
- },
- {
- "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 95%",
- "refId": "B"
- },
- {
- "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} 90%",
- "refId": "C"
- },
- {
- "expr": "rate(python_twisted_reactor_tick_time_sum{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]) / rate(python_twisted_reactor_tick_time_count{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} mean",
- "refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Reactor tick quantiles",
+ "title": "Prometheus scrape time",
"tooltip": {
"shared": false,
"sort": 0,
@@ -1040,15 +1212,16 @@
"label": null,
"logBase": 1,
"max": null,
- "min": null,
+ "min": "0",
"show": true
},
{
- "format": "short",
- "label": null,
+ "decimals": 0,
+ "format": "none",
+ "label": "",
"logBase": 1,
- "max": null,
- "min": null,
+ "max": "0",
+ "min": "-1",
"show": false
}
],
@@ -1063,13 +1236,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 16
+ "y": 39
},
"hiddenSeries": false,
"id": 53,
@@ -1087,10 +1267,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1113,7 +1294,7 @@
"timeShift": null,
"title": "Up",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -1154,16 +1335,23 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 16
+ "y": 39
},
"hiddenSeries": false,
- "id": 49,
+ "id": 120,
"legend": {
"avg": false,
"current": false,
@@ -1176,43 +1364,56 @@
"lines": true,
"linewidth": 1,
"links": [],
- "nullPointMode": "null",
+ "nullPointMode": "null as zero",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
- "paceLength": 10,
"percentage": false,
- "pointradius": 5,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
"points": false,
"renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/^up/",
- "legend": false,
- "yaxis": 2
- }
- ],
+ "seriesOverrides": [],
"spaceLength": 10,
- "stack": false,
+ "stack": true,
"steppedLine": false,
"targets": [
{
- "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
+ "hide": false,
+ "instant": false,
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}}",
- "refId": "A",
- "step": 20
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{name}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
}
],
- "thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Prometheus scrape time",
+ "title": "Stacked CPU usage",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -1226,21 +1427,22 @@
},
"yaxes": [
{
- "format": "s",
+ "$$hashKey": "object:572",
+ "format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
- "min": "0",
+ "min": null,
"show": true
},
{
- "decimals": 0,
- "format": "none",
- "label": "",
+ "$$hashKey": "object:573",
+ "format": "short",
+ "label": null,
"logBase": 1,
- "max": "0",
- "min": "-1",
- "show": false
+ "max": null,
+ "min": null,
+ "show": true
}
],
"yaxis": {
@@ -1254,13 +1456,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 23
+ "y": 46
},
"hiddenSeries": false,
"id": 136,
@@ -1278,9 +1487,10 @@
"linewidth": 1,
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -1306,7 +1516,7 @@
"timeShift": null,
"title": "Outgoing HTTP request rate",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -1340,6 +1550,90 @@
"align": false,
"alignLevel": null
}
+ }
+ ],
+ "repeat": null,
+ "title": "Process info",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 56,
+ "panels": [
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": 0
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 21
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 85,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Event Send Time (Including errors, across all workers)",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": null,
+ "format": "s",
+ "logBase": 2,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
},
{
"aliasColors": {},
@@ -1347,79 +1641,74 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
+ "grid": {},
"gridPos": {
- "h": 7,
+ "h": 9,
"w": 12,
"x": 12,
- "y": 23
+ "y": 21
},
"hiddenSeries": false,
- "id": 120,
+ "id": 33,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
- "show": true,
+ "show": false,
"total": false,
"values": false
},
"lines": true,
- "linewidth": 1,
+ "linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
+ "paceLength": 10,
"percentage": false,
- "pointradius": 2,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
- "stack": true,
+ "stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "hide": false,
- "instant": false,
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
- "refId": "A"
- },
- {
- "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
"format": "time_series",
- "hide": false,
- "instant": false,
"interval": "",
- "intervalFactor": 1,
- "legendFormat": "{{job}}-{{index}} {{name}}",
- "refId": "B"
- }
- ],
- "thresholds": [
- {
- "colorMode": "critical",
- "fill": true,
- "line": true,
- "op": "gt",
- "value": 1,
- "yaxis": "left"
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 20,
+ "target": ""
}
],
+ "thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Stacked CPU usage",
+ "title": "Events Persisted (all workers)",
"tooltip": {
- "shared": false,
+ "shared": true,
"sort": 0,
- "value_type": "individual"
+ "value_type": "cumulative"
},
"type": "graph",
"xaxis": {
@@ -1431,16 +1720,16 @@
},
"yaxes": [
{
- "format": "percentunit",
- "label": null,
+ "$$hashKey": "object:102",
+ "format": "hertz",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
+ "$$hashKey": "object:103",
"format": "short",
- "label": null,
"logBase": 1,
"max": null,
"min": null,
@@ -1451,23 +1740,7 @@
"align": false,
"alignLevel": null
}
- }
- ],
- "repeat": null,
- "title": "Process info",
- "type": "row"
- },
- {
- "collapsed": true,
- "datasource": "${DS_PROMETHEUS}",
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 29
- },
- "id": 56,
- "panels": [
+ },
{
"aliasColors": {},
"bars": false,
@@ -1475,13 +1748,21 @@
"dashes": false,
"datasource": "$datasource",
"decimals": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"fill": 1,
+ "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 58
+ "y": 30
},
+ "hiddenSeries": false,
"id": 40,
"legend": {
"avg": false,
@@ -1496,7 +1777,11 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1561,13 +1846,21 @@
"dashes": false,
"datasource": "$datasource",
"decimals": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"fill": 1,
+ "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 58
+ "y": 30
},
+ "hiddenSeries": false,
"id": 46,
"legend": {
"avg": false,
@@ -1582,7 +1875,11 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1651,13 +1948,21 @@
"dashes": false,
"datasource": "$datasource",
"decimals": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"fill": 1,
+ "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 65
+ "y": 37
},
+ "hiddenSeries": false,
"id": 44,
"legend": {
"alignAsTable": true,
@@ -1675,7 +1980,11 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1741,13 +2050,21 @@
"dashes": false,
"datasource": "$datasource",
"decimals": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"fill": 1,
+ "fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 65
+ "y": 37
},
+ "hiddenSeries": false,
"id": 45,
"legend": {
"alignAsTable": true,
@@ -1765,7 +2082,11 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1823,10 +2144,145 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 118,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeatDirection": "h",
+ "seriesOverrides": [
+ {
+ "alias": "mean",
+ "linewidth": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 99%",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.95, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 95%",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.90, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 90%",
+ "refId": "C"
+ },
+ {
+ "expr": "histogram_quantile(0.50, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method))",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} 50%",
+ "refId": "D"
+ },
+ {
+ "expr": "sum(rate(synapse_http_server_response_time_seconds_sum{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method) / sum(rate(synapse_http_server_response_time_seconds_count{servlet='RoomSendEventRestServlet',instance=\"$instance\",code=~\"2..\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (method)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} mean",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Event send time quantiles by worker",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"repeat": null,
- "title": "Event persist rates",
+ "title": "Event persistence",
"type": "row"
},
{
@@ -1836,7 +2292,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 30
+ "y": 28
},
"id": 57,
"panels": [
@@ -1849,6 +2305,13 @@
"decimals": null,
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 2,
"fillGradient": 0,
"grid": {},
@@ -1878,9 +2341,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -1905,14 +2369,16 @@
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
- "value": 100
+ "value": 100,
+ "yaxis": "left"
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
- "value": 250
+ "value": 250,
+ "yaxis": "left"
}
],
"timeFrom": null,
@@ -1921,7 +2387,7 @@
"title": "Request Count by arrival time",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -1961,6 +2427,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -1986,9 +2459,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2014,7 +2488,7 @@
"title": "Top 10 Request Counts",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "cumulative"
},
"type": "graph",
@@ -2055,6 +2529,13 @@
"decimals": null,
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 2,
"fillGradient": 0,
"grid": {},
@@ -2084,9 +2565,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2129,7 +2611,7 @@
"title": "Total CPU Usage by Endpoint",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -2170,7 +2652,14 @@
"decimals": null,
"editable": true,
"error": false,
- "fill": 2,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
@@ -2199,9 +2688,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2214,7 +2704,7 @@
"expr": "(rate(synapse_http_server_in_flight_requests_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_in_flight_requests_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
- "intervalFactor": 2,
+ "intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A",
"step": 20
@@ -2226,14 +2716,16 @@
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
- "value": 100
+ "value": 100,
+ "yaxis": "left"
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
- "value": 250
+ "value": 250,
+ "yaxis": "left"
}
],
"timeFrom": null,
@@ -2242,7 +2734,7 @@
"title": "Average CPU Usage by Endpoint",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -2282,6 +2774,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -2310,9 +2809,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2325,7 +2825,7 @@
"expr": "rate(synapse_http_server_in_flight_requests_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
"format": "time_series",
"interval": "",
- "intervalFactor": 2,
+ "intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
"refId": "A",
"step": 20
@@ -2338,7 +2838,7 @@
"title": "DB Usage by endpoint",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "cumulative"
},
"type": "graph",
@@ -2379,6 +2879,13 @@
"decimals": null,
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 2,
"fillGradient": 0,
"grid": {},
@@ -2408,9 +2915,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2424,7 +2932,7 @@
"format": "time_series",
"hide": false,
"interval": "",
- "intervalFactor": 2,
+ "intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
"refId": "A",
"step": 20
@@ -2437,7 +2945,7 @@
"title": "Non-sync avg response time",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -2475,6 +2983,13 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2499,13 +3014,21 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "Total",
+ "color": "rgb(255, 255, 255)",
+ "fill": 0,
+ "linewidth": 3
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
@@ -2517,6 +3040,12 @@
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
"refId": "A"
+ },
+ {
+ "expr": "sum(avg_over_time(synapse_http_server_in_flight_requests_count{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Total",
+ "refId": "B"
}
],
"thresholds": [],
@@ -2526,7 +3055,7 @@
"title": "Requests in flight",
"tooltip": {
"shared": false,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -2572,7 +3101,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 31
+ "y": 29
},
"id": 97,
"panels": [
@@ -2582,6 +3111,13 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2605,11 +3141,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2674,6 +3208,13 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2697,11 +3238,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2717,12 +3256,6 @@
"intervalFactor": 1,
"legendFormat": "{{job}}-{{index}} {{name}}",
"refId": "A"
- },
- {
- "expr": "",
- "format": "time_series",
- "intervalFactor": 1,
- "refId": "B"
}
],
"thresholds": [],
@@ -2731,7 +3264,7 @@
"timeShift": null,
"title": "DB usage by background jobs (including scheduling time)",
"tooltip": {
- "shared": true,
+ "shared": false,
"sort": 0,
"value_type": "individual"
},
@@ -2772,6 +3305,13 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2794,10 +3334,8 @@
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -2864,7 +3402,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 32
+ "y": 30
},
"id": 81,
"panels": [
@@ -2874,13 +3412,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 6
+ "y": 33
},
"hiddenSeries": false,
"id": 79,
@@ -2897,11 +3442,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -2970,13 +3513,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 6
+ "y": 33
},
"hiddenSeries": false,
"id": 83,
@@ -2993,11 +3543,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3068,13 +3616,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 15
+ "y": 42
},
"hiddenSeries": false,
"id": 109,
@@ -3091,11 +3646,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3167,13 +3720,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 15
+ "y": 42
},
"hiddenSeries": false,
"id": 111,
@@ -3190,11 +3750,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3258,15 +3816,122 @@
"bars": false,
"dashLength": 10,
"dashes": false,
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "The number of events in the in-memory queues ",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 51
+ },
+ "hiddenSeries": false,
+ "id": 142,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "percentage": false,
+ "pluginVersion": "7.1.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "pending PDUs {{job}}-{{index}}",
+ "refId": "A"
+ },
+ {
+ "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "pending EDUs {{job}}-{{index}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "In-memory federation transmission queues",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "events",
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
"datasource": "$datasource",
"description": "Number of events queued up on the master process for processing by the federation sender",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
- "x": 0,
- "y": 24
+ "x": 12,
+ "y": 51
},
"hiddenSeries": false,
"id": 140,
@@ -3283,11 +3948,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3392,67 +4055,242 @@
}
},
{
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 59
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 166,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "expr": "sum(rate(synapse_event_processing_lag_by_event_bucket{instance=\"$instance\",name=\"federation_sender\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ le }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Federation send PDU lag",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ },
+ {
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "${DS_PROMETHEUS}",
- "description": "The number of events in the in-memory queues ",
- "fill": 1,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
"fillGradient": 0,
"gridPos": {
- "h": 8,
+ "h": 9,
"w": 12,
"x": 12,
- "y": 24
+ "y": 60
},
"hiddenSeries": false,
- "id": 142,
+ "id": 162,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
+ "rightSide": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
+ "linewidth": 0,
+ "links": [],
+ "nullPointMode": "connected",
+ "paceLength": 10,
"percentage": false,
- "pointradius": 2,
+ "pluginVersion": "7.1.3",
+ "pointradius": 5,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "Avg",
+ "fill": 0,
+ "linewidth": 3
+ },
+ {
+ "alias": "99%",
+ "color": "#C4162A",
+ "fillBelowTo": "90%"
+ },
+ {
+ "alias": "90%",
+ "color": "#FF7383",
+ "fillBelowTo": "75%"
+ },
+ {
+ "alias": "75%",
+ "color": "#FFEE52",
+ "fillBelowTo": "50%"
+ },
+ {
+ "alias": "50%",
+ "color": "#73BF69",
+ "fillBelowTo": "25%"
+ },
+ {
+ "alias": "25%",
+ "color": "#1F60C4",
+ "fillBelowTo": "5%"
+ },
+ {
+ "alias": "5%",
+ "lines": false
+ },
+ {
+ "alias": "Average",
+ "color": "rgb(255, 255, 255)",
+ "lines": true,
+ "linewidth": 3
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
- "expr": "synapse_federation_transaction_queue_pending_pdus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": "histogram_quantile(0.99, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
"interval": "",
- "legendFormat": "pending PDUs {{job}}-{{index}}",
+ "intervalFactor": 1,
+ "legendFormat": "99%",
+ "refId": "D"
+ },
+ {
+ "expr": "histogram_quantile(0.9, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "90%",
"refId": "A"
},
{
- "expr": "synapse_federation_transaction_queue_pending_edus{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": "histogram_quantile(0.75, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
"interval": "",
- "legendFormat": "pending EDUs {{job}}-{{index}}",
+ "intervalFactor": 1,
+ "legendFormat": "75%",
+ "refId": "C"
+ },
+ {
+ "expr": "histogram_quantile(0.5, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "50%",
"refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.25, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "interval": "",
+ "legendFormat": "25%",
+ "refId": "F"
+ },
+ {
+ "expr": "histogram_quantile(0.05, sum(rate(synapse_event_processing_lag_by_event_bucket{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) by (le))",
+ "interval": "",
+ "legendFormat": "5%",
+ "refId": "G"
+ },
+ {
+ "expr": "sum(rate(synapse_event_processing_lag_by_event_sum{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size])) / sum(rate(synapse_event_processing_lag_by_event_count{name='federation_sender',index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Average",
+ "refId": "H"
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "warning",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 0.25,
+ "yaxis": "left"
+ },
+ {
+ "colorMode": "critical",
+ "fill": false,
+ "line": true,
+ "op": "gt",
+ "value": 1,
+ "yaxis": "left"
}
],
- "thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "In-memory federation transmission queues",
+ "title": "Federation send PDU lag quantiles",
"tooltip": {
"shared": true,
- "sort": 0,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -3465,21 +4303,20 @@
},
"yaxes": [
{
- "$$hashKey": "object:317",
- "format": "short",
- "label": "events",
+ "decimals": null,
+ "format": "s",
+ "label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
- "$$hashKey": "object:318",
- "format": "short",
+ "format": "hertz",
"label": "",
"logBase": 1,
"max": null,
- "min": null,
+ "min": "0",
"show": true
}
],
@@ -3487,6 +4324,78 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 68
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 164,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "expr": "sum(rate(synapse_federation_server_pdu_process_time_bucket{instance=\"$instance\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{ le }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Handle inbound PDU time",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
}
],
"title": "Federation",
@@ -3499,7 +4408,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 33
+ "y": 31
},
"id": 60,
"panels": [
@@ -3509,6 +4418,13 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -3532,11 +4448,9 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3611,6 +4525,13 @@
"dashes": false,
"datasource": "$datasource",
"description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -3634,10 +4555,8 @@
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -3705,7 +4624,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 34
+ "y": 32
},
"id": 58,
"panels": [
@@ -3715,13 +4634,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 79
+ "y": 8
},
"hiddenSeries": false,
"id": 48,
@@ -3739,10 +4665,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3809,13 +4736,20 @@
"dashes": false,
"datasource": "$datasource",
"description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 79
+ "y": 8
},
"hiddenSeries": false,
"id": 104,
@@ -3834,10 +4768,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -3928,6 +4863,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 0,
"fillGradient": 0,
"grid": {},
@@ -3935,7 +4877,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 86
+ "y": 15
},
"hiddenSeries": false,
"id": 10,
@@ -3955,10 +4897,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4024,6 +4967,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4031,7 +4981,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 86
+ "y": 15
},
"hiddenSeries": false,
"id": 11,
@@ -4051,10 +5001,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4078,7 +5029,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Top DB transactions by total txn time",
+ "title": "DB transactions by total txn time",
"tooltip": {
"shared": false,
"sort": 0,
@@ -4112,6 +5063,111 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 0,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 22
+ },
+ "hiddenSeries": false,
+ "id": 180,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": false
+ },
+ "paceLength": 10,
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{job}}-{{index}} {{desc}}",
+ "refId": "A",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Average DB txn time",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"repeat": null,
@@ -4125,7 +5181,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 35
+ "y": 33
},
"id": 59,
"panels": [
@@ -4137,6 +5193,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4144,7 +5207,7 @@
"h": 13,
"w": 12,
"x": 0,
- "y": 80
+ "y": 9
},
"hiddenSeries": false,
"id": 12,
@@ -4162,11 +5225,9 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4191,8 +5252,8 @@
"timeShift": null,
"title": "Total CPU Usage by Block",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "cumulative"
},
"type": "graph",
@@ -4232,6 +5293,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4239,7 +5307,7 @@
"h": 13,
"w": 12,
"x": 12,
- "y": 80
+ "y": 9
},
"hiddenSeries": false,
"id": 26,
@@ -4257,11 +5325,9 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4286,8 +5352,8 @@
"timeShift": null,
"title": "Average CPU Time per Block",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "cumulative"
},
"type": "graph",
@@ -4327,6 +5393,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4334,7 +5407,7 @@
"h": 13,
"w": 12,
"x": 0,
- "y": 93
+ "y": 22
},
"hiddenSeries": false,
"id": 13,
@@ -4352,11 +5425,9 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4381,8 +5452,8 @@
"timeShift": null,
"title": "Total DB Usage by Block",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "cumulative"
},
"type": "graph",
@@ -4423,6 +5494,13 @@
"description": "The time each database transaction takes to execute, on average, broken down by metrics block.",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4430,7 +5508,7 @@
"h": 13,
"w": 12,
"x": 12,
- "y": 93
+ "y": 22
},
"hiddenSeries": false,
"id": 27,
@@ -4448,11 +5526,9 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4477,8 +5553,8 @@
"timeShift": null,
"title": "Average Database Transaction time, by Block",
"tooltip": {
- "shared": false,
- "sort": 0,
+ "shared": true,
+ "sort": 2,
"value_type": "cumulative"
},
"type": "graph",
@@ -4518,6 +5594,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4525,7 +5608,7 @@
"h": 13,
"w": 12,
"x": 0,
- "y": 106
+ "y": 35
},
"hiddenSeries": false,
"id": 28,
@@ -4542,11 +5625,9 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4612,6 +5693,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4619,7 +5707,7 @@
"h": 13,
"w": 12,
"x": 12,
- "y": 106
+ "y": 35
},
"hiddenSeries": false,
"id": 25,
@@ -4636,11 +5724,9 @@
"linewidth": 2,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4697,6 +5783,99 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 15,
+ "w": 12,
+ "x": 0,
+ "y": 48
+ },
+ "hiddenSeries": false,
+ "id": 154,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "percentage": false,
+ "pluginVersion": "7.1.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{job}}-{{index}} {{block_name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Block count",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"repeat": null,
@@ -4710,7 +5889,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 36
+ "y": 34
},
"id": 61,
"panels": [
@@ -4723,6 +5902,13 @@
"decimals": 2,
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 0,
"fillGradient": 0,
"grid": {},
@@ -4730,7 +5916,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 37
+ "y": 84
},
"hiddenSeries": false,
"id": 1,
@@ -4751,9 +5937,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4821,6 +6008,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4828,7 +6022,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 37
+ "y": 84
},
"hiddenSeries": false,
"id": 8,
@@ -4848,9 +6042,10 @@
"links": [],
"nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -4917,6 +6112,13 @@
"datasource": "$datasource",
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -4924,7 +6126,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 47
+ "y": 94
},
"hiddenSeries": false,
"id": 38,
@@ -4944,9 +6146,10 @@
"links": [],
"nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5010,13 +6213,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
- "y": 47
+ "y": 94
},
"hiddenSeries": false,
"id": 39,
@@ -5035,9 +6245,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5102,13 +6313,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 57
+ "y": 104
},
"hiddenSeries": false,
"id": 65,
@@ -5127,9 +6345,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5200,7 +6419,221 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 37
+ "y": 35
+ },
+ "id": 148,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 146,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "synapse_util_caches_response_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "interval": "",
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Response cache size",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 29
+ },
+ "hiddenSeries": false,
+ "id": 150,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_util_caches_response_cache:hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache:total{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{name}} {{job}}-{{index}}",
+ "refId": "A"
+ },
+ {
+ "expr": "",
+ "interval": "",
+ "legendFormat": "",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Response cache hit rate",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": null,
+ "format": "percentunit",
+ "label": null,
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Response caches",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 36
},
"id": 62,
"panels": [
@@ -5210,13 +6643,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 121
+ "y": 30
},
"hiddenSeries": false,
"id": 91,
@@ -5234,9 +6674,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5305,6 +6746,13 @@
"decimals": 3,
"editable": true,
"error": false,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"grid": {},
@@ -5312,7 +6760,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 121
+ "y": 30
},
"hiddenSeries": false,
"id": 21,
@@ -5331,9 +6779,10 @@
"links": [],
"nullPointMode": "null as zero",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5398,13 +6847,20 @@
"dashes": false,
"datasource": "$datasource",
"description": "'gen 0' shows the number of objects allocated since the last gen0 GC.\n'gen 1' / 'gen 2' show the number of gen0/gen1 GCs since the last gen1/gen2 GC.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 130
+ "y": 39
},
"hiddenSeries": false,
"id": 89,
@@ -5424,9 +6880,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5496,13 +6953,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 130
+ "y": 39
},
"hiddenSeries": false,
"id": 93,
@@ -5520,9 +6984,10 @@
"links": [],
"nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5586,13 +7051,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 139
+ "y": 48
},
"hiddenSeries": false,
"id": 95,
@@ -5610,9 +7082,10 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5686,11 +7159,17 @@
},
"dataFormat": "tsbuckets",
"datasource": "${DS_PROMETHEUS}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 139
+ "y": 48
},
"heatmap": {},
"hideZeroBuckets": true,
@@ -5746,7 +7225,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 38
+ "y": 37
},
"id": 63,
"panels": [
@@ -5756,16 +7235,23 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 66
+ "y": 13
},
"hiddenSeries": false,
- "id": 2,
+ "id": 42,
"legend": {
"avg": false,
"current": false,
@@ -5780,10 +7266,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -5793,51 +7280,19 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_replication_tcp_resource_user_sync{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+ "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
"format": "time_series",
"intervalFactor": 2,
- "legendFormat": "user started/stopped syncing",
+ "legendFormat": "{{job}}-{{index}} {{command}}",
"refId": "A",
"step": 20
- },
- {
- "expr": "rate(synapse_replication_tcp_resource_federation_ack{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "federation ack",
- "refId": "B",
- "step": 20
- },
- {
- "expr": "rate(synapse_replication_tcp_resource_remove_pusher{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "remove pusher",
- "refId": "C",
- "step": 20
- },
- {
- "expr": "rate(synapse_replication_tcp_resource_invalidate_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "invalidate cache",
- "refId": "D",
- "step": 20
- },
- {
- "expr": "rate(synapse_replication_tcp_resource_user_ip_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "user ip cache",
- "refId": "E",
- "step": 20
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Rate of events on replication master",
+ "title": "Rate of incoming commands",
"tooltip": {
"shared": false,
"sort": 0,
@@ -5879,17 +7334,25 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "$datasource",
+ "datasource": "${DS_PROMETHEUS}",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 66
+ "y": 13
},
"hiddenSeries": false,
- "id": 41,
+ "id": 144,
"legend": {
"avg": false,
"current": false,
@@ -5901,14 +7364,13 @@
},
"lines": true,
"linewidth": 1,
- "links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
- "paceLength": 10,
"percentage": false,
- "pointradius": 5,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
@@ -5917,20 +7379,17 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
- "format": "time_series",
+ "expr": "synapse_replication_tcp_command_queue{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"interval": "",
- "intervalFactor": 2,
- "legendFormat": "{{stream_name}}",
- "refId": "A",
- "step": 20
+ "legendFormat": "{{stream_name}} {{job}}-{{index}}",
+ "refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Outgoing stream updates",
+ "title": "Queued incoming RDATA commands, by stream",
"tooltip": {
"shared": false,
"sort": 0,
@@ -5946,7 +7405,7 @@
},
"yaxes": [
{
- "format": "hertz",
+ "format": "short",
"label": null,
"logBase": 1,
"max": null,
@@ -5973,16 +7432,23 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 73
+ "y": 20
},
"hiddenSeries": false,
- "id": 42,
+ "id": 43,
"legend": {
"avg": false,
"current": false,
@@ -5997,10 +7463,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6010,7 +7477,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{job}}-{{index}} {{command}}",
@@ -6022,7 +7489,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Rate of incoming commands",
+ "title": "Rate of outgoing commands",
"tooltip": {
"shared": false,
"sort": 0,
@@ -6065,16 +7532,23 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 73
+ "y": 20
},
"hiddenSeries": false,
- "id": 43,
+ "id": 41,
"legend": {
"avg": false,
"current": false,
@@ -6089,10 +7563,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6102,10 +7577,11 @@
"steppedLine": false,
"targets": [
{
- "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+ "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
"format": "time_series",
+ "interval": "",
"intervalFactor": 2,
- "legendFormat": "{{job}}-{{index}} {{command}}",
+ "legendFormat": "{{stream_name}}",
"refId": "A",
"step": 20
}
@@ -6114,7 +7590,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Rate of outgoing commands",
+ "title": "Outgoing stream updates",
"tooltip": {
"shared": false,
"sort": 0,
@@ -6157,13 +7633,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
- "y": 80
+ "y": 27
},
"hiddenSeries": false,
"id": 113,
@@ -6181,10 +7664,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6255,13 +7739,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
- "y": 80
+ "y": 27
},
"hiddenSeries": false,
"id": 115,
@@ -6279,10 +7770,11 @@
"links": [],
"nullPointMode": "null",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6352,7 +7844,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 39
+ "y": 38
},
"id": 69,
"panels": [
@@ -6362,13 +7854,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 40
+ "y": 41
},
"hiddenSeries": false,
"id": 67,
@@ -6386,10 +7885,11 @@
"links": [],
"nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6399,7 +7899,7 @@
"steppedLine": false,
"targets": [
{
- "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - ignoring(instance,index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+ "expr": "max(synapse_event_persisted_position{instance=\"$instance\"}) - on() group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
@@ -6431,7 +7931,7 @@
"label": "events",
"logBase": 1,
"max": null,
- "min": null,
+ "min": "0",
"show": true
},
{
@@ -6454,13 +7954,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
- "y": 40
+ "y": 41
},
"hiddenSeries": false,
"id": 71,
@@ -6478,10 +7985,11 @@
"links": [],
"nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6524,7 +8032,7 @@
"label": null,
"logBase": 1,
"max": null,
- "min": null,
+ "min": "0",
"show": true
},
{
@@ -6547,13 +8055,20 @@
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
- "y": 49
+ "y": 50
},
"hiddenSeries": false,
"id": 121,
@@ -6572,10 +8087,11 @@
"links": [],
"nullPointMode": "connected",
"options": {
- "dataLinks": []
+ "alertThreshold": true
},
"paceLength": 10,
"percentage": false,
+ "pluginVersion": "7.3.7",
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -6647,7 +8163,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 40
+ "y": 39
},
"id": 126,
"panels": [
@@ -6668,11 +8184,17 @@
"dataFormat": "tsbuckets",
"datasource": "$datasource",
"description": "Colour reflects the number of rooms with the given number of forward extremities, or fewer.\n\nThis is only updated once an hour.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 86
+ "y": 42
},
"heatmap": {},
"hideZeroBuckets": true,
@@ -6725,13 +8247,20 @@
"dashes": false,
"datasource": "$datasource",
"description": "Number of rooms with the given number of forward extremities or fewer.\n\nThis is only updated once an hour.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 86
+ "y": 42
},
"hiddenSeries": false,
"id": 124,
@@ -6748,11 +8277,9 @@
"lines": true,
"linewidth": 1,
"links": [],
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
+ "nullPointMode": "connected",
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -6763,7 +8290,7 @@
"targets": [
{
"expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} > 0",
- "format": "time_series",
+ "format": "heatmap",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{le}}",
@@ -6776,8 +8303,8 @@
"timeShift": null,
"title": "Room counts, by number of extremities",
"tooltip": {
- "shared": false,
- "sort": 1,
+ "shared": true,
+ "sort": 2,
"value_type": "individual"
},
"type": "graph",
@@ -6793,7 +8320,7 @@
"decimals": null,
"format": "none",
"label": "Number of rooms",
- "logBase": 1,
+ "logBase": 10,
"max": null,
"min": null,
"show": true
@@ -6828,11 +8355,17 @@
"dataFormat": "tsbuckets",
"datasource": "$datasource",
"description": "Colour reflects the number of events persisted to rooms with the given number of forward extremities, or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 94
+ "y": 50
},
"heatmap": {},
"hideZeroBuckets": true,
@@ -6885,13 +8418,20 @@
"dashes": false,
"datasource": "$datasource",
"description": "For a given percentage P, the number X where P% of events were persisted to rooms with X forward extremities or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 94
+ "y": 50
},
"hiddenSeries": false,
"id": 128,
@@ -6908,10 +8448,8 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -7006,11 +8544,17 @@
"dataFormat": "tsbuckets",
"datasource": "$datasource",
"description": "Colour reflects the number of events persisted to rooms with the given number of stale forward extremities, or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 102
+ "y": 58
},
"heatmap": {},
"hideZeroBuckets": true,
@@ -7063,13 +8607,20 @@
"dashes": false,
"datasource": "$datasource",
"description": "For given percentage P, the number X where P% of events were persisted to rooms with X stale forward extremities or fewer.\n\nStale forward extremities are those that were in the previous set of extremities as well as the new.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 102
+ "y": 58
},
"hiddenSeries": false,
"id": 130,
@@ -7086,10 +8637,8 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -7184,11 +8733,17 @@
"dataFormat": "tsbuckets",
"datasource": "$datasource",
"description": "Colour reflects the number of state resolution operations performed over the given number of state groups, or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 110
+ "y": 66
},
"heatmap": {},
"hideZeroBuckets": true,
@@ -7242,13 +8797,20 @@
"dashes": false,
"datasource": "$datasource",
"description": "For a given percentage P, the number X where P% of state resolution operations took place over X state groups or fewer.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 110
+ "y": 66
},
"hiddenSeries": false,
"id": 132,
@@ -7266,10 +8828,8 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
"percentage": false,
+ "pluginVersion": "7.1.3",
"pointradius": 2,
"points": false,
"renderer": "flot",
@@ -7351,14 +8911,1366 @@
"align": false,
"alignLevel": null
}
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "When we do a state res while persisting events we try and see if we can prune any stale extremities.",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 74
+ },
+ "hiddenSeries": false,
+ "id": 179,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "percentage": false,
+ "pluginVersion": "7.1.3",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "State res ",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Potential to prune",
+ "refId": "B"
+ },
+ {
+ "expr": "sum(rate(synapse_storage_events_times_pruned_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+ "interval": "",
+ "legendFormat": "Pruned",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Stale extremity dropping",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
],
"title": "Extremities",
"type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 40
+ },
+ "id": 158,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 119
+ },
+ "hiddenSeries": false,
+ "id": 156,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "Max",
+ "color": "#bf1b00",
+ "fill": 0,
+ "linewidth": 2
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "synapse_admin_mau:current{instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Current",
+ "refId": "A"
+ },
+ {
+ "expr": "synapse_admin_mau:max{instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Max",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "MAU Limits",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 119
+ },
+ "hiddenSeries": false,
+ "id": 160,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "synapse_admin_mau_current_mau_by_service{instance=\"$instance\"}",
+ "interval": "",
+ "legendFormat": "{{ app_service }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "MAU by Appservice",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "MAU",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 177,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 173,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pluginVersion": "7.1.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_notifier_users_woken_by_stream{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{stream}} {{index}}",
+ "metric": "synapse_notifier",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Notifier Streams Woken",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 175,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pluginVersion": "7.1.3",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_handler_presence_get_updates{job=~\"$job\",instance=\"$instance\"}[$bucket_size])",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "{{type}} {{index}}",
+ "refId": "A",
+ "step": 2
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Presence Stream Fetch Type Rates",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Notifier",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 42
+ },
+ "id": 170,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 73
+ },
+ "hiddenSeries": false,
+ "id": 168,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_appservice_api_sent_events{instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{exported_service}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Sent Events rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 73
+ },
+ "hiddenSeries": false,
+ "id": 171,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_appservice_api_sent_transactions{instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{exported_service}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Transactions rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Appservices",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 43
+ },
+ "id": 188,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 182,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_handler_presence_notified_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Notified",
+ "refId": "A"
+ },
+ {
+ "expr": "rate(synapse_handler_presence_federation_presence_out{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Remote ping",
+ "refId": "B"
+ },
+ {
+ "expr": "rate(synapse_handler_presence_presence_updates{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Total updates",
+ "refId": "C"
+ },
+ {
+ "expr": "rate(synapse_handler_presence_federation_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Remote updates",
+ "refId": "D"
+ },
+ {
+ "expr": "rate(synapse_handler_presence_bump_active_time{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "Bump active time",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Presence",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 44
+ },
+ "hiddenSeries": false,
+ "id": 184,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_handler_presence_state_transition{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{from}} -> {{to}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Presence state transitions",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 52
+ },
+ "hiddenSeries": false,
+ "id": 186,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_handler_presence_notify_reason{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{reason}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Presence notify reason",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:165",
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:166",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "title": "Presence",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "datasource": "${DS_PROMETHEUS}",
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 44
+ },
+ "id": 197,
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 191,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_external_cache_set{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{ index }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "External Cache Set Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:390",
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:391",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "$datasource",
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 193,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "7.3.7",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate(synapse_external_cache_get{job=\"$job\", instance=\"$instance\", index=~\"$index\"}[$bucket_size])",
+ "interval": "",
+ "legendFormat": "{{ cache_name }} {{ index }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "External Cache Get Rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:390",
+ "format": "hertz",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:391",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "cards": {
+ "cardPadding": -1,
+ "cardRound": null
+ },
+ "color": {
+ "cardColor": "#b4ff00",
+ "colorScale": "sqrt",
+ "colorScheme": "interpolateInferno",
+ "exponent": 0.5,
+ "min": 0,
+ "mode": "spectrum"
+ },
+ "dataFormat": "tsbuckets",
+ "datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "heatmap": {},
+ "hideZeroBuckets": false,
+ "highlightCards": true,
+ "id": 195,
+ "legend": {
+ "show": false
+ },
+ "links": [],
+ "reverseYBuckets": false,
+ "targets": [
+ {
+ "expr": "sum(rate(synapse_external_cache_response_time_seconds_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le)",
+ "format": "heatmap",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "{{le}}",
+ "refId": "A"
+ }
+ ],
+ "title": "External Cache Response Time",
+ "tooltip": {
+ "show": true,
+ "showHistogram": true
+ },
+ "tooltipDecimals": 2,
+ "type": "heatmap",
+ "xAxis": {
+ "show": true
+ },
+ "xBucketNumber": null,
+ "xBucketSize": null,
+ "yAxis": {
+ "decimals": 0,
+ "format": "s",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true,
+ "splitFactor": null
+ },
+ "yBucketBound": "auto",
+ "yBucketNumber": null,
+ "yBucketSize": null
+ }
+ ],
+ "title": "External Cache",
+ "type": "row"
}
],
- "refresh": "5m",
- "schemaVersion": 22,
+ "refresh": false,
+ "schemaVersion": 26,
"style": "dark",
"tags": [
"matrix"
@@ -7368,9 +10280,10 @@
{
"current": {
"selected": false,
- "text": "Prometheus",
- "value": "Prometheus"
+ "text": "default",
+ "value": "default"
},
+ "error": null,
"hide": 0,
"includeAll": false,
"label": null,
@@ -7378,6 +10291,7 @@
"name": "datasource",
"options": [],
"query": "prometheus",
+ "queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
@@ -7387,13 +10301,14 @@
"allFormat": "glob",
"auto": true,
"auto_count": 100,
- "auto_min": "30s",
+ "auto_min": "60s",
"current": {
"selected": false,
"text": "auto",
"value": "$__auto_interval_bucket_size"
},
"datasource": null,
+ "error": null,
"hide": 0,
"includeAll": false,
"label": "Bucket Size",
@@ -7438,6 +10353,7 @@
}
],
"query": "30s,1m,2m,5m,10m,15m",
+ "queryValue": "",
"refresh": 2,
"skipUrlSync": false,
"type": "interval"
@@ -7447,9 +10363,9 @@
"current": {},
"datasource": "$datasource",
"definition": "",
+ "error": null,
"hide": 0,
"includeAll": false,
- "index": -1,
"label": null,
"multi": false,
"name": "instance",
@@ -7458,7 +10374,7 @@
"refresh": 2,
"regex": "",
"skipUrlSync": false,
- "sort": 0,
+ "sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
@@ -7471,10 +10387,10 @@
"current": {},
"datasource": "$datasource",
"definition": "",
+ "error": null,
"hide": 0,
"hideLabel": false,
"includeAll": true,
- "index": -1,
"label": "Job",
"multi": true,
"multiFormat": "regex values",
@@ -7498,10 +10414,10 @@
"current": {},
"datasource": "$datasource",
"definition": "",
+ "error": null,
"hide": 0,
"hideLabel": false,
"includeAll": true,
- "index": -1,
"label": "",
"multi": true,
"multiFormat": "regex values",
@@ -7522,7 +10438,7 @@
]
},
"time": {
- "from": "now-1h",
+ "from": "now-3h",
"to": "now"
},
"timepicker": {
@@ -7554,8 +10470,5 @@
"timezone": "",
"title": "Synapse",
"uid": "000000012",
- "variables": {
- "list": []
- },
- "version": 32
+ "version": 90
}
\ No newline at end of file
|
