summary refs log tree commit diff
path: root/synapse/metrics/background_process_metrics.py
diff options
context:
space:
mode:
authorAndrew Morgan <andrew@amorgan.xyz>2021-04-21 14:53:02 +0100
committerAndrew Morgan <andrew@amorgan.xyz>2021-04-21 14:54:03 +0100
commita0972085ed2b06bdaaf94a2265cb3512c85423b8 (patch)
treef86e21530ab24f07ceb26a9e723d6909d84f28c1 /synapse/metrics/background_process_metrics.py
parentMention Prometheus metrics regression in v1.32.0 (diff)
parentClarify 1.32.0/1 changelog and upgrade notes (diff)
downloadsynapse-a0972085ed2b06bdaaf94a2265cb3512c85423b8.tar.xz
Merge tag 'v1.32.1'
Synapse 1.32.1 (2021-04-21)
===========================

This release fixes [a regression](https://github.com/matrix-org/synapse/issues/9853) in Synapse 1.32.0 that caused connected Prometheus instances to become unstable. If you ran Synapse 1.32.0 with Prometheus metrics, first upgrade to Synapse 1.32.1 and follow [these instructions](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183) to clean up any excess writeahead logs.

Bugfixes
--------

- Fix a regression in Synapse 1.32.0 which caused Synapse to report large numbers of Prometheus time series, potentially overwhelming Prometheus instances. ([\#9854](https://github.com/matrix-org/synapse/issues/9854))
Diffstat (limited to 'synapse/metrics/background_process_metrics.py')
-rw-r--r--synapse/metrics/background_process_metrics.py20
1 files changed, 16 insertions, 4 deletions
diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py

index 78e9cfbc26..3f621539f3 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py
@@ -16,7 +16,7 @@ import logging import threading from functools import wraps -from typing import TYPE_CHECKING, Dict, Optional, Set +from typing import TYPE_CHECKING, Dict, Optional, Set, Union from prometheus_client.core import REGISTRY, Counter, Gauge @@ -199,7 +199,7 @@ def run_as_background_process(desc: str, func, *args, bg_start_span=True, **kwar _background_process_start_count.labels(desc).inc() _background_process_in_flight_count.labels(desc).inc() - with BackgroundProcessLoggingContext("%s-%s" % (desc, count)) as context: + with BackgroundProcessLoggingContext(desc, count) as context: try: ctx = noop_context_manager() if bg_start_span: @@ -244,8 +244,20 @@ class BackgroundProcessLoggingContext(LoggingContext): __slots__ = ["_proc"] - def __init__(self, name: str): - super().__init__(name) + def __init__(self, name: str, instance_id: Optional[Union[int, str]] = None): + """ + + Args: + name: The name of the background process. Each distinct `name` gets a + separate prometheus time series. + + instance_id: an identifer to add to `name` to distinguish this instance of + the named background process in the logs. If this is `None`, one is + made up based on id(self). + """ + if instance_id is None: + instance_id = id(self) + super().__init__("%s-%s" % (name, instance_id)) self._proc = _BackgroundProcess(name, self) def start(self, rusage: "Optional[resource._RUsage]"):