diff options
| -rw-r--r-- | changelog.d/5750.misc | 1 | ||||
| -rw-r--r-- | synapse/util/caches/__init__.py | 17 | ||||
| -rw-r--r-- | synapse/util/caches/descriptors.py | 18 |
3 files changed, 34 insertions, 2 deletions
diff --git a/changelog.d/5750.misc b/changelog.d/5750.misc new file mode 100644 index 0000000000..6beaa460a5 --- /dev/null +++ b/changelog.d/5750.misc @@ -0,0 +1 @@ +Add a prometheus metric for pending cache lookups. \ No newline at end of file diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 8271229015..b50e3503f0 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2019 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -51,7 +52,19 @@ response_cache_evicted = Gauge( response_cache_total = Gauge("synapse_util_caches_response_cache:total", "", ["name"]) -def register_cache(cache_type, cache_name, cache): +def register_cache(cache_type, cache_name, cache, collect_callback=None): + """Register a cache object for metric collection. + + Args: + cache_type (str): + cache_name (str): name of the cache + cache (object): cache itself + collect_callback (callable|None): if not None, a function which is called during + metric collection to update additional metrics. + + Returns: + CacheMetric: an object which provides inc_{hits,misses,evictions} methods + """ # Check if the metric is already registered. Unregister it, if so. # This usually happens during tests, as at runtime these caches are @@ -90,6 +103,8 @@ def register_cache(cache_type, cache_name, cache): cache_hits.labels(cache_name).set(self.hits) cache_evicted.labels(cache_name).set(self.evicted_size) cache_total.labels(cache_name).set(self.hits + self.misses) + if collect_callback: + collect_callback() except Exception as e: logger.warn("Error calculating metrics for %s: %s", cache_name, e) raise diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index a1acacbde9..7e69cf55fb 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -22,6 +22,8 @@ from collections import namedtuple import six from six import itervalues, string_types +from prometheus_client import Gauge + from twisted.internet import defer from synapse.logging.context import make_deferred_yieldable, preserve_fn @@ -37,6 +39,12 @@ from . import register_cache logger = logging.getLogger(__name__) +cache_pending_metric = Gauge( + "synapse_util_caches_cache_pending", + "Number of lookups currently pending for this cache", + ["name"], +) + _CacheSentinel = object() @@ -82,11 +90,19 @@ class Cache(object): self.name = name self.keylen = keylen self.thread = None - self.metrics = register_cache("cache", name, self.cache) + self.metrics = register_cache( + "cache", + name, + self.cache, + collect_callback=self._metrics_collection_callback, + ) def _on_evicted(self, evicted_count): self.metrics.inc_evictions(evicted_count) + def _metrics_collection_callback(self): + cache_pending_metric.labels(self.name).set(len(self._pending_deferred_cache)) + def check_thread(self): expected_thread = self.thread if expected_thread is None: |