From e9f5812efff29b08bd35724689d10a362b410168 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Apr 2021 11:59:43 +0100 Subject: Track memory usage of caches --- synapse/util/caches/__init__.py | 8 +++++++ synapse/util/caches/lrucache.py | 52 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 46af7fa473..8e253cb518 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -32,6 +32,11 @@ cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"]) cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"]) cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"]) cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"]) +cache_memory_usage = Gauge( + "synapse_util_caches_cache_memory_usage", + "Estimated size in bytes of the caches", + ["name"], +) response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"]) response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"]) @@ -52,6 +57,7 @@ class CacheMetric: hits = attr.ib(default=0) misses = attr.ib(default=0) evicted_size = attr.ib(default=0) + memory_usage = attr.ib(default=None) def inc_hits(self): self.hits += 1 @@ -81,6 +87,8 @@ class CacheMetric: cache_total.labels(self._cache_name).set(self.hits + self.misses) if getattr(self._cache, "max_size", None): cache_max_size.labels(self._cache_name).set(self._cache.max_size) + if self.memory_usage is not None: + cache_memory_usage.labels(self._cache_name).set(self.memory_usage) if self._collect_callback: self._collect_callback() except Exception as e: diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index a21d34fcb4..97cc77156a 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -33,6 +33,33 @@ from synapse.config import cache as cache_config from synapse.util.caches import CacheMetric, register_cache from synapse.util.caches.treecache import TreeCache +try: + from pympler.asizeof import Asizer + + sizer = Asizer() + sizer.exclude_refs((), None, "") + + def _get_size_of(val: Any, *, recurse=True) -> int: + """Get an estimate of the size in bytes of the object. + + Args: + val: The object to size. + recurse: If true will include referenced values in the size, + otherwise only sizes the given object. + """ + return sizer.asizeof(val, limit=100 if recurse else 0) + + +except ImportError: + + def _get_size_of(val: Any, recurse=True) -> int: + return 0 + + +# Whether to track estimated memory usage of the LruCaches. +TRACK_MEMORY_USAGE = True + + # Function type: the type used for invalidation callbacks FT = TypeVar("FT", bound=Callable[..., Any]) @@ -54,7 +81,7 @@ def enumerate_leaves(node, depth): class _Node: - __slots__ = ["prev_node", "next_node", "key", "value", "callbacks"] + __slots__ = ["prev_node", "next_node", "key", "value", "callbacks", "memory"] def __init__( self, prev_node, next_node, key, value, callbacks: Optional[set] = None @@ -65,6 +92,16 @@ class _Node: self.value = value self.callbacks = callbacks or set() + self.memory = 0 + if TRACK_MEMORY_USAGE: + self.memory = ( + _get_size_of(key) + + _get_size_of(value) + + _get_size_of(self.callbacks, recurse=False) + + _get_size_of(self, recurse=False) + ) + self.memory += _get_size_of(self.memory, recurse=False) + class LruCache(Generic[KT, VT]): """ @@ -136,6 +173,9 @@ class LruCache(Generic[KT, VT]): self, collect_callback=metrics_collection_callback, ) # type: Optional[CacheMetric] + + if TRACK_MEMORY_USAGE and metrics: + metrics.memory_usage = 0 else: metrics = None @@ -188,6 +228,9 @@ class LruCache(Generic[KT, VT]): if size_callback: cached_cache_len[0] += size_callback(node.value) + if TRACK_MEMORY_USAGE and metrics: + metrics.memory_usage += node.memory + def move_node_to_front(node): prev_node = node.prev_node next_node = node.next_node @@ -214,6 +257,10 @@ class LruCache(Generic[KT, VT]): for cb in node.callbacks: cb() node.callbacks.clear() + + if TRACK_MEMORY_USAGE and metrics: + metrics.memory_usage -= node.memory + return deleted_len @overload @@ -332,6 +379,9 @@ class LruCache(Generic[KT, VT]): if size_callback: cached_cache_len[0] = 0 + if TRACK_MEMORY_USAGE and metrics: + metrics.memory_usage = 0 + @synchronized def cache_contains(key: KT) -> bool: return key in cache -- cgit 1.5.1 From 5003bd29d24c9ecb74ffe7c8d1898c339ef317f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 23 Apr 2021 17:16:49 +0100 Subject: Don't have a global Asizer --- synapse/util/caches/lrucache.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 97cc77156a..fdf3b726b0 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -36,9 +36,6 @@ from synapse.util.caches.treecache import TreeCache try: from pympler.asizeof import Asizer - sizer = Asizer() - sizer.exclude_refs((), None, "") - def _get_size_of(val: Any, *, recurse=True) -> int: """Get an estimate of the size in bytes of the object. @@ -47,12 +44,14 @@ try: recurse: If true will include referenced values in the size, otherwise only sizes the given object. """ + sizer = Asizer() + sizer.exclude_refs((), None, "") return sizer.asizeof(val, limit=100 if recurse else 0) except ImportError: - def _get_size_of(val: Any, recurse=True) -> int: + def _get_size_of(val: Any, *, recurse=True) -> int: return 0 -- cgit 1.5.1 From 0c9bab290f921514532e73fdce6ef5d0b87222d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Apr 2021 10:29:26 +0100 Subject: Ignore singletons --- synapse/util/caches/lrucache.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index fdf3b726b0..b018733288 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -44,6 +44,10 @@ try: recurse: If true will include referenced values in the size, otherwise only sizes the given object. """ + # Ignore singleton values when calculating memory usage. + if val in ((), None, ""): + return 0 + sizer = Asizer() sizer.exclude_refs((), None, "") return sizer.asizeof(val, limit=100 if recurse else 0) -- cgit 1.5.1 From 567fe5e387dc29ab1de30481f1e37c5047bb25cf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Apr 2021 10:37:26 +0100 Subject: Make TRACK_MEMORY_USAGE configurable --- synapse/app/generic_worker.py | 1 + synapse/app/homeserver.py | 1 + synapse/config/cache.py | 11 +++++++++++ synapse/python_dependencies.py | 2 +- synapse/util/caches/lrucache.py | 2 +- 5 files changed, 15 insertions(+), 2 deletions(-) (limited to 'synapse/util/caches') diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 7b2ac3ca64..d831f793b9 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -452,6 +452,7 @@ def start(config_options): config.server.update_user_directory = False synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts + synapse.util.caches.lrucache.TRACK_MEMORY_USAGE = config.caches.track_memory_usage hs = GenericWorkerServer( config.server_name, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8be8b520eb..fd7958cecd 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -339,6 +339,7 @@ def setup(config_options): sys.exit(0) events.USE_FROZEN_DICTS = config.use_frozen_dicts + synapse.util.caches.lrucache.TRACK_MEMORY_USAGE = config.caches.track_memory_usage hs = SynapseHomeServer( config.server_name, diff --git a/synapse/config/cache.py b/synapse/config/cache.py index 41b9b3f51f..91165ee1ce 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -17,6 +17,8 @@ import re import threading from typing import Callable, Dict +from synapse.python_dependencies import DependencyException, check_requirements + from ._base import Config, ConfigError # The prefix for all cache factor-related environment variables @@ -189,6 +191,15 @@ class CacheConfig(Config): ) self.cache_factors[cache] = factor + self.track_memory_usage = cache_config.get("track_memory_usage", False) + if self.track_memory_usage: + try: + check_requirements("cache_memory") + except DependencyException as e: + raise ConfigError( + e.message # noqa: B306, DependencyException.message is a property + ) + # Resize all caches (if necessary) with the new factors we've loaded self.resize_all_caches() diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 77176a6cd9..d709033c27 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -121,7 +121,7 @@ CONDITIONAL_REQUIREMENTS = { # hiredis is not a *strict* dependency, but it makes things much faster. # (if it is not installed, we fall back to slow code.) "redis": ["txredisapi>=1.4.7", "hiredis"], - "cache_memroy": ["pympler"], + "cache_memory": ["pympler"], } ALL_OPTIONAL_REQUIREMENTS = set() # type: Set[str] diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index b018733288..1719b0a1ca 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -60,7 +60,7 @@ except ImportError: # Whether to track estimated memory usage of the LruCaches. -TRACK_MEMORY_USAGE = True +TRACK_MEMORY_USAGE = False # Function type: the type used for invalidation callbacks -- cgit 1.5.1 From 99fb72e63e75c55a969ae176213d3f6c689d880e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Apr 2021 10:50:15 +0100 Subject: Move TRACK_MEMORY_USAGE to root --- synapse/util/caches/__init__.py | 5 +++++ synapse/util/caches/lrucache.py | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 8e253cb518..b2504c2fad 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -24,6 +24,11 @@ from synapse.config.cache import add_resizable_cache logger = logging.getLogger(__name__) + +# Whether to track estimated memory usage of the LruCaches. +TRACK_MEMORY_USAGE = False + + caches_by_name = {} # type: Dict[str, Sized] collectors_by_name = {} # type: Dict[str, CacheMetric] diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 1719b0a1ca..5fc26cedd2 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -30,7 +30,7 @@ from typing import ( from typing_extensions import Literal from synapse.config import cache as cache_config -from synapse.util.caches import CacheMetric, register_cache +from synapse.util.caches import TRACK_MEMORY_USAGE, CacheMetric, register_cache from synapse.util.caches.treecache import TreeCache try: @@ -59,10 +59,6 @@ except ImportError: return 0 -# Whether to track estimated memory usage of the LruCaches. -TRACK_MEMORY_USAGE = False - - # Function type: the type used for invalidation callbacks FT = TypeVar("FT", bound=Callable[..., Any]) -- cgit 1.5.1 From bcf8858b67e44f1a649caa58003a2a0ff0495c13 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Apr 2021 10:56:42 +0100 Subject: Don't explode if memory has been twiddled --- synapse/util/caches/__init__.py | 13 +++++++++++++ synapse/util/caches/lrucache.py | 9 +++------ 2 files changed, 16 insertions(+), 6 deletions(-) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index b2504c2fad..600d10b89e 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -73,6 +73,19 @@ class CacheMetric: def inc_evictions(self, size=1): self.evicted_size += size + def inc_memory_usage(self, memory: int): + if self.memory_usage is None: + self.memory_usage = 0 + + self.memory_usage += memory + + def dec_memory_usage(self, memory: int): + self.memory_usage -= memory + + def clear_memory_usage(self, memory: int): + if self.memory_usage is not None: + self.memory_usage = 0 + def describe(self): return [] diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 5fc26cedd2..5c8ef444fa 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -172,9 +172,6 @@ class LruCache(Generic[KT, VT]): self, collect_callback=metrics_collection_callback, ) # type: Optional[CacheMetric] - - if TRACK_MEMORY_USAGE and metrics: - metrics.memory_usage = 0 else: metrics = None @@ -228,7 +225,7 @@ class LruCache(Generic[KT, VT]): cached_cache_len[0] += size_callback(node.value) if TRACK_MEMORY_USAGE and metrics: - metrics.memory_usage += node.memory + metrics.inc_memory_usage(node.memory) def move_node_to_front(node): prev_node = node.prev_node @@ -258,7 +255,7 @@ class LruCache(Generic[KT, VT]): node.callbacks.clear() if TRACK_MEMORY_USAGE and metrics: - metrics.memory_usage -= node.memory + metrics.dec_memory_usage(node.memory) return deleted_len @@ -379,7 +376,7 @@ class LruCache(Generic[KT, VT]): cached_cache_len[0] = 0 if TRACK_MEMORY_USAGE and metrics: - metrics.memory_usage = 0 + metrics.clear_memory_usage() @synchronized def cache_contains(key: KT) -> bool: -- cgit 1.5.1 From 2bf93f9b34a5d55f29728e212a4883eae8890b0d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Apr 2021 10:58:04 +0100 Subject: Fix --- synapse/util/caches/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 600d10b89e..fc97d956e9 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -82,7 +82,7 @@ class CacheMetric: def dec_memory_usage(self, memory: int): self.memory_usage -= memory - def clear_memory_usage(self, memory: int): + def clear_memory_usage(self): if self.memory_usage is not None: self.memory_usage = 0 -- cgit 1.5.1 From a99524f3831b3da25da5ddad905f33bfd3f2502e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 May 2021 14:29:26 +0100 Subject: Apply suggestions from code review Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- synapse/app/generic_worker.py | 2 +- synapse/app/homeserver.py | 2 +- synapse/util/caches/__init__.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse/util/caches') diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index d831f793b9..d463ca3d87 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -452,7 +452,7 @@ def start(config_options): config.server.update_user_directory = False synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts - synapse.util.caches.lrucache.TRACK_MEMORY_USAGE = config.caches.track_memory_usage + synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage hs = GenericWorkerServer( config.server_name, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fd7958cecd..11933c699f 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -339,7 +339,7 @@ def setup(config_options): sys.exit(0) events.USE_FROZEN_DICTS = config.use_frozen_dicts - synapse.util.caches.lrucache.TRACK_MEMORY_USAGE = config.caches.track_memory_usage + synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage hs = SynapseHomeServer( config.server_name, diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index fc97d956e9..bac10dc47a 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -38,8 +38,8 @@ cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"]) cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"]) cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"]) cache_memory_usage = Gauge( - "synapse_util_caches_cache_memory_usage", - "Estimated size in bytes of the caches", + "synapse_util_caches_cache_size_bytes", + "Estimated memory usage of the caches", ["name"], ) -- cgit 1.5.1 From 78e3502ada403144d5fbce2cd7edf8a1c36c739a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 May 2021 14:32:42 +0100 Subject: Always report memory usage metrics when TRACK_MEMORY_USAGE is True --- synapse/util/caches/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'synapse/util/caches') diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index bac10dc47a..ca36f07c20 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -105,8 +105,13 @@ class CacheMetric: cache_total.labels(self._cache_name).set(self.hits + self.misses) if getattr(self._cache, "max_size", None): cache_max_size.labels(self._cache_name).set(self._cache.max_size) - if self.memory_usage is not None: - cache_memory_usage.labels(self._cache_name).set(self.memory_usage) + + if TRACK_MEMORY_USAGE: + # self.memory_usage can be None if nothing has been inserted + # into the cache yet. + cache_memory_usage.labels(self._cache_name).set( + self.memory_usage or 0 + ) if self._collect_callback: self._collect_callback() except Exception as e: -- cgit 1.5.1