From a21702fe764db8737d35a895bf5c364edbdf66f7 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 17 Jan 2020 01:48:27 +1100 Subject: cleanup so it can refer to late-config --- synapse/app/homeserver.py | 2 +- synapse/config/cache.py | 2 ++ synapse/config/database.py | 2 -- synapse/storage/data_stores/main/events_worker.py | 2 +- synapse/storage/data_stores/main/relations.py | 4 ++-- synapse/storage/data_stores/state/store.py | 6 ++---- synapse/util/caches/descriptors.py | 5 ++++- tests/storage/test_appservice.py | 10 +++++----- tests/storage/test_base.py | 3 ++- tests/util/test_lrucache.py | 4 ++-- 10 files changed, 21 insertions(+), 19 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0feb5bfa86..2b114551a6 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -484,7 +484,7 @@ def phone_stats_home(hs, stats, stats_process=_stats_process): daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages() stats["daily_sent_messages"] = daily_sent_messages stats["cache_factor"] = hs.config.caches.global_factor - stats["event_cache_size"] = hs.config.event_cache_size + stats["event_cache_size"] = hs.config.caches.event_cache_size # # Performance statistics diff --git a/synapse/config/cache.py b/synapse/config/cache.py index d7921ce050..1d65c64e1a 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -53,6 +53,8 @@ class CacheConfig(Config): section = "caches" def read_config(self, config, **kwargs): + self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K")) + global DEFAULT_CACHE_SIZE_FACTOR cache_config = config.get("caches", {}) diff --git a/synapse/config/database.py b/synapse/config/database.py index 219b32f670..2244ebff3a 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -57,8 +57,6 @@ class DatabaseConfig(Config): section = "database" def read_config(self, config, **kwargs): - self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K")) - # We *experimentally* support specifying multiple databases via the # `databases` key. This is a map from a label to database config in the # same format as the `database` config option, plus an extra diff --git a/synapse/storage/data_stores/main/events_worker.py b/synapse/storage/data_stores/main/events_worker.py index 0cce5232f5..c34a2efe27 100644 --- a/synapse/storage/data_stores/main/events_worker.py +++ b/synapse/storage/data_stores/main/events_worker.py @@ -72,7 +72,7 @@ class EventsWorkerStore(SQLBaseStore): super(EventsWorkerStore, self).__init__(database, db_conn, hs) self._get_event_cache = Cache( - "*getEvent*", keylen=3, max_entries=hs.config.event_cache_size + "*getEvent*", keylen=3, max_entries=hs.config.caches.event_cache_size ) self._event_fetch_lock = threading.Condition() diff --git a/synapse/storage/data_stores/main/relations.py b/synapse/storage/data_stores/main/relations.py index 046c2b4845..98e6149bc6 100644 --- a/synapse/storage/data_stores/main/relations.py +++ b/synapse/storage/data_stores/main/relations.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) class RelationsWorkerStore(SQLBaseStore): - @cached(tree=True) + @cached(tree=True, max_entries="event_cache_size") def get_relations_for_event( self, event_id, @@ -133,7 +133,7 @@ class RelationsWorkerStore(SQLBaseStore): "get_recent_references_for_event", _get_recent_references_for_event_txn ) - @cached(tree=True) + @cached(tree=True, max_entries="event_cache_size") def get_aggregation_groups_for_event( self, event_id, diff --git a/synapse/storage/data_stores/state/store.py b/synapse/storage/data_stores/state/store.py index c4ee9b7ccb..e4c5b904e1 100644 --- a/synapse/storage/data_stores/state/store.py +++ b/synapse/storage/data_stores/state/store.py @@ -28,7 +28,6 @@ from synapse.storage.data_stores.state.bg_updates import StateBackgroundUpdateSt from synapse.storage.database import Database from synapse.storage.state import StateFilter from synapse.types import StateMap -from synapse.util.caches import get_cache_factor_for from synapse.util.caches.descriptors import cached from synapse.util.caches.dictionary_cache import DictionaryCache @@ -90,11 +89,10 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): self._state_group_cache = DictionaryCache( "*stateGroupCache*", # TODO: this hasn't been tuned yet - 50000 * get_cache_factor_for("stateGroupCache"), + 50000, ) self._state_group_members_cache = DictionaryCache( - "*stateGroupMembersCache*", - 500000 * get_cache_factor_for("stateGroupMembersCache"), + "*stateGroupMembersCache*", 500000, ) @cached(max_entries=10000, iterable=True) diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index c4cf38abc7..8c24965bd8 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -377,7 +377,10 @@ class CacheDescriptor(_CacheDescriptorBase): self.tree = tree self.iterable = iterable - def __get__(self, obj, objtype=None): + def __get__(self, obj, owner): + if isinstance(self.max_entries, str): + self.max_entries = getattr(obj.hs.config.caches, self.max_entries) + cache = Cache( name=self.orig.__name__, max_entries=self.max_entries, diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index fd52512696..62a4b31fb8 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -43,7 +43,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): ) hs.config.app_service_config_files = self.as_yaml_files - hs.config.event_cache_size = 1 + hs.config.caches.event_cache_size = 1 hs.config.password_providers = [] self.as_token = "token1" @@ -110,7 +110,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): ) hs.config.app_service_config_files = self.as_yaml_files - hs.config.event_cache_size = 1 + hs.config.caches.event_cache_size = 1 hs.config.password_providers = [] self.as_list = [ @@ -422,7 +422,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): ) hs.config.app_service_config_files = [f1, f2] - hs.config.event_cache_size = 1 + hs.config.caches.event_cache_size = 1 hs.config.password_providers = [] database = hs.get_datastores().databases[0] @@ -440,7 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): ) hs.config.app_service_config_files = [f1, f2] - hs.config.event_cache_size = 1 + hs.config.caches.event_cache_size = 1 hs.config.password_providers = [] with self.assertRaises(ConfigError) as cm: @@ -464,7 +464,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): ) hs.config.app_service_config_files = [f1, f2] - hs.config.event_cache_size = 1 + hs.config.caches.event_cache_size = 1 hs.config.password_providers = [] with self.assertRaises(ConfigError) as cm: diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index cdee0a9e60..278961c331 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -51,7 +51,8 @@ class SQLBaseStoreTestCase(unittest.TestCase): config = Mock() config._disable_native_upserts = True - config.event_cache_size = 1 + config.caches = Mock() + config.caches.event_cache_size = 1 hs = TestHomeServer("test", config=config) sqlite_config = {"name": "sqlite3"} diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index 1cee7ba91a..0adb2174af 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -84,7 +84,7 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEquals(len(cache), 0) -class LruCacheCallbacksTestCase(unittest.TestCase): +class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_get(self): m = Mock() cache = LruCache(1) @@ -233,7 +233,7 @@ class LruCacheCallbacksTestCase(unittest.TestCase): self.assertEquals(m3.call_count, 1) -class LruCacheSizedTestCase(unittest.TestCase): +class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_evict(self): cache = LruCache(5, size_callback=len) cache["key1"] = [0] -- cgit 1.5.1