diff options
Diffstat (limited to 'synapse/config')
| -rw-r--r-- | synapse/config/_base.py | 81 | ||||
| -rw-r--r-- | synapse/config/_base.pyi | 15 | ||||
| -rw-r--r-- | synapse/config/cache.py | 82 | ||||
| -rw-r--r-- | synapse/config/room.py | 47 | ||||
| -rw-r--r-- | synapse/config/server.py | 2 |
5 files changed, 200 insertions, 27 deletions
diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 179aa7ff88..42364fc133 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -16,14 +16,18 @@ import argparse import errno +import logging import os from collections import OrderedDict from hashlib import sha256 from textwrap import dedent from typing import ( Any, + ClassVar, + Collection, Dict, Iterable, + Iterator, List, MutableMapping, Optional, @@ -40,6 +44,8 @@ import yaml from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter +logger = logging.getLogger(__name__) + class ConfigError(Exception): """Represents a problem parsing the configuration @@ -55,6 +61,38 @@ class ConfigError(Exception): self.path = path +def format_config_error(e: ConfigError) -> Iterator[str]: + """ + Formats a config error neatly + + The idea is to format the immediate error, plus the "causes" of those errors, + hopefully in a way that makes sense to the user. For example: + + Error in configuration at 'oidc_config.user_mapping_provider.config.display_name_template': + Failed to parse config for module 'JinjaOidcMappingProvider': + invalid jinja template: + unexpected end of template, expected 'end of print statement'. + + Args: + e: the error to be formatted + + Returns: An iterator which yields string fragments to be formatted + """ + yield "Error in configuration" + + if e.path: + yield " at '%s'" % (".".join(e.path),) + + yield ":\n %s" % (e.msg,) + + parent_e = e.__cause__ + indent = 1 + while parent_e: + indent += 1 + yield ":\n%s%s" % (" " * indent, str(parent_e)) + parent_e = parent_e.__cause__ + + # We split these messages out to allow packages to override with package # specific instructions. MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ @@ -119,7 +157,7 @@ class Config: defined in subclasses. """ - section: str + section: ClassVar[str] def __init__(self, root_config: "RootConfig" = None): self.root = root_config @@ -309,9 +347,12 @@ class RootConfig: class, lower-cased and with "Config" removed. """ - config_classes = [] + config_classes: List[Type[Config]] = [] + + def __init__(self, config_files: Collection[str] = ()): + # Capture absolute paths here, so we can reload config after we daemonize. + self.config_files = [os.path.abspath(path) for path in config_files] - def __init__(self): for config_class in self.config_classes: if config_class.section is None: raise ValueError("%r requires a section name" % (config_class,)) @@ -512,12 +553,10 @@ class RootConfig: object from parser.parse_args(..)` """ - obj = cls() - config_args = parser.parse_args(argv) config_files = find_config_files(search_paths=config_args.config_path) - + obj = cls(config_files) if not config_files: parser.error("Must supply a config file.") @@ -627,7 +666,7 @@ class RootConfig: generate_missing_configs = config_args.generate_missing_configs - obj = cls() + obj = cls(config_files) if config_args.generate_config: if config_args.report_stats is None: @@ -727,6 +766,34 @@ class RootConfig: ) -> None: self.invoke_all("generate_files", config_dict, config_dir_path) + def reload_config_section(self, section_name: str) -> Config: + """Reconstruct the given config section, leaving all others unchanged. + + This works in three steps: + + 1. Create a new instance of the relevant `Config` subclass. + 2. Call `read_config` on that instance to parse the new config. + 3. Replace the existing config instance with the new one. + + :raises ValueError: if the given `section` does not exist. + :raises ConfigError: for any other problems reloading config. + + :returns: the previous config object, which no longer has a reference to this + RootConfig. + """ + existing_config: Optional[Config] = getattr(self, section_name, None) + if existing_config is None: + raise ValueError(f"Unknown config section '{section_name}'") + logger.info("Reloading config section '%s'", section_name) + + new_config_data = read_config_files(self.config_files) + new_config = type(existing_config)(self) + new_config.read_config(new_config_data) + setattr(self, section_name, new_config) + + existing_config.root = None + return existing_config + def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]: """Read the config files into a dict diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index bd092f956d..71d6655fda 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -1,15 +1,19 @@ import argparse from typing import ( Any, + Collection, Dict, Iterable, + Iterator, List, + Literal, MutableMapping, Optional, Tuple, Type, TypeVar, Union, + overload, ) import jinja2 @@ -64,6 +68,8 @@ class ConfigError(Exception): self.msg = msg self.path = path +def format_config_error(e: ConfigError) -> Iterator[str]: ... + MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS: str MISSING_REPORT_STATS_SPIEL: str MISSING_SERVER_NAME: str @@ -117,7 +123,8 @@ class RootConfig: background_updates: background_updates.BackgroundUpdateConfig config_classes: List[Type["Config"]] = ... - def __init__(self) -> None: ... + config_files: List[str] + def __init__(self, config_files: Collection[str] = ...) -> None: ... def invoke_all( self, func_name: str, *args: Any, **kwargs: Any ) -> MutableMapping[str, Any]: ... @@ -157,6 +164,12 @@ class RootConfig: def generate_missing_files( self, config_dict: dict, config_dir_path: str ) -> None: ... + @overload + def reload_config_section( + self, section_name: Literal["caches"] + ) -> cache.CacheConfig: ... + @overload + def reload_config_section(self, section_name: str) -> Config: ... class Config: root: RootConfig diff --git a/synapse/config/cache.py b/synapse/config/cache.py index 94d852f413..d2f55534d7 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -69,11 +69,11 @@ def _canonicalise_cache_name(cache_name: str) -> str: def add_resizable_cache( cache_name: str, cache_resize_callback: Callable[[float], None] ) -> None: - """Register a cache that's size can dynamically change + """Register a cache whose size can dynamically change Args: cache_name: A reference to the cache - cache_resize_callback: A callback function that will be ran whenever + cache_resize_callback: A callback function that will run whenever the cache needs to be resized """ # Some caches have '*' in them which we strip out. @@ -96,6 +96,13 @@ class CacheConfig(Config): section = "caches" _environ = os.environ + event_cache_size: int + cache_factors: Dict[str, float] + global_factor: float + track_memory_usage: bool + expiry_time_msec: Optional[int] + sync_response_cache_duration: int + @staticmethod def reset() -> None: """Resets the caches to their defaults. Used for tests.""" @@ -115,6 +122,12 @@ class CacheConfig(Config): # A cache 'factor' is a multiplier that can be applied to each of # Synapse's caches in order to increase or decrease the maximum # number of entries that can be stored. + # + # The configuration for cache factors (caches.global_factor and + # caches.per_cache_factors) can be reloaded while the application is running, + # by sending a SIGHUP signal to the Synapse process. Changes to other parts of + # the caching config will NOT be applied after a SIGHUP is received; a restart + # is necessary. # The number of events to cache in memory. Not affected by # caches.global_factor. @@ -163,6 +176,24 @@ class CacheConfig(Config): # #cache_entry_ttl: 30m + # This flag enables cache autotuning, and is further specified by the sub-options `max_cache_memory_usage`, + # `target_cache_memory_usage`, `min_cache_ttl`. These flags work in conjunction with each other to maintain + # a balance between cache memory usage and cache entry availability. You must be using jemalloc to utilize + # this option, and all three of the options must be specified for this feature to work. + #cache_autotuning: + # This flag sets a ceiling on much memory the cache can use before caches begin to be continuously evicted. + # They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in + # the flag below, or until the `min_cache_ttl` is hit. + #max_cache_memory_usage: 1024M + + # This flag sets a rough target for the desired memory usage of the caches. + #target_cache_memory_usage: 758M + + # 'min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when + # caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches + # from being emptied while Synapse is evicting due to memory. + #min_cache_ttl: 5m + # Controls how long the results of a /sync request are cached for after # a successful response is returned. A higher duration can help clients with # intermittent connections, at the cost of higher memory usage. @@ -174,21 +205,21 @@ class CacheConfig(Config): """ def read_config(self, config: JsonDict, **kwargs: Any) -> None: + """Populate this config object with values from `config`. + + This method does NOT resize existing or future caches: use `resize_all_caches`. + We use two separate methods so that we can reject bad config before applying it. + """ self.event_cache_size = self.parse_size( config.get("event_cache_size", _DEFAULT_EVENT_CACHE_SIZE) ) - self.cache_factors: Dict[str, float] = {} + self.cache_factors = {} cache_config = config.get("caches") or {} - self.global_factor = cache_config.get( - "global_factor", properties.default_factor_size - ) + self.global_factor = cache_config.get("global_factor", _DEFAULT_FACTOR_SIZE) if not isinstance(self.global_factor, (int, float)): raise ConfigError("caches.global_factor must be a number.") - # Set the global one so that it's reflected in new caches - properties.default_factor_size = self.global_factor - # Load cache factors from the config individual_factors = cache_config.get("per_cache_factors") or {} if not isinstance(individual_factors, dict): @@ -230,7 +261,7 @@ class CacheConfig(Config): cache_entry_ttl = cache_config.get("cache_entry_ttl", "30m") if expire_caches: - self.expiry_time_msec: Optional[int] = self.parse_duration(cache_entry_ttl) + self.expiry_time_msec = self.parse_duration(cache_entry_ttl) else: self.expiry_time_msec = None @@ -250,23 +281,38 @@ class CacheConfig(Config): ) self.expiry_time_msec = self.parse_duration(expiry_time) + self.cache_autotuning = cache_config.get("cache_autotuning") + if self.cache_autotuning: + max_memory_usage = self.cache_autotuning.get("max_cache_memory_usage") + self.cache_autotuning["max_cache_memory_usage"] = self.parse_size( + max_memory_usage + ) + + target_mem_size = self.cache_autotuning.get("target_cache_memory_usage") + self.cache_autotuning["target_cache_memory_usage"] = self.parse_size( + target_mem_size + ) + + min_cache_ttl = self.cache_autotuning.get("min_cache_ttl") + self.cache_autotuning["min_cache_ttl"] = self.parse_duration(min_cache_ttl) + self.sync_response_cache_duration = self.parse_duration( cache_config.get("sync_response_cache_duration", 0) ) - # Resize all caches (if necessary) with the new factors we've loaded - self.resize_all_caches() - - # Store this function so that it can be called from other classes without - # needing an instance of Config - properties.resize_all_caches_func = self.resize_all_caches - def resize_all_caches(self) -> None: - """Ensure all cache sizes are up to date + """Ensure all cache sizes are up-to-date. For each cache, run the mapped callback function with either a specific cache factor or the default, global one. """ + # Set the global factor size, so that new caches are appropriately sized. + properties.default_factor_size = self.global_factor + + # Store this function so that it can be called from other classes without + # needing an instance of CacheConfig + properties.resize_all_caches_func = self.resize_all_caches + # block other threads from modifying _CACHES while we iterate it. with _CACHES_LOCK: for cache_name, callback in _CACHES.items(): diff --git a/synapse/config/room.py b/synapse/config/room.py index e18a87ea37..462d85ac1d 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -63,6 +63,19 @@ class RoomConfig(Config): "Invalid value for encryption_enabled_by_default_for_room_type" ) + self.default_power_level_content_override = config.get( + "default_power_level_content_override", + None, + ) + if self.default_power_level_content_override is not None: + for preset in self.default_power_level_content_override: + if preset not in vars(RoomCreationPreset).values(): + raise ConfigError( + "Unrecognised room preset %s in default_power_level_content_override" + % preset + ) + # We validate the actual overrides when we try to apply them. + def generate_config_section(self, **kwargs: Any) -> str: return """\ ## Rooms ## @@ -83,4 +96,38 @@ class RoomConfig(Config): # will also not affect rooms created by other servers. # #encryption_enabled_by_default_for_room_type: invite + + # Override the default power levels for rooms created on this server, per + # room creation preset. + # + # The appropriate dictionary for the room preset will be applied on top + # of the existing power levels content. + # + # Useful if you know that your users need special permissions in rooms + # that they create (e.g. to send particular types of state events without + # needing an elevated power level). This takes the same shape as the + # `power_level_content_override` parameter in the /createRoom API, but + # is applied before that parameter. + # + # Valid keys are some or all of `private_chat`, `trusted_private_chat` + # and `public_chat`. Inside each of those should be any of the + # properties allowed in `power_level_content_override` in the + # /createRoom API. If any property is missing, its default value will + # continue to be used. If any property is present, it will overwrite + # the existing default completely (so if the `events` property exists, + # the default event power levels will be ignored). + # + #default_power_level_content_override: + # private_chat: + # "events": + # "com.example.myeventtype" : 0 + # "m.room.avatar": 50 + # "m.room.canonical_alias": 50 + # "m.room.encryption": 100 + # "m.room.history_visibility": 100 + # "m.room.name": 50 + # "m.room.power_levels": 100 + # "m.room.server_acl": 100 + # "m.room.tombstone": 100 + # "events_default": 1 """ diff --git a/synapse/config/server.py b/synapse/config/server.py index 005a3ee48c..f73d5e1f66 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -996,7 +996,7 @@ class ServerConfig(Config): # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # - # keys: the key discovery API (/_matrix/keys). + # keys: the key discovery API (/_matrix/key). # # media: the media API (/_matrix/media). # |