From e1fd4751de8e96907ea97afaf91525e68ce22227 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Nov 2017 11:08:08 +0000 Subject: Build MediaRepositoryResource as a homeserver dependency This avoids the scenario where we have four different PreviewUrlResources configured on a single app, each of which have their own caches and cache clearing jobs. --- synapse/app/homeserver.py | 3 +-- synapse/app/media_repository.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 9e26146338..4b6164baa2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -43,7 +43,6 @@ from synapse.rest import ClientRestResource from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource -from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.server import HomeServer from synapse.storage import are_all_users_on_domain from synapse.storage.engines import IncorrectDatabaseSetup, create_engine @@ -195,7 +194,7 @@ class SynapseHomeServer(HomeServer): }) if name in ["media", "federation", "client"]: - media_repo = MediaRepositoryResource(self) + media_repo = self.get_media_repository_resource() resources.update({ MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 36c18bdbcb..f54beeb15d 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -35,7 +35,6 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto from synapse.replication.slave.storage.transactions import TransactionStore from synapse.replication.tcp.client import ReplicationClientHandler from synapse.rest.media.v0.content_repository import ContentRepoResource -from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.server import HomeServer from synapse.storage.engines import create_engine from synapse.storage.media_repository import MediaRepositoryStore @@ -89,7 +88,7 @@ class MediaRepositoryServer(HomeServer): if name == "metrics": resources[METRICS_PREFIX] = MetricsResource(self) elif name == "media": - media_repo = MediaRepositoryResource(self) + media_repo = self.get_media_repository_resource() resources.update({ MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo, -- cgit 1.4.1 From 68ca8641419ee42606192787b92152353f5c112e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Nov 2017 13:29:39 +0000 Subject: Add config option to disable media_repo on main synapse ... to stop us doing the cache cleanup jobs on the master. --- docs/workers.rst | 5 ++++- synapse/app/homeserver.py | 21 +++++++++++++-------- synapse/app/media_repository.py | 7 +++++++ synapse/config/server.py | 6 ++++++ 4 files changed, 30 insertions(+), 9 deletions(-) (limited to 'synapse/app') diff --git a/docs/workers.rst b/docs/workers.rst index 3cc8b3d82e..b39f79058e 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -170,6 +170,10 @@ Handles the media repository. It can handle all endpoints starting with:: /_matrix/media/ +You should also set ``enable_media_repo: False`` in the shared configuration +file to stop the main synapse running background jobs related to managing the +media repository. + Note this worker cannot be load-balanced: only one instance should be active. ``synapse.app.client_reader`` @@ -203,4 +207,3 @@ the ``worker_main_http_uri`` setting in the frontend_proxy worker configuration file. For example:: worker_main_http_uri: http://127.0.0.1:8008 - diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 4b6164baa2..6b8875afb4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -194,14 +194,19 @@ class SynapseHomeServer(HomeServer): }) if name in ["media", "federation", "client"]: - media_repo = self.get_media_repository_resource() - resources.update({ - MEDIA_PREFIX: media_repo, - LEGACY_MEDIA_PREFIX: media_repo, - CONTENT_REPO_PREFIX: ContentRepoResource( - self, self.config.uploads_path - ), - }) + if self.get_config().enable_media_repo: + media_repo = self.get_media_repository_resource() + resources.update({ + MEDIA_PREFIX: media_repo, + LEGACY_MEDIA_PREFIX: media_repo, + CONTENT_REPO_PREFIX: ContentRepoResource( + self, self.config.uploads_path + ), + }) + elif name == "media": + raise ConfigError( + "'media' resource conflicts with enable_media_repo=False", + ) if name in ["keys", "federation"]: resources.update({ diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index f54beeb15d..c4e5f0965d 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -150,6 +150,13 @@ def start(config_options): assert config.worker_app == "synapse.app.media_repository" + if config.enable_media_repo: + _base.quit_with_error( + "enable_media_repo must be disabled in the main synapse process\n" + "before the media repo can be run in a separate worker.\n" + "Please add ``enable_media_repo: false`` to the main config\n" + ) + setup_logging(config, use_worker_options=True) events.USE_FROZEN_DICTS = config.use_frozen_dicts diff --git a/synapse/config/server.py b/synapse/config/server.py index 4d9193536d..edb90a1348 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -41,6 +41,12 @@ class ServerConfig(Config): # false only if we are updating the user directory in a worker self.update_user_directory = config.get("update_user_directory", True) + # whether to enable the media repository endpoints. This should be set + # to false if the media repository is running as a separate endpoint; + # doing so ensures that we will not run cache cleanup jobs on the + # master, potentially causing inconsistency. + self.enable_media_repo = config.get("enable_media_repo", True) + self.filter_timeline_limit = config.get("filter_timeline_limit", -1) # Whether we should block invites sent to users on this server -- cgit 1.4.1 From 7f14f0ae3861bf9596358614133ed3bb7b8cefc4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 24 Nov 2017 00:32:04 +0000 Subject: Remove dead sync_callback This is never used; let's remove it to stop confusing things. --- synapse/app/synchrotron.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse/app') diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 576ac6fb7e..f68f45ea4f 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -343,8 +343,6 @@ class SyncReplicationHandler(ReplicationClientHandler): self.presence_handler = hs.get_presence_handler() self.notifier = hs.get_notifier() - self.presence_handler.sync_callback = self.send_user_sync - def on_rdata(self, stream_name, token, rows): super(SyncReplicationHandler, self).on_rdata(stream_name, token, rows) -- cgit 1.4.1 From 795b0849f3654d292132e9ead691d12158bd0304 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 24 Nov 2017 00:34:56 +0000 Subject: Add a comment which might save some confusion --- synapse/app/synchrotron.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/app') diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index f68f45ea4f..323fddee21 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -340,6 +340,7 @@ class SyncReplicationHandler(ReplicationClientHandler): self.store = hs.get_datastore() self.typing_handler = hs.get_typing_handler() + # NB this is a SynchrotronPresence, not a normal PresenceHandler self.presence_handler = hs.get_presence_handler() self.notifier = hs.get_notifier() -- cgit 1.4.1