From 495cb100d127212d55a46c177706d732950e70be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 6 Aug 2018 14:46:17 +0100 Subject: Allow profile changes to happen on workers --- synapse/handlers/profile.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'synapse/handlers/profile.py') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index cb5c6d587e..a3bdb1830f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import AuthError, CodeMessageException, SynapseError from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.replication.http.profile import ReplicationHandleProfileChangeRestServlet from synapse.types import UserID, get_domain_from_id from ._base import BaseHandler @@ -45,6 +46,10 @@ class ProfileHandler(BaseHandler): self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS, ) + self._notify_master_profile_change = ( + ReplicationHandleProfileChangeRestServlet.make_client(hs) + ) + @defer.inlineCallbacks def get_profile(self, user_id): target_user = UserID.from_string(user_id) @@ -147,10 +152,16 @@ class ProfileHandler(BaseHandler): ) if self.hs.config.user_directory_search_all_users: - profile = yield self.store.get_profileinfo(target_user.localpart) - yield self.user_directory_handler.handle_local_profile_change( - target_user.to_string(), profile - ) + if self.hs.config.worker_app is None: + profile = yield self.store.get_profileinfo(target_user.localpart) + yield self.user_directory_handler.handle_local_profile_change( + target_user.to_string(), profile + ) + else: + yield self._notify_master_profile_change( + requester=requester, + user_id=target_user.to_string(), + ) yield self._update_join_states(requester, target_user) @@ -196,11 +207,16 @@ class ProfileHandler(BaseHandler): target_user.localpart, new_avatar_url ) - if self.hs.config.user_directory_search_all_users: + if self.hs.config.worker_app is None: profile = yield self.store.get_profileinfo(target_user.localpart) yield self.user_directory_handler.handle_local_profile_change( target_user.to_string(), profile ) + else: + yield self._notify_master_profile_change( + requester=requester, + user_id=target_user.to_string(), + ) yield self._update_join_states(requester, target_user) -- cgit 1.5.1 From ca87ad1defac1082462367854cb4a656b7a96e90 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Aug 2018 11:43:16 +0100 Subject: Split ProfileHandler into master and worker --- synapse/handlers/profile.py | 21 ++++++++++++++------- synapse/server.py | 7 +++++-- tests/handlers/test_profile.py | 4 ++-- 3 files changed, 21 insertions(+), 11 deletions(-) (limited to 'synapse/handlers/profile.py') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 6d1fbb1a5c..8b349f6ad6 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -33,12 +33,12 @@ from ._base import BaseHandler logger = logging.getLogger(__name__) -class ProfileHandler(BaseHandler): +class WorkerProfileHandler(BaseHandler): PROFILE_UPDATE_MS = 60 * 1000 PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000 def __init__(self, hs): - super(ProfileHandler, self).__init__(hs) + super(WorkerProfileHandler, self).__init__(hs) self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( @@ -47,11 +47,6 @@ class ProfileHandler(BaseHandler): self.user_directory_handler = hs.get_user_directory_handler() - if hs.config.worker_app is None: - self.clock.looping_call( - self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS, - ) - self._notify_master_profile_change = ( ReplicationHandleProfileChangeRestServlet.make_client(hs) ) @@ -298,6 +293,18 @@ class ProfileHandler(BaseHandler): room_id, str(e.message) ) + +class MasterProfileHandler(WorkerProfileHandler): + PROFILE_UPDATE_MS = 60 * 1000 + PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000 + + def __init__(self, hs): + super(MasterProfileHandler, self).__init__(hs) + + self.clock.looping_call( + self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS, + ) + def _start_update_remote_profile_cache(self): return run_as_background_process( "Update remote profile", self._update_remote_profile_cache, diff --git a/synapse/server.py b/synapse/server.py index 140be9ebe8..be85aad8cf 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -55,7 +55,7 @@ from synapse.handlers.initial_sync import InitialSyncHandler from synapse.handlers.message import EventCreationHandler, MessageHandler from synapse.handlers.pagination import PaginationHandler from synapse.handlers.presence import PresenceHandler -from synapse.handlers.profile import ProfileHandler +from synapse.handlers.profile import MasterProfileHandler, WorkerProfileHandler from synapse.handlers.read_marker import ReadMarkerHandler from synapse.handlers.receipts import ReceiptsHandler from synapse.handlers.room import RoomContextHandler, RoomCreationHandler @@ -307,7 +307,10 @@ class HomeServer(object): return InitialSyncHandler(self) def build_profile_handler(self): - return ProfileHandler(self) + if self.config.worker_app: + return WorkerProfileHandler(self) + else: + return MasterProfileHandler(self) def build_event_creation_handler(self): return EventCreationHandler(self) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index dc17918a3d..07cf5f4c8e 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -20,7 +20,7 @@ from twisted.internet import defer import synapse.types from synapse.api.errors import AuthError -from synapse.handlers.profile import ProfileHandler +from synapse.handlers.profile import MasterProfileHandler from synapse.types import UserID from tests import unittest @@ -29,7 +29,7 @@ from tests.utils import setup_test_homeserver class ProfileHandlers(object): def __init__(self, hs): - self.profile_handler = ProfileHandler(hs) + self.profile_handler = MasterProfileHandler(hs) class ProfileTestCase(unittest.TestCase): -- cgit 1.5.1 From 91cdb6de08aa20f2bc7f8df906eb5b56df387309 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Aug 2018 15:24:16 +0100 Subject: Call UserDirectoryHandler methods directly Turns out that the user directory handling is fairly racey as a bunch of stuff assumes that the processing happens on master, which it doesn't when there is a synapse.app.user_dir worker. So lets just call the function directly until we actually get round to fixing it, since it doesn't make the situation any worse. --- synapse/app/event_creator.py | 2 + synapse/handlers/profile.py | 26 +++--------- synapse/replication/http/__init__.py | 3 +- synapse/replication/http/profile.py | 81 ------------------------------------ 4 files changed, 8 insertions(+), 104 deletions(-) delete mode 100644 synapse/replication/http/profile.py (limited to 'synapse/handlers/profile.py') diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 2c9a735859..a2bdaf2a2c 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -58,6 +58,7 @@ from synapse.rest.client.v1.room import ( ) from synapse.server import HomeServer from synapse.storage.engines import create_engine +from synapse.storage.user_directory import UserDirectoryStore from synapse.util.httpresourcetree import create_resource_tree from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole @@ -67,6 +68,7 @@ logger = logging.getLogger("synapse.app.event_creator") class EventCreatorSlavedStore( + UserDirectoryStore, DirectoryStore, SlavedTransactionStore, SlavedProfileStore, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 8b349f6ad6..3e1d95d5e5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -25,7 +25,6 @@ from synapse.api.errors import ( SynapseError, ) from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.replication.http.profile import ReplicationHandleProfileChangeRestServlet from synapse.types import UserID, get_domain_from_id from ._base import BaseHandler @@ -47,10 +46,6 @@ class WorkerProfileHandler(BaseHandler): self.user_directory_handler = hs.get_user_directory_handler() - self._notify_master_profile_change = ( - ReplicationHandleProfileChangeRestServlet.make_client(hs) - ) - @defer.inlineCallbacks def get_profile(self, user_id): target_user = UserID.from_string(user_id) @@ -166,16 +161,10 @@ class WorkerProfileHandler(BaseHandler): ) if self.hs.config.user_directory_search_all_users: - if self.hs.config.worker_app is None: - profile = yield self.store.get_profileinfo(target_user.localpart) - yield self.user_directory_handler.handle_local_profile_change( - target_user.to_string(), profile - ) - else: - yield self._notify_master_profile_change( - requester=requester, - user_id=target_user.to_string(), - ) + profile = yield self.store.get_profileinfo(target_user.localpart) + yield self.user_directory_handler.handle_local_profile_change( + target_user.to_string(), profile + ) yield self._update_join_states(requester, target_user) @@ -225,16 +214,11 @@ class WorkerProfileHandler(BaseHandler): target_user.localpart, new_avatar_url ) - if self.hs.config.worker_app is None: + if self.hs.config.user_directory_search_all_users: profile = yield self.store.get_profileinfo(target_user.localpart) yield self.user_directory_handler.handle_local_profile_change( target_user.to_string(), profile ) - else: - yield self._notify_master_profile_change( - requester=requester, - user_id=target_user.to_string(), - ) yield self._update_join_states(requester, target_user) diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index d8487df614..19f214281e 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, profile, send_event +from synapse.replication.http import federation, membership, send_event REPLICATION_PREFIX = "/_synapse/replication" @@ -27,5 +27,4 @@ class ReplicationRestResource(JsonResource): def register_servlets(self, hs): send_event.register_servlets(hs, self) membership.register_servlets(hs, self) - profile.register_servlets(hs, self) federation.register_servlets(hs, self) diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py deleted file mode 100644 index aa700c2434..0000000000 --- a/synapse/replication/http/profile.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from twisted.internet import defer - -from synapse.http.servlet import parse_json_object_from_request -from synapse.replication.http._base import ReplicationEndpoint -from synapse.types import Requester, UserID - -logger = logging.getLogger(__name__) - - -class ReplicationHandleProfileChangeRestServlet(ReplicationEndpoint): - """Notifies that a users profile has changed - - Request format: - - POST /_synapse/replication/profile_changed/:user_id - - { - "requester": ... - } - """ - - NAME = "profile_changed" - PATH_ARGS = ("user_id",) - - def __init__(self, hs): - super(ReplicationHandleProfileChangeRestServlet, self).__init__(hs) - - self.user_directory_handler = hs.get_user_directory_handler() - self.store = hs.get_datastore() - self.clock = hs.get_clock() - - @staticmethod - def _serialize_payload(requester, user_id): - """ - Args: - requester (Requester) - user_id (str) - """ - - return { - "requester": requester.serialize(), - } - - @defer.inlineCallbacks - def _handle_request(self, request, user_id): - content = parse_json_object_from_request(request) - - requester = Requester.deserialize(self.store, content["requester"]) - - if requester.user: - request.authenticated_entity = requester.user.to_string() - - target_user = UserID.from_string(user_id) - - profile = yield self.store.get_profileinfo(target_user.localpart) - yield self.user_directory_handler.handle_local_profile_change( - user_id, profile - ) - - defer.returnValue((200, {})) - - -def register_servlets(hs, http_server): - ReplicationHandleProfileChangeRestServlet(hs).register(http_server) -- cgit 1.5.1 From 47b25ba5f3040d07f8be80584b292ca0386fdb30 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Aug 2018 10:09:05 +0100 Subject: Remove redundant vars --- synapse/handlers/profile.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'synapse/handlers/profile.py') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 3e1d95d5e5..ec9cb1f50c 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -33,9 +33,6 @@ logger = logging.getLogger(__name__) class WorkerProfileHandler(BaseHandler): - PROFILE_UPDATE_MS = 60 * 1000 - PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000 - def __init__(self, hs): super(WorkerProfileHandler, self).__init__(hs) -- cgit 1.5.1 From a81f1408807683d212b7bf3c4f5f36df4b0e0d33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Aug 2018 10:11:21 +0100 Subject: Add assert to ensure handler is only run on master --- synapse/handlers/profile.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse/handlers/profile.py') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index ec9cb1f50c..278f131d5e 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -282,6 +282,8 @@ class MasterProfileHandler(WorkerProfileHandler): def __init__(self, hs): super(MasterProfileHandler, self).__init__(hs) + assert hs.config.worker_app is None + self.clock.looping_call( self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS, ) -- cgit 1.5.1 From 8432e2ebd76d0462d39a329967671e845da7e404 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Aug 2018 10:13:40 +0100 Subject: Rename WorkerProfileHandler to BaseProfileHandler --- synapse/handlers/profile.py | 13 ++++++++++--- synapse/server.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) (limited to 'synapse/handlers/profile.py') diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 278f131d5e..75b8b7ce6a 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -32,9 +32,16 @@ from ._base import BaseHandler logger = logging.getLogger(__name__) -class WorkerProfileHandler(BaseHandler): +class BaseProfileHandler(BaseHandler): + """Handles fetching and updating user profile information. + + BaseProfileHandler can be instantiated directly on workers and will + delegate to master when necessary. The master process should use the + subclass MasterProfileHandler + """ + def __init__(self, hs): - super(WorkerProfileHandler, self).__init__(hs) + super(BaseProfileHandler, self).__init__(hs) self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( @@ -275,7 +282,7 @@ class WorkerProfileHandler(BaseHandler): ) -class MasterProfileHandler(WorkerProfileHandler): +class MasterProfileHandler(BaseProfileHandler): PROFILE_UPDATE_MS = 60 * 1000 PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000 diff --git a/synapse/server.py b/synapse/server.py index fcfa7e6557..a6fbc6ec0c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -56,7 +56,7 @@ from synapse.handlers.initial_sync import InitialSyncHandler from synapse.handlers.message import EventCreationHandler, MessageHandler from synapse.handlers.pagination import PaginationHandler from synapse.handlers.presence import PresenceHandler -from synapse.handlers.profile import MasterProfileHandler, WorkerProfileHandler +from synapse.handlers.profile import BaseProfileHandler, MasterProfileHandler from synapse.handlers.read_marker import ReadMarkerHandler from synapse.handlers.receipts import ReceiptsHandler from synapse.handlers.room import RoomContextHandler, RoomCreationHandler @@ -309,7 +309,7 @@ class HomeServer(object): def build_profile_handler(self): if self.config.worker_app: - return WorkerProfileHandler(self) + return BaseProfileHandler(self) else: return MasterProfileHandler(self) -- cgit 1.5.1