From f5bafd70f4d641b24303958b650efcf7dc18ab92 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:34:23 +1100 Subject: add cache remover endpoint and wire it up --- synapse/replication/http/__init__.py | 3 ++- synapse/rest/client/v2_alpha/register.py | 11 +++++++++++ synapse/storage/registration.py | 11 +++++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 19f214281e..f46a580be2 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, send_event +from synapse.replication.http import federation, membership, send_event, registration REPLICATION_PREFIX = "/_synapse/replication" @@ -28,3 +28,4 @@ class ReplicationRestResource(JsonResource): send_event.register_servlets(hs, self) membership.register_servlets(hs, self) federation.register_servlets(hs, self) + registration.register_servlets(hs, self) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 7f812b8209..35319f7a79 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -24,6 +24,7 @@ from twisted.internet import defer import synapse import synapse.types +from synapse.replication.http.registration import RegistrationUserCacheInvalidationServlet from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.config.server import is_threepid_reserved @@ -193,6 +194,10 @@ class RegisterRestServlet(RestServlet): self.device_handler = hs.get_device_handler() self.macaroon_gen = hs.get_macaroon_generator() + self._invalidate_caches_client = ( + RegistrationUserCacheInvalidationServlet.make_client(hs) + ) + @interactive_auth_handler @defer.inlineCallbacks def on_POST(self, request): @@ -266,6 +271,9 @@ class RegisterRestServlet(RestServlet): # == Shared Secret Registration == (e.g. create new user scripts) if 'mac' in body: + if self.hs.config.worker_app: + raise SynapseError(403, "Not available at this endpoint") + # FIXME: Should we really be determining if this is shared secret # auth based purely on the 'mac' key? result = yield self._do_shared_secret_registration( @@ -456,6 +464,9 @@ class RegisterRestServlet(RestServlet): ) yield self.registration_handler.post_consent_actions(registered_user_id) + if self.hs.config.worker_app: + self._invalidate_caches_client(registered_user_id) + defer.returnValue((200, return_dict)) def on_OPTIONS(self, _): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c9e11c3135..8b4554f6af 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -146,6 +146,7 @@ class RegistrationStore(RegistrationWorkerStore, def __init__(self, db_conn, hs): super(RegistrationStore, self).__init__(db_conn, hs) + self.hs = hs self.clock = hs.get_clock() self.register_background_index_update( @@ -321,10 +322,12 @@ class RegistrationStore(RegistrationWorkerStore, (user_id_obj.localpart, create_profile_with_displayname) ) - self._invalidate_cache_and_stream( - txn, self.get_user_by_id, (user_id,) - ) - txn.call_after(self.is_guest.invalidate, (user_id,)) + # Don't invalidate here, it will be done through replication to the worker. + if not self.hs.config.worker_app: + self._invalidate_cache_and_stream( + txn, self.get_user_by_id, (user_id,) + ) + txn.call_after(self.is_guest.invalidate, (user_id,)) def get_users_by_id_case_insensitive(self, user_id): """Gets users that match user_id case insensitively. -- cgit 1.5.1 From 7b5c04312efd0829855a45b19839971c55f81599 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:35:27 +1100 Subject: isort --- synapse/handlers/user_directory.py | 1 - synapse/replication/http/__init__.py | 2 +- synapse/rest/client/v2_alpha/register.py | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index f5c3ba23a6..0dacd9e357 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -14,7 +14,6 @@ # limitations under the License. import logging -import synapse.metrics from six import iteritems diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index f46a580be2..3807d2ac6f 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, send_event, registration +from synapse.replication.http import federation, membership, registration, send_event REPLICATION_PREFIX = "/_synapse/replication" diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 35319f7a79..ca9a850817 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -24,7 +24,6 @@ from twisted.internet import defer import synapse import synapse.types -from synapse.replication.http.registration import RegistrationUserCacheInvalidationServlet from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.config.server import is_threepid_reserved @@ -34,6 +33,9 @@ from synapse.http.servlet import ( parse_json_object_from_request, parse_string, ) +from synapse.replication.http.registration import ( + RegistrationUserCacheInvalidationServlet, +) from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.threepids import check_3pid_allowed -- cgit 1.5.1 From ce5f3b1ba5cefe5d453bd82c06faa8eaa336137f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:35:58 +1100 Subject: add all the files --- synapse/replication/http/registration.py | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 synapse/replication/http/registration.py diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py new file mode 100644 index 0000000000..5030bde499 --- /dev/null +++ b/synapse/replication/http/registration.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.events import event_type_from_format_version +from synapse.events.snapshot import EventContext +from synapse.http.servlet import parse_json_object_from_request +from synapse.replication.http._base import ReplicationEndpoint +from synapse.types import Requester, UserID +from synapse.util.metrics import Measure + +logger = logging.getLogger(__name__) + + +class RegistrationUserCacheInvalidationServlet(ReplicationEndpoint): + """ + Invalidate the caches that a registration usually invalidates. + + Request format: + + POST /_synapse/replication/fed_query/:fed_cleanup_room/:txn_id + + {} + """ + + NAME = "reg_invalidate_user_caches" + PATH_ARGS = ("user_id",) + + def __init__(self, hs): + super(RegistrationUserCacheInvalidationServlet, self).__init__(hs) + self.store = hs.get_datastore() + + @staticmethod + def _serialize_payload(user_id, args): + """ + Args: + user_id (str) + """ + return {} + + @defer.inlineCallbacks + def _handle_request(self, request, user_id): + + def invalidate(txn): + self.store._invalidate_cache_and_stream( + txn, self.store.get_user_by_id, (user_id,) + ) + txn.call_after(self.store.is_guest.invalidate, (user_id,)) + + yield self.store.runInteraction("user_invalidate_caches", invalidate) + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + RegistrationUserCacheInvalidationServlet(hs).register(http_server) -- cgit 1.5.1 From d9235b9e29c88f8992b64904382e35905749351b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:39:49 +1100 Subject: fix appservice, add to frontend proxy --- synapse/app/frontend_proxy.py | 6 ++++++ synapse/rest/client/v2_alpha/register.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d5b954361d..5d450718c6 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -39,8 +39,12 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto from synapse.replication.tcp.client import ReplicationClientHandler from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns from synapse.rest.client.v2_alpha._base import client_v2_patterns +from synapse.rest.client.v2_alpha.register import ( + register_servlets as register_registration_servlets, +) from synapse.server import HomeServer from synapse.storage.engines import create_engine +from synapse.storage.registration import RegistrationStore from synapse.util.httpresourcetree import create_resource_tree from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole @@ -141,6 +145,7 @@ class FrontendProxySlavedStore( SlavedClientIpStore, SlavedApplicationServiceStore, SlavedRegistrationStore, + RegistrationStore, BaseSlavedStore, ): pass @@ -161,6 +166,7 @@ class FrontendProxyServer(HomeServer): elif name == "client": resource = JsonResource(self, canonical_json=False) KeyUploadServlet(self).register(resource) + register_registration_servlets(self, resource) # If presence is disabled, use the stub servlet that does # not allow sending presence diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ca9a850817..5ca9421a07 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -479,6 +479,10 @@ class RegisterRestServlet(RestServlet): user_id = yield self.registration_handler.appservice_register( username, as_token ) + + if self.hs.config.worker_app: + self._invalidate_caches_client(registered_user_id) + defer.returnValue((yield self._create_registration_details(user_id, body))) @defer.inlineCallbacks -- cgit 1.5.1 From e4381ed5144ba16faae0f2aedf07927344e69a4f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:42:04 +1100 Subject: pep8 --- synapse/replication/http/registration.py | 5 ----- synapse/rest/client/v2_alpha/register.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py index 5030bde499..513dafab39 100644 --- a/synapse/replication/http/registration.py +++ b/synapse/replication/http/registration.py @@ -17,12 +17,7 @@ import logging from twisted.internet import defer -from synapse.events import event_type_from_format_version -from synapse.events.snapshot import EventContext -from synapse.http.servlet import parse_json_object_from_request from synapse.replication.http._base import ReplicationEndpoint -from synapse.types import Requester, UserID -from synapse.util.metrics import Measure logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 5ca9421a07..9f750f7cf5 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -481,7 +481,7 @@ class RegisterRestServlet(RestServlet): ) if self.hs.config.worker_app: - self._invalidate_caches_client(registered_user_id) + self._invalidate_caches_client(user_id) defer.returnValue((yield self._create_registration_details(user_id, body))) -- cgit 1.5.1 From 84528e4fb2e191a065e51ae978755719b8b2ae71 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:49:09 +1100 Subject: cleanup --- synapse/rest/client/v2_alpha/register.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 9f750f7cf5..da9ebd540d 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -467,7 +467,7 @@ class RegisterRestServlet(RestServlet): yield self.registration_handler.post_consent_actions(registered_user_id) if self.hs.config.worker_app: - self._invalidate_caches_client(registered_user_id) + yield self._invalidate_caches_client(registered_user_id) defer.returnValue((200, return_dict)) @@ -481,7 +481,7 @@ class RegisterRestServlet(RestServlet): ) if self.hs.config.worker_app: - self._invalidate_caches_client(user_id) + yield self._invalidate_caches_client(user_id) defer.returnValue((yield self._create_registration_details(user_id, body))) -- cgit 1.5.1