From 4b090cb27363b84abeb8bfedfe22ebc4e6e3f7a1 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 22 Jan 2018 12:13:41 +0100 Subject: add federation_domain_whitelist gives a way to restrict which domains your HS is allowed to federate with. useful mainly for gracefully preventing a private but internet-connected HS from trying to federate to the wider public Matrix network --- synapse/config/server.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index 436dd8a6fe..b413a91c74 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -55,6 +55,15 @@ class ServerConfig(Config): "block_non_admin_invites", False, ) + federation_domain_whitelist = config.get( + "federation_domain_whitelist", [] + ) + # turn the whitelist into a hash for speed of lookup + self.federation_domain_whitelist = {} + for domain in federation_domain_whitelist: + self.federation_domain_whitelist[domain] = True + # FIXME: federation_domain_whitelist needs sytests + if self.public_baseurl is not None: if self.public_baseurl[-1] != '/': self.public_baseurl += '/' @@ -210,6 +219,16 @@ class ServerConfig(Config): # (except those sent by local server admins). The default is False. # block_non_admin_invites: True + # Restrict federation to the following whitelist of domains. + # N.B. we recommend also firewalling your federation listener to limit + # inbound federation traffic as early as possible, rather than relying + # purely on this application-layer restriction. + # + # federation_domain_whitelist: + # - lon.example.com + # - nyc.example.com + # - syd.example.com + # List of ports that Synapse should listen on, their purpose and their # configuration. listeners: -- cgit 1.4.1 From 313a489fc99f18773131814bc1f3843ccb45ad11 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 22 Jan 2018 14:54:46 +0100 Subject: incorporate PR feedback --- synapse/config/server.py | 15 +++++++++------ synapse/federation/federation_client.py | 2 +- synapse/federation/transaction_queue.py | 2 +- synapse/federation/transport/server.py | 2 +- synapse/handlers/device.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/rest/key/v2/remote_key_resource.py | 2 +- synapse/rest/media/v1/media_repository.py | 8 ++++---- 8 files changed, 19 insertions(+), 16 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index b413a91c74..f9c38199cb 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -55,14 +55,16 @@ class ServerConfig(Config): "block_non_admin_invites", False, ) + # FIXME: federation_domain_whitelist needs sytests + self.federation_domain_whitelist = None federation_domain_whitelist = config.get( - "federation_domain_whitelist", [] + "federation_domain_whitelist", None ) # turn the whitelist into a hash for speed of lookup - self.federation_domain_whitelist = {} - for domain in federation_domain_whitelist: - self.federation_domain_whitelist[domain] = True - # FIXME: federation_domain_whitelist needs sytests + if federation_domain_whitelist is not None: + self.federation_domain_whitelist = {} + for domain in federation_domain_whitelist: + self.federation_domain_whitelist[domain] = True if self.public_baseurl is not None: if self.public_baseurl[-1] != '/': @@ -222,7 +224,8 @@ class ServerConfig(Config): # Restrict federation to the following whitelist of domains. # N.B. we recommend also firewalling your federation listener to limit # inbound federation traffic as early as possible, rather than relying - # purely on this application-layer restriction. + # purely on this application-layer restriction. If not specified, the + # default is to whitelist nothing. # # federation_domain_whitelist: # - lon.example.com diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 22e1ba6efd..813907f7f2 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -267,7 +267,7 @@ class FederationClient(FederationBase): logger.info(e.message) continue except FederationDeniedError as e: - logger.debug(e.message) + logger.info(e.message) continue except Exception as e: pdu_attempts[destination] = now diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 343e5f75ab..a141ec9953 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -491,7 +491,7 @@ class TransactionQueue(object): ), ) except FederationDeniedError as e: - logger.debug(e) + logger.info(e) except Exception as e: logger.warn( "TX [%s] Failed to send transaction: %s", diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 38b445604d..06c16ba4fa 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -94,7 +94,7 @@ class Authenticator(object): } if ( - self.federation_domain_whitelist and + self.federation_domain_whitelist is not None and self.server_name not in self.federation_domain_whitelist ): raise FederationDeniedError(self.server_name) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 7e150b19bb..0e83453851 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -515,7 +515,7 @@ class DeviceListEduUpdater(object): # eventually become consistent. return except FederationDeniedError as e: - logger.debug(e) + logger.info(e) return except Exception: # TODO: Remember that we are now out of sync and try again diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c6344f322f..677532c87b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -784,7 +784,7 @@ class FederationHandler(BaseHandler): logger.info(e.message) continue except FederationDeniedError as e: - logger.debug(e) + logger.info(e) continue except Exception as e: logger.exception( diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 94110bf592..17e6079cba 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -139,7 +139,7 @@ class RemoteKey(Resource): store_queries = [] for server_name, key_ids in query.items(): if ( - self.federation_domain_whitelist and + self.federation_domain_whitelist is not None and server_name not in self.federation_domain_whitelist ): logger.debug("Federation denied with %s", server_name) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 7cd7a8ec1f..332ba4b128 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -226,7 +226,7 @@ class MediaRepository(object): to request """ if ( - self.federation_domain_whitelist and + self.federation_domain_whitelist is not None and server_name not in self.federation_domain_whitelist ): raise FederationDeniedError(server_name) @@ -266,7 +266,7 @@ class MediaRepository(object): Deferred[dict]: The media_info of the file """ if ( - self.federation_domain_whitelist and + self.federation_domain_whitelist is not None and server_name not in self.federation_domain_whitelist ): raise FederationDeniedError(server_name) @@ -387,8 +387,8 @@ class MediaRepository(object): logger.warn("Not retrying destination %r", server_name) raise SynapseError(502, "Failed to fetch remote media") except FederationDeniedError as e: - logger.debug(e) - raise SynapseError(403, e.message) + logger.info(e) + raise e except Exception: logger.exception("Failed to fetch remote media %s/%s", server_name, media_id) -- cgit 1.4.1 From fa80b492a5afabe919e75461bf0d2d050411dae8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 22 Jan 2018 18:42:36 +0100 Subject: fix thinko --- synapse/config/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/server.py b/synapse/config/server.py index f9c38199cb..8f0b6d1f28 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -225,7 +225,7 @@ class ServerConfig(Config): # N.B. we recommend also firewalling your federation listener to limit # inbound federation traffic as early as possible, rather than relying # purely on this application-layer restriction. If not specified, the - # default is to whitelist nothing. + # default is to whitelist everything. # # federation_domain_whitelist: # - lon.example.com -- cgit 1.4.1 From e44607747883af3ec4ebc3ae42f38f2c00010385 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 13 Mar 2018 01:34:20 +0000 Subject: delegate to the IS to check 3PID signup eligibility --- synapse/config/registration.py | 6 ++++++ synapse/util/threepids.py | 25 +++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 336959094b..14ef535b27 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -33,6 +33,7 @@ class RegistrationConfig(Config): self.registrations_require_3pid = config.get("registrations_require_3pid", []) self.allowed_local_3pids = config.get("allowed_local_3pids", []) + self.check_is_for_allowed_local_3pids = config.get("check_is_for_allowed_local_3pids", False) self.registration_shared_secret = config.get("registration_shared_secret") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) @@ -63,6 +64,10 @@ class RegistrationConfig(Config): # Mandate that users are only allowed to associate certain formats of # 3PIDs with accounts on this server. # + # Use an Identity Server to establish which 3PIDs are allowed to register? + # Overrides allowed_local_3pids below. + # check_is_for_allowed_local_3pids: matrix.org + # # allowed_local_3pids: # - medium: email # pattern: ".*@matrix\\.org" @@ -71,6 +76,7 @@ class RegistrationConfig(Config): # - medium: msisdn # pattern: "\\+44" + # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py index 75efa0117b..cd629c2ec9 100644 --- a/synapse/util/threepids.py +++ b/synapse/util/threepids.py @@ -16,9 +16,12 @@ import logging import re +from twisted.internet import defer + logger = logging.getLogger(__name__) +@defer.inlineCallbacks def check_3pid_allowed(hs, medium, address): """Checks whether a given format of 3PID is allowed to be used on this HS @@ -28,9 +31,20 @@ def check_3pid_allowed(hs, medium, address): address (str): address within that medium (e.g. "wotan@matrix.org") msisdns need to first have been canonicalised Returns: - bool: whether the 3PID medium/address is allowed to be added to this HS + defered bool: whether the 3PID medium/address is allowed to be added to this HS """ + if hs.config.check_is_for_allowed_local_3pids: + data = yield hs.http_client.get_json( + "https://%s%s" % ( + hs.config.check_is_for_allowed_local_3pids, + "/_matrix/identity/api/v1/discover_urls" + ), + {'medium': medium, 'address': address } + ) + defer.returnValue(data.hs_url+"/" == self.hs.config.public_baseurl) + return + if hs.config.allowed_local_3pids: for constraint in hs.config.allowed_local_3pids: logger.debug( @@ -41,8 +55,11 @@ def check_3pid_allowed(hs, medium, address): medium == constraint['medium'] and re.match(constraint['pattern'], address) ): - return True + defer.returnValue(True) + return else: - return True + defer.returnValue(True) + return - return False + defer.returnValue(False) + return -- cgit 1.4.1 From 739d3500feef8a4cb84a60839bba486001691697 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 13 Mar 2018 01:50:32 +0000 Subject: pep8 --- synapse/config/registration.py | 4 +++- synapse/handlers/register.py | 4 +++- synapse/util/threepids.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 14ef535b27..6cd830a78e 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -33,7 +33,9 @@ class RegistrationConfig(Config): self.registrations_require_3pid = config.get("registrations_require_3pid", []) self.allowed_local_3pids = config.get("allowed_local_3pids", []) - self.check_is_for_allowed_local_3pids = config.get("check_is_for_allowed_local_3pids", False) + self.check_is_for_allowed_local_3pids = config.get( + "check_is_for_allowed_local_3pids", False + ) self.registration_shared_secret = config.get("registration_shared_secret") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 87d0a45806..96f480f836 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -308,7 +308,9 @@ class RegistrationHandler(BaseHandler): logger.info("got threepid with medium '%s' and address '%s'", threepid['medium'], threepid['address']) - if not (yield check_3pid_allowed(self.hs, threepid['medium'], threepid['address'])): + if not ( + yield check_3pid_allowed(self.hs, threepid['medium'], threepid['address']) + ): raise RegistrationError( 403, "Third party identifier is not allowed" ) diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py index d2f646db9c..e1cf5dbb13 100644 --- a/synapse/util/threepids.py +++ b/synapse/util/threepids.py @@ -40,7 +40,7 @@ def check_3pid_allowed(hs, medium, address): hs.config.check_is_for_allowed_local_3pids, "/_matrix/identity/api/v1/discover_urls" ), - {'medium': medium, 'address': address } + {'medium': medium, 'address': address} ) defer.returnValue(data['hs_url'] + "/" == hs.config.public_baseurl) return -- cgit 1.4.1 From 5c341c99f67b2533980fbf567b2e80c039f68a43 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 13 Mar 2018 21:15:14 +0000 Subject: add 'allow_invited_3pids' option to invited 3PIDs to register --- synapse/config/registration.py | 7 +++++++ synapse/util/threepids.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 6cd830a78e..dc3c85a517 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -36,6 +36,7 @@ class RegistrationConfig(Config): self.check_is_for_allowed_local_3pids = config.get( "check_is_for_allowed_local_3pids", False ) + self.allow_invited_3pids = config.get("allow_invited_3pids", False) self.registration_shared_secret = config.get("registration_shared_secret") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) @@ -70,6 +71,12 @@ class RegistrationConfig(Config): # Overrides allowed_local_3pids below. # check_is_for_allowed_local_3pids: matrix.org # + # If you are using an IS you can also check whether that IS registers + # pending invites for the given 3PID (and then allow it to sign up on + # the platform): + # + # allow_invited_3pids: False + # # allowed_local_3pids: # - medium: email # pattern: ".*@matrix\\.org" diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py index e1cf5dbb13..94c0852f0c 100644 --- a/synapse/util/threepids.py +++ b/synapse/util/threepids.py @@ -42,7 +42,10 @@ def check_3pid_allowed(hs, medium, address): ), {'medium': medium, 'address': address} ) - defer.returnValue(data['hs_url'] + "/" == hs.config.public_baseurl) + if hs.config.allow_invited_3pids and data.get('invited'): + defer.returnValue(True) + else: + defer.returnValue(data['hs_url'] + "/" == hs.config.public_baseurl) return if hs.config.allowed_local_3pids: -- cgit 1.4.1 From 2e4a6c5aaba238760c4ad4e97ca132ccbbb2cc7b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 14 Mar 2018 22:09:08 +0000 Subject: incorporate PR feedback and rename URL --- synapse/config/registration.py | 3 +-- synapse/util/threepids.py | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index dc3c85a517..2854a48d0a 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -34,7 +34,7 @@ class RegistrationConfig(Config): self.registrations_require_3pid = config.get("registrations_require_3pid", []) self.allowed_local_3pids = config.get("allowed_local_3pids", []) self.check_is_for_allowed_local_3pids = config.get( - "check_is_for_allowed_local_3pids", False + "check_is_for_allowed_local_3pids", None ) self.allow_invited_3pids = config.get("allow_invited_3pids", False) self.registration_shared_secret = config.get("registration_shared_secret") @@ -85,7 +85,6 @@ class RegistrationConfig(Config): # - medium: msisdn # pattern: "\\+44" - # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py index 94c0852f0c..353d220bad 100644 --- a/synapse/util/threepids.py +++ b/synapse/util/threepids.py @@ -38,15 +38,14 @@ def check_3pid_allowed(hs, medium, address): data = yield hs.get_simple_http_client().get_json( "https://%s%s" % ( hs.config.check_is_for_allowed_local_3pids, - "/_matrix/identity/api/v1/discover_urls" + "/_matrix/identity/api/v1/info" ), {'medium': medium, 'address': address} ) if hs.config.allow_invited_3pids and data.get('invited'): defer.returnValue(True) else: - defer.returnValue(data['hs_url'] + "/" == hs.config.public_baseurl) - return + defer.returnValue(data['hs'] == hs.config.server_name) if hs.config.allowed_local_3pids: for constraint in hs.config.allowed_local_3pids: @@ -59,10 +58,7 @@ def check_3pid_allowed(hs, medium, address): re.match(constraint['pattern'], address) ): defer.returnValue(True) - return else: defer.returnValue(True) - return defer.returnValue(False) - return -- cgit 1.4.1 From e654230a51affdeca69b365b0ec43ff95134bdf6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 Apr 2018 17:41:58 +0100 Subject: Written but untested profile replication --- synapse/config/registration.py | 8 +++ synapse/handlers/profile.py | 70 +++++++++++++++++++++- synapse/storage/prepare_database.py | 2 +- synapse/storage/profile.py | 56 +++++++++++++++++ synapse/storage/schema/delta/48/profiles_batch.sql | 23 +++++++ 5 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 synapse/storage/schema/delta/48/profiles_batch.sql (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 2854a48d0a..8f4d051fba 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -49,6 +49,10 @@ class RegistrationConfig(Config): self.auto_join_rooms = config.get("auto_join_rooms", []) + self.replicate_user_profiles_to = config.get("replicate_user_profiles_to", []) + if not isinstance(self.replicate_user_profiles_to, list): + self.replicate_user_profiles_to = [self.replicate_user_profiles_to,] + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) @@ -106,6 +110,10 @@ class RegistrationConfig(Config): - vector.im - riot.im + # If enabled, user IDs, display names and avatar URLs will be replicated + # to this server whenever they change. + # replicate_user_profiles_to: example.com + # Users who register on this homeserver will automatically be joined # to these rooms #auto_join_rooms: diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index cb710fe796..be9d1d8c41 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -15,12 +15,14 @@ import logging -from twisted.internet import defer +from twisted.internet import defer, reactor from synapse.api.errors import SynapseError, AuthError, CodeMessageException from synapse.types import UserID, get_domain_from_id from ._base import BaseHandler +from signedjson.sign import sign_json + logger = logging.getLogger(__name__) @@ -28,6 +30,8 @@ class ProfileHandler(BaseHandler): PROFILE_UPDATE_MS = 60 * 1000 PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000 + PROFILE_REPLICATE_INTERVAL = 2 * 60 * 1000 + def __init__(self, hs): super(ProfileHandler, self).__init__(hs) @@ -38,8 +42,72 @@ class ProfileHandler(BaseHandler): self.user_directory_handler = hs.get_user_directory_handler() + self.http_client = hs.get_simple_http_client() + self.clock.looping_call(self._update_remote_profile_cache, self.PROFILE_UPDATE_MS) + reactor.callWhenRunning(self._assign_profile_replication_batches) + reactor.callWhenRunning(self._replicate_profiles) + self.clock.looping_call(self._replicate_profiles, self.PROFILE_REPLICATE_INTERVAL) + + @defer.inlineCallbacks + def _assign_profile_replication_batches(self): + """If no profile replication has been done yet, allocate replication batch + numbers to each profile to start the replication process. + """ + logger.info("Assigning profile batch numbers...") + total = 0 + while True: + assigned = yield self.store.assign_profile_batch() + total += assigned + if assigned == 0: + break + logger.info("Assigned %d profile batch numbers", total) + + @defer.inlineCallbacks + def _replicate_profiles(self): + """If any profile data has been updated and not pushed to the replication targets, + replicate it. + """ + host_batches = yield self.store.get_replication_hosts() + latest_batch = yield self.store.get_latest_profile_replication_batch_number() + for repl_host in self.hs.config.replicate_user_profiles_to: + if repl_host not in host_batches: + host_batches[repl_host] = -1 + try: + for i in xrange(host_batches[repl_host] + 1, latest_batch + 1): + yield self._replicate_host_profile_batch(repl_host, i) + except: + logger.exception( + "Exception while replicating to %s: aborting for now", repl_host, + ) + + @defer.inlineCallbacks + def _replicate_host_profile_batch(self, host, batchnum): + logger.info("Replicating profile batch %d to %s", batchnum, host) + batch_rows = yield self.store.get_profile_batch(batchnum) + batch = { + UserID(r["user_id"], self.hs.hostname).to_string(): { + "displayname": r["displayname"], + "avatar_url": r["avatar_url"], + } for r in batch_rows + } + + url = "https://%s/_matrix/federation/v1/replicate_profiles" % (host,) + signed_batch = { + "batchnum": batchnum, + "signed_batch": sign_json(batch, self.hs.hostname, self.hs.config.signing_key[0]), + "origin_server": self.hs.hostname, + } + try: + yield self.http_client.post_json_get_json(url, signed_batch) + self.store.update_replication_batch_for_host(host, batchnum) + logger.info("Sucessfully replicated profile batch %d to %s", batchnum, host) + except: + # This will get retried when the looping call next comes around + logger.exception("Failed to replicate profile batch %d to %s", batchnum, host) + raise + @defer.inlineCallbacks def get_profile(self, user_id): target_user = UserID.from_string(user_id) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index c845a0cec5..68675e15d2 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 47 +SCHEMA_VERSION = 48 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index ec02e73bc2..cfc4a0606e 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +21,8 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore +BATCH_SIZE = 100 + class ProfileStore(SQLBaseStore): def create_profile(self, user_localpart): @@ -85,6 +88,59 @@ class ProfileStore(SQLBaseStore): desc="set_profile_avatar_url", ) + @defer.inlineCallbacks + def get_latest_profile_replication_batch_number(self): + def f(txn): + txn.execute("SELECT MAX(batch) as maxbatch FROM profiles") + rows = self.cursor_to_dict(txn) + return rows[0]['maxbatch'] + max_batch = yield self.runInteraction("get_latest_profile_replication_batch_number", f) + defer.returnValue(max_batch) + + def get_profile_batch(self, batchnum): + return self._simple_select_list( + table="profiles", + keyvalues={ + "batch": batchnum, + }, + retcols=("user_id", "displayname", "avatar_url"), + desc="get_profile_batch", + ) + + @defer.inlineCallbacks + def assign_profile_batch(self): + def f(txn): + sql = ( + "UPDATE profiles SET batch = " + "(SELECT IFNULL(MAX(batch), -1) + 1 FROM profiles) " + "WHERE user_id in (" + " SELECT user_id FROM profiles WHERE batch is NULL limit ?" + ")" + ) + txn.execute(sql, (BATCH_SIZE,)) + return txn.rowcount + assigned = yield self.runInteraction("assign_profile_batch", f) + defer.returnValue(assigned) + + @defer.inlineCallbacks + def get_replication_hosts(self): + def f(txn): + txn.execute("SELECT host, last_synced_batch FROM profile_replication_status") + rows = self.cursor_to_dict(txn) + return { r['host']: r['last_synced_batch'] for r in rows } + result = yield self.runInteraction("get_replication_hosts", f) + defer.returnValue(result) + + def update_replication_batch_for_host(self, host, last_synced_batch): + return self._simple_upsert( + table="profile_replication_status", + keyvalues={"host": host}, + values={ + "last_synced_batch": last_synced_batch, + }, + desc="update_replication_batch_for_host", + ) + def get_from_remote_profile_cache(self, user_id): return self._simple_select_one( table="remote_profile_cache", diff --git a/synapse/storage/schema/delta/48/profiles_batch.sql b/synapse/storage/schema/delta/48/profiles_batch.sql new file mode 100644 index 0000000000..7639ff22d5 --- /dev/null +++ b/synapse/storage/schema/delta/48/profiles_batch.sql @@ -0,0 +1,23 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE profiles ADD COLUMN batch BIGINT DEFAULT NULL; + +CREATE INDEX profiles_batch_idx ON profiles(batch); + +CREATE TABLE profile_replication_status ( + host TEXT NOT NULL, + last_synced_batch BIGINT NOT NULL +); -- cgit 1.4.1 From 8743f42b495fdb8a7cf14d1c16021655b0d0222e Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 17 Apr 2018 10:34:04 +0100 Subject: pep8 --- synapse/config/registration.py | 2 +- synapse/handlers/profile.py | 4 ++-- synapse/storage/profile.py | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 14373c502d..5c21287998 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -51,7 +51,7 @@ class RegistrationConfig(Config): self.replicate_user_profiles_to = config.get("replicate_user_profiles_to", []) if not isinstance(self.replicate_user_profiles_to, list): - self.replicate_user_profiles_to = [self.replicate_user_profiles_to,] + self.replicate_user_profiles_to = [self.replicate_user_profiles_to, ] def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index d56adf069b..bf91eedb7f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -83,7 +83,7 @@ class ProfileHandler(BaseHandler): try: for i in xrange(host_batches[repl_host] + 1, latest_batch + 1): yield self._replicate_host_profile_batch(repl_host, i) - except: + except Exception: logger.exception( "Exception while replicating to %s: aborting for now", repl_host, ) @@ -110,7 +110,7 @@ class ProfileHandler(BaseHandler): yield self.http_client.post_json_get_json(url, signed_body) self.store.update_replication_batch_for_host(host, batchnum) logger.info("Sucessfully replicated profile batch %d to %s", batchnum, host) - except: + except Exception: # This will get retried when the looping call next comes around logger.exception("Failed to replicate profile batch %d to %s", batchnum, host) raise diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index a3f144eb41..048f48dcc1 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -71,7 +71,9 @@ class ProfileWorkerStore(SQLBaseStore): txn.execute("SELECT MAX(batch) as maxbatch FROM profiles") rows = self.cursor_to_dict(txn) return rows[0]['maxbatch'] - max_batch = yield self.runInteraction("get_latest_profile_replication_batch_number", f) + max_batch = yield self.runInteraction( + "get_latest_profile_replication_batch_number", f, + ) defer.returnValue(max_batch) def get_profile_batch(self, batchnum): @@ -104,7 +106,7 @@ class ProfileWorkerStore(SQLBaseStore): def f(txn): txn.execute("SELECT host, last_synced_batch FROM profile_replication_status") rows = self.cursor_to_dict(txn) - return { r['host']: r['last_synced_batch'] for r in rows } + return {r['host']: r['last_synced_batch'] for r in rows} result = yield self.runInteraction("get_replication_hosts", f) defer.returnValue(result) -- cgit 1.4.1 From 6554253f487f668e4b63b0ace7b7a170078d3045 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 19 Apr 2018 19:28:12 +0100 Subject: Option to defer to an ID server for user_directory --- synapse/config/user_directory.py | 8 ++++++++ synapse/rest/client/v2_alpha/user_directory.py | 10 ++++++++++ 2 files changed, 18 insertions(+) (limited to 'synapse/config') diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py index 38e8947843..093ebc4d03 100644 --- a/synapse/config/user_directory.py +++ b/synapse/config/user_directory.py @@ -23,11 +23,15 @@ class UserDirectoryConfig(Config): def read_config(self, config): self.user_directory_search_all_users = False + self.user_directory_defer_to_id_server = None user_directory_config = config.get("user_directory", None) if user_directory_config: self.user_directory_search_all_users = ( user_directory_config.get("search_all_users", False) ) + self.user_directory_defer_to_id_server = ( + user_directory_config.get("defer_to_id_server", None) + ) def default_config(self, config_dir_path, server_name, **kwargs): return """ @@ -41,4 +45,8 @@ class UserDirectoryConfig(Config): # #user_directory: # search_all_users: false + # + # If this is set, user search will be delegated to this ID server instead + # of synapse performing the saerch itself. + # defer_to_id_server: id.example.com """ diff --git a/synapse/rest/client/v2_alpha/user_directory.py b/synapse/rest/client/v2_alpha/user_directory.py index 2d4a43c353..fd36389148 100644 --- a/synapse/rest/client/v2_alpha/user_directory.py +++ b/synapse/rest/client/v2_alpha/user_directory.py @@ -16,6 +16,7 @@ import logging from twisted.internet import defer +from signedjson.sign import sign_json from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet, parse_json_object_from_request @@ -36,6 +37,7 @@ class UserDirectorySearchRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() self.user_directory_handler = hs.get_user_directory_handler() + self.http_client = hs.get_simple_http_client() @defer.inlineCallbacks def on_POST(self, request): @@ -60,6 +62,14 @@ class UserDirectorySearchRestServlet(RestServlet): body = parse_json_object_from_request(request) + if self.hs.config.user_directory_defer_to_id_server: + signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0]) + url = "https://%s/_matrix/identity/api/v1/user_directory/search" % ( + self.hs.config.user_directory_defer_to_id_server, + ) + resp = yield self.http_client.post_json_get_json(url, signed_body) + defer.returnValue((200, resp)) + limit = body.get("limit", 10) limit = min(limit, 50) -- cgit 1.4.1 From 643c89d4974eccda45802bc72d783904784a52a3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Apr 2018 11:40:37 +0100 Subject: Fix spelling & add experimental API comment --- synapse/config/user_directory.py | 3 ++- synapse/rest/client/v2_alpha/user_directory.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse/config') diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py index 093ebc4d03..9b0ae91289 100644 --- a/synapse/config/user_directory.py +++ b/synapse/config/user_directory.py @@ -47,6 +47,7 @@ class UserDirectoryConfig(Config): # search_all_users: false # # If this is set, user search will be delegated to this ID server instead - # of synapse performing the saerch itself. + # of synapse performing the search itself. + # This is an experimental API. # defer_to_id_server: id.example.com """ diff --git a/synapse/rest/client/v2_alpha/user_directory.py b/synapse/rest/client/v2_alpha/user_directory.py index fd36389148..9b1a019978 100644 --- a/synapse/rest/client/v2_alpha/user_directory.py +++ b/synapse/rest/client/v2_alpha/user_directory.py @@ -64,7 +64,7 @@ class UserDirectorySearchRestServlet(RestServlet): if self.hs.config.user_directory_defer_to_id_server: signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0]) - url = "https://%s/_matrix/identity/api/v1/user_directory/search" % ( + url = "http://%s/_matrix/identity/api/v1/user_directory/search" % ( self.hs.config.user_directory_defer_to_id_server, ) resp = yield self.http_client.post_json_get_json(url, signed_body) -- cgit 1.4.1 From de341bec1b04b68449a80939cb1738e122b4076c Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Apr 2018 11:51:57 +0100 Subject: Add 'ex[erimental API' comment --- synapse/config/registration.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse/config') diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 5c21287998..34326718ad 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -114,6 +114,8 @@ class RegistrationConfig(Config): # If enabled, user IDs, display names and avatar URLs will be replicated # to this server whenever they change. + # This is an experimental API currently implemented by sydent to support + # cross-homeserver user directories. # replicate_user_profiles_to: example.com # Users who register on this homeserver will automatically be joined -- cgit 1.4.1