summary refs log tree commit diff
path: root/synapse
diff options
context:
space:
mode:
Diffstat (limited to 'synapse')
-rw-r--r--synapse/api/constants.py3
-rw-r--r--synapse/config/registration.py51
-rw-r--r--synapse/config/user_directory.py9
-rw-r--r--synapse/handlers/deactivate_account.py4
-rw-r--r--synapse/handlers/profile.py148
-rw-r--r--synapse/handlers/register.py49
-rw-r--r--synapse/handlers/room.py11
-rw-r--r--synapse/handlers/sync.py44
-rw-r--r--synapse/rest/client/v2_alpha/account.py15
-rw-r--r--synapse/rest/client/v2_alpha/register.py103
-rw-r--r--synapse/rest/client/v2_alpha/user_directory.py11
-rw-r--r--synapse/rulecheck/__init__.py0
-rw-r--r--synapse/rulecheck/domain_rule_checker.py100
-rw-r--r--synapse/storage/profile.py93
-rw-r--r--synapse/storage/registration.py14
-rw-r--r--synapse/storage/schema/delta/48/profiles_batch.sql36
-rw-r--r--synapse/storage/schema/delta/50/profiles_deactivated_users.sql23
-rw-r--r--synapse/types.py12
-rw-r--r--synapse/util/threepids.py24
19 files changed, 664 insertions, 86 deletions
diff --git a/synapse/api/constants.py b/synapse/api/constants.py

index c2630c4c64..9ad1939ed5 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py
@@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd. +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -71,6 +71,7 @@ class EventTypes(object): CanonicalAlias = "m.room.canonical_alias" RoomAvatar = "m.room.avatar" GuestAccess = "m.room.guest_access" + Encryption = "m.room.encryption" # These are used for validation Message = "m.room.message" diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index 7480ed5145..0f41a6602e 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py
@@ -33,7 +33,15 @@ class RegistrationConfig(Config): self.registrations_require_3pid = config.get("registrations_require_3pid", []) self.allowed_local_3pids = config.get("allowed_local_3pids", []) + self.check_is_for_allowed_local_3pids = config.get( + "check_is_for_allowed_local_3pids", None + ) + self.allow_invited_3pids = config.get("allow_invited_3pids", False) + + self.disable_3pid_changes = config.get("disable_3pid_changes", False) + self.registration_shared_secret = config.get("registration_shared_secret") + self.register_mxid_from_3pid = config.get("register_mxid_from_3pid") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"] @@ -49,6 +57,13 @@ class RegistrationConfig(Config): raise ConfigError('Invalid auto_join_rooms entry %s' % (room_alias,)) self.autocreate_auto_join_rooms = config.get("autocreate_auto_join_rooms", True) + self.disable_set_displayname = config.get("disable_set_displayname", False) + self.disable_set_avatar_url = config.get("disable_set_avatar_url", False) + + self.replicate_user_profiles_to = config.get("replicate_user_profiles_to", []) + if not isinstance(self.replicate_user_profiles_to, list): + self.replicate_user_profiles_to = [self.replicate_user_profiles_to, ] + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) @@ -64,9 +79,26 @@ class RegistrationConfig(Config): # - email # - msisdn + # Derive the user's matrix ID from a type of 3PID used when registering. + # This overrides any matrix ID the user proposes when calling /register + # The 3PID type should be present in registrations_require_3pid to avoid + # users failing to register if they don't specify the right kind of 3pid. + # + # register_mxid_from_3pid: email + # Mandate that users are only allowed to associate certain formats of # 3PIDs with accounts on this server. # + # Use an Identity Server to establish which 3PIDs are allowed to register? + # Overrides allowed_local_3pids below. + # check_is_for_allowed_local_3pids: matrix.org + # + # If you are using an IS you can also check whether that IS registers + # pending invites for the given 3PID (and then allow it to sign up on + # the platform): + # + # allow_invited_3pids: False + # # allowed_local_3pids: # - medium: email # pattern: ".*@matrix\\.org" @@ -75,6 +107,11 @@ class RegistrationConfig(Config): # - medium: msisdn # pattern: "\\+44" + # If true, stop users from trying to change the 3PIDs associated with + # their accounts. + # + # disable_3pid_changes: False + # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" @@ -98,6 +135,20 @@ class RegistrationConfig(Config): - vector.im - riot.im + # If enabled, user IDs, display names and avatar URLs will be replicated + # to this server whenever they change. + # This is an experimental API currently implemented by sydent to support + # cross-homeserver user directories. + # replicate_user_profiles_to: example.com + + # If enabled, don't let users set their own display names/avatars + # other than for the very first time (unless they are a server admin). + # Useful when provisioning users based on the contents of a 3rd party + # directory and to avoid ambiguities. + # + # disable_set_displayname: False + # disable_set_avatar_url: False + # Users who register on this homeserver will automatically be joined # to these rooms #auto_join_rooms: diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py
index 38e8947843..eba99976ac 100644 --- a/synapse/config/user_directory.py +++ b/synapse/config/user_directory.py
@@ -23,11 +23,15 @@ class UserDirectoryConfig(Config): def read_config(self, config): self.user_directory_search_all_users = False + self.user_directory_defer_to_id_server = None user_directory_config = config.get("user_directory", None) if user_directory_config: self.user_directory_search_all_users = ( user_directory_config.get("search_all_users", False) ) + self.user_directory_defer_to_id_server = ( + user_directory_config.get("defer_to_id_server", None) + ) def default_config(self, config_dir_path, server_name, **kwargs): return """ @@ -41,4 +45,9 @@ class UserDirectoryConfig(Config): # #user_directory: # search_all_users: false + # + # If this is set, user search will be delegated to this ID server instead + # of synapse performing the search itself. + # This is an experimental API. + # defer_to_id_server: https://id.example.com """ diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py
index 75fe50c42c..7f9435179a 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py
@@ -33,6 +33,7 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() self._identity_handler = hs.get_handlers().identity_handler + self._profile_handler = hs.get_profile_handler() self.user_directory_handler = hs.get_user_directory_handler() # Flag that indicates whether the process to part users from rooms is running @@ -94,6 +95,9 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_set_password_hash(user_id, None) + user = UserID.from_string(user_id) + yield self._profile_handler.set_active(user, False) + # Add the user to a table of users pending deactivation (ie. # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 1dfbde84fd..65e29518f0 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py
@@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +16,9 @@ import logging -from twisted.internet import defer +from signedjson.sign import sign_json + +from twisted.internet import defer, reactor from synapse.api.errors import ( AuthError, @@ -26,6 +29,7 @@ from synapse.api.errors import ( ) from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import UserID, get_domain_from_id +from synapse.util.logcontext import run_in_background from ._base import BaseHandler @@ -40,6 +44,8 @@ class BaseProfileHandler(BaseHandler): subclass MasterProfileHandler """ + PROFILE_REPLICATE_INTERVAL = 2 * 60 * 1000 + def __init__(self, hs): super(BaseProfileHandler, self).__init__(hs) @@ -50,6 +56,84 @@ class BaseProfileHandler(BaseHandler): self.user_directory_handler = hs.get_user_directory_handler() + self.http_client = hs.get_simple_http_client() + + if hs.config.worker_app is None: + self.clock.looping_call( + self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS, + ) + + if len(self.hs.config.replicate_user_profiles_to) > 0: + reactor.callWhenRunning(self._assign_profile_replication_batches) + reactor.callWhenRunning(self._replicate_profiles) + # Add a looping call to replicate_profiles: this handles retries + # if the replication is unsuccessful when the user updated their + # profile. + self.clock.looping_call( + self._replicate_profiles, self.PROFILE_REPLICATE_INTERVAL + ) + + @defer.inlineCallbacks + def _assign_profile_replication_batches(self): + """If no profile replication has been done yet, allocate replication batch + numbers to each profile to start the replication process. + """ + logger.info("Assigning profile batch numbers...") + total = 0 + while True: + assigned = yield self.store.assign_profile_batch() + total += assigned + if assigned == 0: + break + logger.info("Assigned %d profile batch numbers", total) + + @defer.inlineCallbacks + def _replicate_profiles(self): + """If any profile data has been updated and not pushed to the replication targets, + replicate it. + """ + host_batches = yield self.store.get_replication_hosts() + latest_batch = yield self.store.get_latest_profile_replication_batch_number() + if latest_batch is None: + latest_batch = -1 + for repl_host in self.hs.config.replicate_user_profiles_to: + if repl_host not in host_batches: + host_batches[repl_host] = -1 + try: + for i in xrange(host_batches[repl_host] + 1, latest_batch + 1): + yield self._replicate_host_profile_batch(repl_host, i) + except Exception: + logger.exception( + "Exception while replicating to %s: aborting for now", repl_host, + ) + + @defer.inlineCallbacks + def _replicate_host_profile_batch(self, host, batchnum): + logger.info("Replicating profile batch %d to %s", batchnum, host) + batch_rows = yield self.store.get_profile_batch(batchnum) + batch = { + UserID(r["user_id"], self.hs.hostname).to_string(): ({ + "display_name": r["displayname"], + "avatar_url": r["avatar_url"], + } if r["active"] else None) for r in batch_rows + } + + url = "https://%s/_matrix/identity/api/v1/replicate_profiles" % (host,) + body = { + "batchnum": batchnum, + "batch": batch, + "origin_server": self.hs.hostname, + } + signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0]) + try: + yield self.http_client.post_json_get_json(url, signed_body) + yield self.store.update_replication_batch_for_host(host, batchnum) + logger.info("Sucessfully replicated profile batch %d to %s", batchnum, host) + except Exception: + # This will get retried when the looping call next comes around + logger.exception("Failed to replicate profile batch %d to %s", batchnum, host) + raise + @defer.inlineCallbacks def get_profile(self, user_id): target_user = UserID.from_string(user_id) @@ -147,19 +231,30 @@ class BaseProfileHandler(BaseHandler): @defer.inlineCallbacks def set_displayname(self, target_user, requester, new_displayname, by_admin=False): - """target_user is the user whose displayname is to be changed; - auth_user is the user attempting to make this change.""" + """target_user is the UserID whose displayname is to be changed; + requester is the authenticated user attempting to make this change.""" if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") - if not by_admin and target_user != requester.user: + if not by_admin and requester and target_user != requester.user: raise AuthError(400, "Cannot set another user's displayname") + if not by_admin and self.hs.config.disable_set_displayname: + profile = yield self.store.get_profileinfo(target_user.localpart) + if profile.display_name: + raise SynapseError(400, "Changing displayname is disabled on this server") + if new_displayname == '': new_displayname = None + if len(self.hs.config.replicate_user_profiles_to) > 0: + cur_batchnum = yield self.store.get_latest_profile_replication_batch_number() + new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1 + else: + new_batchnum = None + yield self.store.set_profile_displayname( - target_user.localpart, new_displayname + target_user.localpart, new_displayname, new_batchnum ) if self.hs.config.user_directory_search_all_users: @@ -168,7 +263,32 @@ class BaseProfileHandler(BaseHandler): target_user.to_string(), profile ) - yield self._update_join_states(requester, target_user) + if requester: + yield self._update_join_states(requester, target_user) + + # start a profile replication push + run_in_background(self._replicate_profiles) + + @defer.inlineCallbacks + def set_active(self, target_user, active): + """ + Sets the 'active' flag on a user profile. If set to false, the user account is + considered deactivated. + Note that unlike set_displayname and set_avatar_url, this does *not* perform + authorization checks! This is because the only place it's used currently is + in account deactivation where we've already done these checks anyway. + """ + if len(self.hs.config.replicate_user_profiles_to) > 0: + cur_batchnum = yield self.store.get_latest_profile_replication_batch_number() + new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1 + else: + new_batchnum = None + yield self.store.set_profile_active( + target_user.localpart, active, new_batchnum + ) + + # start a profile replication push + run_in_background(self._replicate_profiles) @defer.inlineCallbacks def get_avatar_url(self, target_user): @@ -210,8 +330,19 @@ class BaseProfileHandler(BaseHandler): if not by_admin and target_user != requester.user: raise AuthError(400, "Cannot set another user's avatar_url") + if not by_admin and self.hs.config.disable_set_avatar_url: + profile = yield self.store.get_profileinfo(target_user.localpart) + if profile.avatar_url: + raise SynapseError(400, "Changing avatar url is disabled on this server") + + if len(self.hs.config.replicate_user_profiles_to) > 0: + cur_batchnum = yield self.store.get_latest_profile_replication_batch_number() + new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1 + else: + new_batchnum = None + yield self.store.set_profile_avatar_url( - target_user.localpart, new_avatar_url + target_user.localpart, new_avatar_url, new_batchnum, ) if self.hs.config.user_directory_search_all_users: @@ -222,6 +353,9 @@ class BaseProfileHandler(BaseHandler): yield self._update_join_states(requester, target_user) + # start a profile replication push + run_in_background(self._replicate_profiles) + @defer.inlineCallbacks def on_profile_query(self, args): user = UserID.from_string(args["user_id"]) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index e9d7b25a36..757eacd214 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py
@@ -125,6 +125,7 @@ class RegistrationHandler(BaseHandler): generate_token=True, guest_access_token=None, make_guest=False, + display_name=None, admin=False, threepid=None, ): @@ -141,6 +142,7 @@ class RegistrationHandler(BaseHandler): since it offers no means of associating a device_id with the access_token. Instead you should call auth_handler.issue_access_token after registration. + display_name (str): The displayname to set for this user, if any Returns: A tuple of (user_id, access_token). Raises: @@ -179,13 +181,20 @@ class RegistrationHandler(BaseHandler): password_hash=password_hash, was_guest=was_guest, make_guest=make_guest, - create_profile_with_localpart=( - # If the user was a guest then they already have a profile - None if was_guest else user.localpart - ), admin=admin, ) + if display_name is None: + display_name = ( + # If the user was a guest then they already have a profile + None if was_guest else user.localpart + ) + + if display_name: + yield self.profile_handler.set_displayname( + user, None, display_name, by_admin=True, + ) + if self.hs.config.user_directory_search_all_users: profile = yield self.store.get_profileinfo(localpart) yield self.user_directory_handler.handle_local_profile_change( @@ -210,8 +219,12 @@ class RegistrationHandler(BaseHandler): token=token, password_hash=password_hash, make_guest=make_guest, - create_profile_with_localpart=user.localpart, ) + + yield self.profile_handler.set_displayname( + user, None, user.localpart, by_admin=True, + ) + except SynapseError: # if user id is taken, just generate another user = None @@ -282,8 +295,12 @@ class RegistrationHandler(BaseHandler): user_id=user_id, password_hash="", appservice_id=service_id, - create_profile_with_localpart=user.localpart, ) + + yield self.profile_handler.set_displayname( + user, None, user.localpart, by_admin=True, + ) + defer.returnValue(user_id) @defer.inlineCallbacks @@ -330,7 +347,10 @@ class RegistrationHandler(BaseHandler): user_id=user_id, token=token, password_hash=None, - create_profile_with_localpart=user.localpart, + ) + + yield self.profile_handler.set_displayname( + user, None, user.localpart, by_admin=True, ) except Exception as e: yield self.store.add_access_token_to_user(user_id, token) @@ -361,7 +381,9 @@ class RegistrationHandler(BaseHandler): logger.info("got threepid with medium '%s' and address '%s'", threepid['medium'], threepid['address']) - if not check_3pid_allowed(self.hs, threepid['medium'], threepid['address']): + if not ( + yield check_3pid_allowed(self.hs, threepid['medium'], threepid['address']) + ): raise RegistrationError( 403, "Third party identifier is not allowed" ) @@ -489,18 +511,15 @@ class RegistrationHandler(BaseHandler): user_id=user_id, token=token, password_hash=password_hash, - create_profile_with_localpart=user.localpart, ) + if displayname is not None: + yield self.profile_handler.set_displayname( + user, None, displayname, by_admin=True, + ) else: yield self._auth_handler.delete_access_tokens_for_user(user_id) yield self.store.add_access_token_to_user(user_id=user_id, token=token) - if displayname is not None: - logger.info("setting user display name: %s -> %s", user_id, displayname) - yield self.profile_handler.set_displayname( - user, requester, displayname, by_admin=True, - ) - defer.returnValue((user_id, token)) def auth_handler(self): diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 3ba92bdb4c..3870e433ec 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py
@@ -53,12 +53,14 @@ class RoomCreationHandler(BaseHandler): "history_visibility": "shared", "original_invitees_have_ops": False, "guest_can_join": True, + "encryption_alg": "m.megolm.v1.aes-sha2", }, RoomCreationPreset.TRUSTED_PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, "history_visibility": "shared", "original_invitees_have_ops": True, "guest_can_join": True, + "encryption_alg": "m.megolm.v1.aes-sha2", }, RoomCreationPreset.PUBLIC_CHAT: { "join_rules": JoinRules.PUBLIC, @@ -427,6 +429,15 @@ class RoomCreationHandler(BaseHandler): content=content, ) + if "encryption_alg" in config: + send( + etype=EventTypes.Encryption, + state_key="", + content={ + 'algorithm': config["encryption_alg"], + } + ) + class RoomContextHandler(object): def __init__(self, hs): diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 09739f2862..c802db9792 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py
@@ -928,7 +928,7 @@ class SyncHandler(object): res = yield self._generate_sync_entry_for_rooms( sync_result_builder, account_data_by_room ) - newly_joined_rooms, newly_joined_users, _, _ = res + newly_joined_rooms, newly_joined_or_invited_users, _, _ = res _, _, newly_left_rooms, newly_left_users = res block_all_presence_data = ( @@ -937,7 +937,7 @@ class SyncHandler(object): ) if self.hs_config.use_presence and not block_all_presence_data: yield self._generate_sync_entry_for_presence( - sync_result_builder, newly_joined_rooms, newly_joined_users + sync_result_builder, newly_joined_rooms, newly_joined_or_invited_users ) yield self._generate_sync_entry_for_to_device(sync_result_builder) @@ -945,7 +945,7 @@ class SyncHandler(object): device_lists = yield self._generate_sync_entry_for_device_list( sync_result_builder, newly_joined_rooms=newly_joined_rooms, - newly_joined_users=newly_joined_users, + newly_joined_or_invited_users=newly_joined_or_invited_users, newly_left_rooms=newly_left_rooms, newly_left_users=newly_left_users, ) @@ -1021,7 +1021,8 @@ class SyncHandler(object): @measure_func("_generate_sync_entry_for_device_list") @defer.inlineCallbacks def _generate_sync_entry_for_device_list(self, sync_result_builder, - newly_joined_rooms, newly_joined_users, + newly_joined_rooms, + newly_joined_or_invited_users, newly_left_rooms, newly_left_users): user_id = sync_result_builder.sync_config.user.to_string() since_token = sync_result_builder.since_token @@ -1035,7 +1036,7 @@ class SyncHandler(object): # share a room with? for room_id in newly_joined_rooms: joined_users = yield self.state.get_current_user_in_room(room_id) - newly_joined_users.update(joined_users) + newly_joined_or_invited_users.update(joined_users) for room_id in newly_left_rooms: left_users = yield self.state.get_current_user_in_room(room_id) @@ -1043,7 +1044,7 @@ class SyncHandler(object): # TODO: Check that these users are actually new, i.e. either they # weren't in the previous sync *or* they left and rejoined. - changed.update(newly_joined_users) + changed.update(newly_joined_or_invited_users) if not changed and not newly_left_users: defer.returnValue(DeviceLists( @@ -1161,7 +1162,7 @@ class SyncHandler(object): @defer.inlineCallbacks def _generate_sync_entry_for_presence(self, sync_result_builder, newly_joined_rooms, - newly_joined_users): + newly_joined_or_invited_users): """Generates the presence portion of the sync response. Populates the `sync_result_builder` with the result. @@ -1169,8 +1170,9 @@ class SyncHandler(object): sync_result_builder(SyncResultBuilder) newly_joined_rooms(list): List of rooms that the user has joined since the last sync (or empty if an initial sync) - newly_joined_users(list): List of users that have joined rooms - since the last sync (or empty if an initial sync) + newly_joined_or_invited_users(list): List of users that have joined + or been invited to rooms since the last sync (or empty if an initial + sync) """ now_token = sync_result_builder.now_token sync_config = sync_result_builder.sync_config @@ -1196,7 +1198,7 @@ class SyncHandler(object): "presence_key", presence_key ) - extra_users_ids = set(newly_joined_users) + extra_users_ids = set(newly_joined_or_invited_users) for room_id in newly_joined_rooms: users = yield self.state.get_current_user_in_room(room_id) extra_users_ids.update(users) @@ -1228,7 +1230,8 @@ class SyncHandler(object): Returns: Deferred(tuple): Returns a 4-tuple of - `(newly_joined_rooms, newly_joined_users, newly_left_rooms, newly_left_users)` + `(newly_joined_rooms, newly_joined_or_invited_users, + newly_left_rooms, newly_left_users)` """ user_id = sync_result_builder.sync_config.user.to_string() block_all_room_ephemeral = ( @@ -1299,8 +1302,8 @@ class SyncHandler(object): sync_result_builder.invited.extend(invited) - # Now we want to get any newly joined users - newly_joined_users = set() + # Now we want to get any newly joined or invited users + newly_joined_or_invited_users = set() newly_left_users = set() if since_token: for joined_sync in sync_result_builder.joined: @@ -1309,19 +1312,22 @@ class SyncHandler(object): ) for event in it: if event.type == EventTypes.Member: - if event.membership == Membership.JOIN: - newly_joined_users.add(event.state_key) + if ( + event.membership == Membership.JOIN or + event.membership == Membership.INVITE + ): + newly_joined_or_invited_users.add(event.state_key) else: prev_content = event.unsigned.get("prev_content", {}) prev_membership = prev_content.get("membership", None) if prev_membership == Membership.JOIN: newly_left_users.add(event.state_key) - newly_left_users -= newly_joined_users + newly_left_users -= newly_joined_or_invited_users defer.returnValue(( newly_joined_rooms, - newly_joined_users, + newly_joined_or_invited_users, newly_left_rooms, newly_left_users, )) @@ -1366,7 +1372,7 @@ class SyncHandler(object): where: room_entries is a list [RoomSyncResultBuilder] invited_rooms is a list [InvitedSyncResult] - newly_joined rooms is a list[str] of room ids + newly_joined_rooms is a list[str] of room ids newly_left_rooms is a list[str] of room ids """ user_id = sync_result_builder.sync_config.user.to_string() @@ -1401,7 +1407,7 @@ class SyncHandler(object): if room_id in sync_result_builder.joined_room_ids and non_joins: # Always include if the user (re)joined the room, especially # important so that device list changes are calculated correctly. - # If there are non join member events, but we are still in the room, + # If there are non-join member events, but we are still in the room, # then the user must have left and joined newly_joined_rooms.append(room_id) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 37b32dd37b..ea84729915 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py
@@ -51,7 +51,7 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): 'id_server', 'client_secret', 'email', 'send_attempt' ]) - if not check_3pid_allowed(self.hs, "email", body['email']): + if not (yield check_3pid_allowed(self.hs, "email", body['email'])): raise SynapseError( 403, "Your email domain is not authorized on this server", @@ -89,7 +89,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - if not check_3pid_allowed(self.hs, "msisdn", msisdn): + if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)): raise SynapseError( 403, "Account phone numbers are not authorized on this server", @@ -243,7 +243,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): ['id_server', 'client_secret', 'email', 'send_attempt'], ) - if not check_3pid_allowed(self.hs, "email", body['email']): + if not (yield check_3pid_allowed(self.hs, "email", body['email'])): raise SynapseError( 403, "Your email domain is not authorized on this server", @@ -280,7 +280,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - if not check_3pid_allowed(self.hs, "msisdn", msisdn): + if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)): raise SynapseError( 403, "Account phone numbers are not authorized on this server", @@ -321,6 +321,9 @@ class ThreepidRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): + if self.hs.config.disable_3pid_changes: + raise SynapseError(400, "3PID changes disabled on this server") + body = parse_json_object_from_request(request) threePidCreds = body.get('threePidCreds') @@ -367,11 +370,15 @@ class ThreepidDeleteRestServlet(RestServlet): def __init__(self, hs): super(ThreepidDeleteRestServlet, self).__init__() + self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() @defer.inlineCallbacks def on_POST(self, request): + if self.hs.config.disable_3pid_changes: + raise SynapseError(400, "3PID changes disabled on this server") + body = parse_json_object_from_request(request) assert_params_in_dict(body, ['medium', 'address']) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 192f52e462..c7c8287882 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py
@@ -16,7 +16,9 @@ import hmac import logging +import re from hashlib import sha1 +from string import capwords from six import string_types @@ -73,7 +75,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): 'id_server', 'client_secret', 'email', 'send_attempt' ]) - if not check_3pid_allowed(self.hs, "email", body['email']): + if not (yield check_3pid_allowed(self.hs, "email", body['email'])): raise SynapseError( 403, "Your email domain is not authorized to register on this server", @@ -115,7 +117,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - if not check_3pid_allowed(self.hs, "msisdn", msisdn): + if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)): raise SynapseError( 403, "Phone numbers are not authorized to register on this server", @@ -227,6 +229,8 @@ class RegisterRestServlet(RestServlet): raise SynapseError(400, "Invalid username") desired_username = body['username'] + desired_display_name = None + appservice = None if self.auth.has_access_token(request): appservice = yield self.auth.get_appservice_by_req(request) @@ -302,13 +306,6 @@ class RegisterRestServlet(RestServlet): session_id, "registered_user_id", None ) - if desired_username is not None: - yield self.registration_handler.check_username( - desired_username, - guest_access_token=guest_access_token, - assigned_user_id=registered_user_id, - ) - # Only give msisdn flows if the x_show_msisdn flag is given: # this is a hack to work around the fact that clients were shipped # that use fallback registration if they see any flows that they don't @@ -375,7 +372,7 @@ class RegisterRestServlet(RestServlet): medium = auth_result[login_type]['medium'] address = auth_result[login_type]['address'] - if not check_3pid_allowed(self.hs, medium, address): + if not (yield check_3pid_allowed(self.hs, medium, address)): raise SynapseError( 403, "Third party identifiers (email/phone numbers)" + @@ -383,6 +380,81 @@ class RegisterRestServlet(RestServlet): Codes.THREEPID_DENIED, ) + if self.hs.config.register_mxid_from_3pid: + # override the desired_username based on the 3PID if any. + # reset it first to avoid folks picking their own username. + desired_username = None + + # we should have an auth_result at this point if we're going to progress + # to register the user (i.e. we haven't picked up a registered_user_id + # from our session store), in which case get ready and gen the + # desired_username + if auth_result: + if ( + self.hs.config.register_mxid_from_3pid == 'email' and + LoginType.EMAIL_IDENTITY in auth_result + ): + address = auth_result[LoginType.EMAIL_IDENTITY]['address'] + desired_username = synapse.types.strip_invalid_mxid_characters( + address.replace('@', '-').lower() + ) + + # find a unique mxid for the account, suffixing numbers + # if needed + while True: + try: + yield self.registration_handler.check_username( + desired_username, + guest_access_token=guest_access_token, + assigned_user_id=registered_user_id, + ) + # if we got this far we passed the check. + break + except SynapseError as e: + if e.errcode == Codes.USER_IN_USE: + m = re.match(r'^(.*?)(\d+)$', desired_username) + if m: + desired_username = m.group(1) + str( + int(m.group(2)) + 1 + ) + else: + desired_username += "1" + else: + # something else went wrong. + break + + # XXX: a nasty heuristic to turn an email address into + # a displayname, as part of register_mxid_from_3pid + parts = address.replace('.', ' ').split('@') + org_parts = parts[1].split(' ') + + if org_parts[-2] == "matrix" and org_parts[-1] == "org": + org = "Tchap Admin" + elif org_parts[-2] == "gouv" and org_parts[-1] == "fr": + org = org_parts[-3] if len(org_parts) > 2 else org_parts[-2] + else: + org = org_parts[-2] + + desired_display_name = ( + capwords(parts[0]) + " [" + capwords(org) + "]" + ) + elif ( + self.hs.config.register_mxid_from_3pid == 'msisdn' and + LoginType.MSISDN in auth_result + ): + desired_username = auth_result[LoginType.MSISDN]['address'] + else: + raise SynapseError( + 400, "Cannot derive mxid from 3pid; no recognised 3pid" + ) + + if desired_username is not None: + yield self.registration_handler.check_username( + desired_username, + guest_access_token=guest_access_token, + assigned_user_id=registered_user_id, + ) + if registered_user_id is not None: logger.info( "Already registered user ID %r for this session", @@ -395,10 +467,18 @@ class RegisterRestServlet(RestServlet): # NB: This may be from the auth handler and NOT from the POST assert_params_in_dict(params, ["password"]) - desired_username = params.get("username", None) + if not self.hs.config.register_mxid_from_3pid: + desired_username = params.get("username", None) + else: + # we keep the original desired_username derived from the 3pid above + pass + guest_access_token = params.get("guest_access_token", None) new_password = params.get("password", None) + # XXX: don't we need to validate these for length etc like we did on + # the ones from the JSON body earlier on in the method? + if desired_username is not None: desired_username = desired_username.lower() @@ -411,6 +491,7 @@ class RegisterRestServlet(RestServlet): password=new_password, guest_access_token=guest_access_token, generate_token=False, + display_name=desired_display_name, threepid=threepid, ) # Necessary due to auth checks prior to the threepid being diff --git a/synapse/rest/client/v2_alpha/user_directory.py b/synapse/rest/client/v2_alpha/user_directory.py
index cac0624ba7..8799dc3efe 100644 --- a/synapse/rest/client/v2_alpha/user_directory.py +++ b/synapse/rest/client/v2_alpha/user_directory.py
@@ -15,6 +15,8 @@ import logging +from signedjson.sign import sign_json + from twisted.internet import defer from synapse.api.errors import SynapseError @@ -37,6 +39,7 @@ class UserDirectorySearchRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() self.user_directory_handler = hs.get_user_directory_handler() + self.http_client = hs.get_simple_http_client() @defer.inlineCallbacks def on_POST(self, request): @@ -61,6 +64,14 @@ class UserDirectorySearchRestServlet(RestServlet): body = parse_json_object_from_request(request) + if self.hs.config.user_directory_defer_to_id_server: + signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0]) + url = "%s/_matrix/identity/api/v1/user_directory/search" % ( + self.hs.config.user_directory_defer_to_id_server, + ) + resp = yield self.http_client.post_json_get_json(url, signed_body) + defer.returnValue((200, resp)) + limit = body.get("limit", 10) limit = min(limit, 50) diff --git a/synapse/rulecheck/__init__.py b/synapse/rulecheck/__init__.py new file mode 100644
index 0000000000..e69de29bb2 --- /dev/null +++ b/synapse/rulecheck/__init__.py
diff --git a/synapse/rulecheck/domain_rule_checker.py b/synapse/rulecheck/domain_rule_checker.py new file mode 100644
index 0000000000..3caa6b34cb --- /dev/null +++ b/synapse/rulecheck/domain_rule_checker.py
@@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from synapse.config._base import ConfigError + +logger = logging.getLogger(__name__) + + +class DomainRuleChecker(object): + """ + A re-implementation of the SpamChecker that prevents users in one domain from + inviting users in other domains to rooms, based on a configuration. + + Takes a config in the format: + + spam_checker: + module: "rulecheck.DomainRuleChecker" + config: + domain_mapping: + "inviter_domain": [ "invitee_domain_permitted", "other_domain_permitted" ] + "other_inviter_domain": [ "invitee_domain_permitted" ] + default: False + } + + Don't forget to consider if you can invite users from your own domain. + """ + + def __init__(self, config): + self.domain_mapping = config["domain_mapping"] or {} + self.default = config["default"] + + def check_event_for_spam(self, event): + """Implements synapse.events.SpamChecker.check_event_for_spam + """ + return False + + def user_may_invite(self, inviter_userid, invitee_userid, room_id): + """Implements synapse.events.SpamChecker.user_may_invite + """ + inviter_domain = self._get_domain_from_id(inviter_userid) + invitee_domain = self._get_domain_from_id(invitee_userid) + + if inviter_domain not in self.domain_mapping: + return self.default + + return invitee_domain in self.domain_mapping[inviter_domain] + + def user_may_create_room(self, userid): + """Implements synapse.events.SpamChecker.user_may_create_room + """ + return True + + def user_may_create_room_alias(self, userid, room_alias): + """Implements synapse.events.SpamChecker.user_may_create_room_alias + """ + return True + + def user_may_publish_room(self, userid, room_id): + """Implements synapse.events.SpamChecker.user_may_publish_room + """ + return True + + @staticmethod + def parse_config(config): + """Implements synapse.events.SpamChecker.parse_config + """ + if "default" in config: + return config + else: + raise ConfigError("No default set for spam_config DomainRuleChecker") + + @staticmethod + def _get_domain_from_id(mxid): + """Parses a string and returns the domain part of the mxid. + + Args: + mxid (str): a valid mxid + + Returns: + str: the domain part of the mxid + + """ + idx = mxid.find(":") + if idx == -1: + raise Exception("Invalid ID: %r" % (mxid,)) + return mxid[idx + 1:] diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index 88b50f33b5..10133f0a4a 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py
@@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +21,8 @@ from synapse.storage.roommember import ProfileInfo from ._base import SQLBaseStore +BATCH_SIZE = 100 + class ProfileWorkerStore(SQLBaseStore): @defer.inlineCallbacks @@ -62,6 +65,55 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_profile_avatar_url", ) + def get_latest_profile_replication_batch_number(self): + def f(txn): + txn.execute("SELECT MAX(batch) as maxbatch FROM profiles") + rows = self.cursor_to_dict(txn) + return rows[0]['maxbatch'] + return self.runInteraction( + "get_latest_profile_replication_batch_number", f, + ) + + def get_profile_batch(self, batchnum): + return self._simple_select_list( + table="profiles", + keyvalues={ + "batch": batchnum, + }, + retcols=("user_id", "displayname", "avatar_url", "active"), + desc="get_profile_batch", + ) + + def assign_profile_batch(self): + def f(txn): + sql = ( + "UPDATE profiles SET batch = " + "(SELECT COALESCE(MAX(batch), -1) + 1 FROM profiles) " + "WHERE user_id in (" + " SELECT user_id FROM profiles WHERE batch is NULL limit ?" + ")" + ) + txn.execute(sql, (BATCH_SIZE,)) + return txn.rowcount + return self.runInteraction("assign_profile_batch", f) + + def get_replication_hosts(self): + def f(txn): + txn.execute("SELECT host, last_synced_batch FROM profile_replication_status") + rows = self.cursor_to_dict(txn) + return {r['host']: r['last_synced_batch'] for r in rows} + return self.runInteraction("get_replication_hosts", f) + + def update_replication_batch_for_host(self, host, last_synced_batch): + return self._simple_upsert( + table="profile_replication_status", + keyvalues={"host": host}, + values={ + "last_synced_batch": last_synced_batch, + }, + desc="update_replication_batch_for_host", + ) + def get_from_remote_profile_cache(self, user_id): return self._simple_select_one( table="remote_profile_cache", @@ -71,27 +123,44 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_from_remote_profile_cache", ) - def create_profile(self, user_localpart): - return self._simple_insert( + def set_profile_displayname(self, user_localpart, new_displayname, batchnum): + return self._simple_upsert( table="profiles", - values={"user_id": user_localpart}, - desc="create_profile", + keyvalues={"user_id": user_localpart}, + values={ + "displayname": new_displayname, + "batch": batchnum, + }, + desc="set_profile_displayname", + lock=False # we can do this because user_id has a unique index ) - def set_profile_displayname(self, user_localpart, new_displayname): - return self._simple_update_one( + def set_profile_avatar_url(self, user_localpart, new_avatar_url, batchnum): + return self._simple_upsert( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname}, - desc="set_profile_displayname", + values={ + "avatar_url": new_avatar_url, + "batch": batchnum, + }, + desc="set_profile_avatar_url", + lock=False # we can do this because user_id has a unique index ) - def set_profile_avatar_url(self, user_localpart, new_avatar_url): - return self._simple_update_one( + def set_profile_active(self, user_localpart, active, batchnum): + values = { + "active": int(active), + "batch": batchnum, + } + if not active: + values["avatar_url"] = None + values["displayname"] = None + return self._simple_upsert( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"avatar_url": new_avatar_url}, - desc="set_profile_avatar_url", + values=values, + desc="set_profile_active", + lock=False # we can do this because user_id has a unique index ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 80d76bf9d7..65061f4c61 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py
@@ -167,7 +167,7 @@ class RegistrationStore(RegistrationWorkerStore, def register(self, user_id, token=None, password_hash=None, was_guest=False, make_guest=False, appservice_id=None, - create_profile_with_localpart=None, admin=False): + admin=False): """Attempts to register an account. Args: @@ -181,8 +181,6 @@ class RegistrationStore(RegistrationWorkerStore, make_guest (boolean): True if the the new user should be guest, false to add a regular user account. appservice_id (str): The ID of the appservice registering the user. - create_profile_with_localpart (str): Optionally create a profile for - the given localpart. Raises: StoreError if the user_id could not be registered. """ @@ -195,7 +193,6 @@ class RegistrationStore(RegistrationWorkerStore, was_guest, make_guest, appservice_id, - create_profile_with_localpart, admin ) @@ -208,7 +205,6 @@ class RegistrationStore(RegistrationWorkerStore, was_guest, make_guest, appservice_id, - create_profile_with_localpart, admin, ): now = int(self.clock.time()) @@ -273,14 +269,6 @@ class RegistrationStore(RegistrationWorkerStore, (next_id, user_id, token,) ) - if create_profile_with_localpart: - # set a default displayname serverside to avoid ugly race - # between auto-joins and clients trying to set displaynames - txn.execute( - "INSERT INTO profiles(user_id, displayname) VALUES (?,?)", - (create_profile_with_localpart, create_profile_with_localpart) - ) - self._invalidate_cache_and_stream( txn, self.get_user_by_id, (user_id,) ) diff --git a/synapse/storage/schema/delta/48/profiles_batch.sql b/synapse/storage/schema/delta/48/profiles_batch.sql new file mode 100644
index 0000000000..e744c02fe8 --- /dev/null +++ b/synapse/storage/schema/delta/48/profiles_batch.sql
@@ -0,0 +1,36 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Add a batch number to track changes to profiles and the + * order they're made in so we can replicate user profiles + * to other hosts as they change + */ +ALTER TABLE profiles ADD COLUMN batch BIGINT DEFAULT NULL; + +/* + * Index on the batch number so we can get profiles + * by their batch + */ +CREATE INDEX profiles_batch_idx ON profiles(batch); + +/* + * A table to track what batch of user profiles has been + * synced to what profile replication target. + */ +CREATE TABLE profile_replication_status ( + host TEXT NOT NULL, + last_synced_batch BIGINT NOT NULL +); diff --git a/synapse/storage/schema/delta/50/profiles_deactivated_users.sql b/synapse/storage/schema/delta/50/profiles_deactivated_users.sql new file mode 100644
index 0000000000..c8893ecbe8 --- /dev/null +++ b/synapse/storage/schema/delta/50/profiles_deactivated_users.sql
@@ -0,0 +1,23 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * A flag saying whether the user owning the profile has been deactivated + * This really belongs on the users table, not here, but the users table + * stores users by their full user_id and profiles stores them by localpart, + * so we can't easily join between the two tables. Plus, the batch number + * realy ought to represent data in this table that has changed. + */ +ALTER TABLE profiles ADD COLUMN active SMALLINT DEFAULT 1 NOT NULL; diff --git a/synapse/types.py b/synapse/types.py
index 41afb27a74..8a77eda357 100644 --- a/synapse/types.py +++ b/synapse/types.py
@@ -228,6 +228,18 @@ def contains_invalid_mxid_characters(localpart): return any(c not in mxid_localpart_allowed_characters for c in localpart) +def strip_invalid_mxid_characters(localpart): + """Removes any invalid characters from an mxid + + Args: + localpart (basestring): the localpart to be stripped + + Returns: + localpart (basestring): the localpart having been stripped + """ + return filter(lambda c: c in mxid_localpart_allowed_characters, localpart) + + class StreamToken( namedtuple("Token", ( "room_key", diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py
index 75efa0117b..353d220bad 100644 --- a/synapse/util/threepids.py +++ b/synapse/util/threepids.py
@@ -16,9 +16,12 @@ import logging import re +from twisted.internet import defer + logger = logging.getLogger(__name__) +@defer.inlineCallbacks def check_3pid_allowed(hs, medium, address): """Checks whether a given format of 3PID is allowed to be used on this HS @@ -28,9 +31,22 @@ def check_3pid_allowed(hs, medium, address): address (str): address within that medium (e.g. "wotan@matrix.org") msisdns need to first have been canonicalised Returns: - bool: whether the 3PID medium/address is allowed to be added to this HS + defered bool: whether the 3PID medium/address is allowed to be added to this HS """ + if hs.config.check_is_for_allowed_local_3pids: + data = yield hs.get_simple_http_client().get_json( + "https://%s%s" % ( + hs.config.check_is_for_allowed_local_3pids, + "/_matrix/identity/api/v1/info" + ), + {'medium': medium, 'address': address} + ) + if hs.config.allow_invited_3pids and data.get('invited'): + defer.returnValue(True) + else: + defer.returnValue(data['hs'] == hs.config.server_name) + if hs.config.allowed_local_3pids: for constraint in hs.config.allowed_local_3pids: logger.debug( @@ -41,8 +57,8 @@ def check_3pid_allowed(hs, medium, address): medium == constraint['medium'] and re.match(constraint['pattern'], address) ): - return True + defer.returnValue(True) else: - return True + defer.returnValue(True) - return False + defer.returnValue(False)