diff options
32 files changed, 1114 insertions, 102 deletions
diff --git a/res/templates-dinsic/mail-Vector.css b/res/templates-dinsic/mail-Vector.css new file mode 100644 index 0000000000..6a3e36eda1 --- /dev/null +++ b/res/templates-dinsic/mail-Vector.css @@ -0,0 +1,7 @@ +.header { + border-bottom: 4px solid #e4f7ed ! important; +} + +.notif_link a, .footer a { + color: #76CFA6 ! important; +} diff --git a/res/templates-dinsic/mail.css b/res/templates-dinsic/mail.css new file mode 100644 index 0000000000..5ab3e1b06d --- /dev/null +++ b/res/templates-dinsic/mail.css @@ -0,0 +1,156 @@ +body { + margin: 0px; +} + +pre, code { + word-break: break-word; + white-space: pre-wrap; +} + +#page { + font-family: 'Open Sans', Helvetica, Arial, Sans-Serif; + font-color: #454545; + font-size: 12pt; + width: 100%; + padding: 20px; +} + +#inner { + width: 640px; +} + +.header { + width: 100%; + height: 87px; + color: #454545; + border-bottom: 4px solid #e5e5e5; +} + +.logo { + text-align: right; + margin-left: 20px; +} + +.salutation { + padding-top: 10px; + font-weight: bold; +} + +.summarytext { +} + +.room { + width: 100%; + color: #454545; + border-bottom: 1px solid #e5e5e5; +} + +.room_header td { + padding-top: 38px; + padding-bottom: 10px; + border-bottom: 1px solid #e5e5e5; +} + +.room_name { + vertical-align: middle; + font-size: 18px; + font-weight: bold; +} + +.room_header h2 { + margin-top: 0px; + margin-left: 75px; + font-size: 20px; +} + +.room_avatar { + width: 56px; + line-height: 0px; + text-align: center; + vertical-align: middle; +} + +.room_avatar img { + width: 48px; + height: 48px; + object-fit: cover; + border-radius: 24px; +} + +.notif { + border-bottom: 1px solid #e5e5e5; + margin-top: 16px; + padding-bottom: 16px; +} + +.historical_message .sender_avatar { + opacity: 0.3; +} + +/* spell out opacity and historical_message class names for Outlook aka Word */ +.historical_message .sender_name { + color: #e3e3e3; +} + +.historical_message .message_time { + color: #e3e3e3; +} + +.historical_message .message_body { + color: #c7c7c7; +} + +.historical_message td, +.message td { + padding-top: 10px; +} + +.sender_avatar { + width: 56px; + text-align: center; + vertical-align: top; +} + +.sender_avatar img { + margin-top: -2px; + width: 32px; + height: 32px; + border-radius: 16px; +} + +.sender_name { + display: inline; + font-size: 13px; + color: #a2a2a2; +} + +.message_time { + text-align: right; + width: 100px; + font-size: 11px; + color: #a2a2a2; +} + +.message_body { +} + +.notif_link td { + padding-top: 10px; + padding-bottom: 10px; + font-weight: bold; +} + +.notif_link a, .footer a { + color: #454545; + text-decoration: none; +} + +.debug { + font-size: 10px; + color: #888; +} + +.footer { + margin-top: 20px; + text-align: center; +} \ No newline at end of file diff --git a/res/templates-dinsic/notif.html b/res/templates-dinsic/notif.html new file mode 100644 index 0000000000..bcdfeea9da --- /dev/null +++ b/res/templates-dinsic/notif.html @@ -0,0 +1,45 @@ +{% for message in notif.messages %} + <tr class="{{ "historical_message" if message.is_historical else "message" }}"> + <td class="sender_avatar"> + {% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %} + {% if message.sender_avatar_url %} + <img alt="" class="sender_avatar" src="{{ message.sender_avatar_url|mxc_to_http(32,32) }}" /> + {% else %} + {% if message.sender_hash % 3 == 0 %} + <img class="sender_avatar" src="https://vector.im/beta/img/76cfa6.png" /> + {% elif message.sender_hash % 3 == 1 %} + <img class="sender_avatar" src="https://vector.im/beta/img/50e2c2.png" /> + {% else %} + <img class="sender_avatar" src="https://vector.im/beta/img/f4c371.png" /> + {% endif %} + {% endif %} + {% endif %} + </td> + <td class="message_contents"> + {% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %} + <div class="sender_name">{% if message.msgtype == "m.emote" %}*{% endif %} {{ message.sender_name }}</div> + {% endif %} + <div class="message_body"> + {% if message.msgtype == "m.text" %} + {{ message.body_text_html }} + {% elif message.msgtype == "m.emote" %} + {{ message.body_text_html }} + {% elif message.msgtype == "m.notice" %} + {{ message.body_text_html }} + {% elif message.msgtype == "m.image" %} + <img src="{{ message.image_url|mxc_to_http(640, 480, scale) }}" /> + {% elif message.msgtype == "m.file" %} + <span class="filename">{{ message.body_text_plain }}</span> + {% endif %} + </div> + </td> + <td class="message_time">{{ message.ts|format_ts("%H:%M") }}</td> + </tr> +{% endfor %} +<tr class="notif_link"> + <td></td> + <td> + <a href="{{ notif.link }}">Voir {{ room.title }}</a> + </td> + <td></td> +</tr> diff --git a/res/templates-dinsic/notif.txt b/res/templates-dinsic/notif.txt new file mode 100644 index 0000000000..3dff1bb570 --- /dev/null +++ b/res/templates-dinsic/notif.txt @@ -0,0 +1,16 @@ +{% for message in notif.messages %} +{% if message.msgtype == "m.emote" %}* {% endif %}{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }}) +{% if message.msgtype == "m.text" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.emote" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.notice" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.image" %} +{{ message.body_text_plain }} +{% elif message.msgtype == "m.file" %} +{{ message.body_text_plain }} +{% endif %} +{% endfor %} + +Voir {{ room.title }} à {{ notif.link }} diff --git a/res/templates-dinsic/notif_mail.html b/res/templates-dinsic/notif_mail.html new file mode 100644 index 0000000000..1e1efa74b2 --- /dev/null +++ b/res/templates-dinsic/notif_mail.html @@ -0,0 +1,55 @@ +<!doctype html> +<html lang="en"> + <head> + <style type="text/css"> + {% include 'mail.css' without context %} + {% include "mail-%s.css" % app_name ignore missing without context %} + </style> + </head> + <body> + <table id="page"> + <tr> + <td> </td> + <td id="inner"> + <table class="header"> + <tr> + <td> + <div class="salutation">Bonjour {{ user_display_name }},</div> + <div class="summarytext">{{ summary_text }}</div> + </td> + <td class="logo"> + {% if app_name == "Riot" %} + <img src="http://matrix.org/img/riot-logo-email.png" width="83" height="83" alt="[Riot]"/> + {% elif app_name == "Vector" %} + <img src="http://matrix.org/img/vector-logo-email.png" width="64" height="83" alt="[Vector]"/> + {% else %} + <img src="http://matrix.org/img/matrix-120x51.png" width="120" height="51" alt="[matrix]"/> + {% endif %} + </td> + </tr> + </table> + {% for room in rooms %} + {% include 'room.html' with context %} + {% endfor %} + <div class="footer"> + <a href="{{ unsubscribe_link }}">Se désinscrire</a> + <br/> + <br/> + <div class="debug"> + Sending email at {{ reason.now|format_ts("%c") }} due to activity in room {{ reason.room_name }} because + an event was received at {{ reason.received_at|format_ts("%c") }} + which is more than {{ "%.1f"|format(reason.delay_before_mail_ms / (60*1000)) }} ({{ reason.delay_before_mail_ms }}) mins ago, + {% if reason.last_sent_ts %} + and the last time we sent a mail for this room was {{ reason.last_sent_ts|format_ts("%c") }}, + which is more than {{ "%.1f"|format(reason.throttle_ms / (60*1000)) }} (current throttle_ms) mins ago. + {% else %} + and we don't have a last time we sent a mail for this room. + {% endif %} + </div> + </div> + </td> + <td> </td> + </tr> + </table> + </body> +</html> diff --git a/res/templates-dinsic/notif_mail.txt b/res/templates-dinsic/notif_mail.txt new file mode 100644 index 0000000000..fae877426f --- /dev/null +++ b/res/templates-dinsic/notif_mail.txt @@ -0,0 +1,10 @@ +Bonjour {{ user_display_name }}, + +{{ summary_text }} + +{% for room in rooms %} +{% include 'room.txt' with context %} +{% endfor %} + +Vous pouvez désactiver ces notifications en cliquant ici {{ unsubscribe_link }} + diff --git a/res/templates-dinsic/room.html b/res/templates-dinsic/room.html new file mode 100644 index 0000000000..0487b1b11c --- /dev/null +++ b/res/templates-dinsic/room.html @@ -0,0 +1,33 @@ +<table class="room"> + <tr class="room_header"> + <td class="room_avatar"> + {% if room.avatar_url %} + <img alt="" src="{{ room.avatar_url|mxc_to_http(48,48) }}" /> + {% else %} + {% if room.hash % 3 == 0 %} + <img alt="" src="https://vector.im/beta/img/76cfa6.png" /> + {% elif room.hash % 3 == 1 %} + <img alt="" src="https://vector.im/beta/img/50e2c2.png" /> + {% else %} + <img alt="" src="https://vector.im/beta/img/f4c371.png" /> + {% endif %} + {% endif %} + </td> + <td class="room_name" colspan="2"> + {{ room.title }} + </td> + </tr> + {% if room.invite %} + <tr> + <td></td> + <td> + <a href="{{ room.link }}">Rejoindre la conversation.</a> + </td> + <td></td> + </tr> + {% else %} + {% for notif in room.notifs %} + {% include 'notif.html' with context %} + {% endfor %} + {% endif %} +</table> diff --git a/res/templates-dinsic/room.txt b/res/templates-dinsic/room.txt new file mode 100644 index 0000000000..dd36d01d21 --- /dev/null +++ b/res/templates-dinsic/room.txt @@ -0,0 +1,9 @@ +{{ room.title }} + +{% if room.invite %} +  Vous avez été invité, rejoignez la conversation en cliquant sur le lien suivant {{ room.link }} +{% else %} + {% for notif in room.notifs %} + {% include 'notif.txt' with context %} + {% endfor %} +{% endif %} diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 4df930c8d1..00aa622b34 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -70,6 +71,7 @@ class EventTypes(object): CanonicalAlias = "m.room.canonical_alias" RoomAvatar = "m.room.avatar" GuestAccess = "m.room.guest_access" + Encryption = "m.room.encryption" # These are used for validation Message = "m.room.message" diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 0fb964eb67..685c78bc7f 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -33,7 +33,15 @@ class RegistrationConfig(Config): self.registrations_require_3pid = config.get("registrations_require_3pid", []) self.allowed_local_3pids = config.get("allowed_local_3pids", []) + self.check_is_for_allowed_local_3pids = config.get( + "check_is_for_allowed_local_3pids", None + ) + self.allow_invited_3pids = config.get("allow_invited_3pids", False) + + self.disable_3pid_changes = config.get("disable_3pid_changes", False) + self.registration_shared_secret = config.get("registration_shared_secret") + self.register_mxid_from_3pid = config.get("register_mxid_from_3pid") self.bcrypt_rounds = config.get("bcrypt_rounds", 12) self.trusted_third_party_id_servers = config["trusted_third_party_id_servers"] @@ -45,6 +53,13 @@ class RegistrationConfig(Config): self.auto_join_rooms = config.get("auto_join_rooms", []) + self.disable_set_displayname = config.get("disable_set_displayname", False) + self.disable_set_avatar_url = config.get("disable_set_avatar_url", False) + + self.replicate_user_profiles_to = config.get("replicate_user_profiles_to", []) + if not isinstance(self.replicate_user_profiles_to, list): + self.replicate_user_profiles_to = [self.replicate_user_profiles_to, ] + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) @@ -60,9 +75,26 @@ class RegistrationConfig(Config): # - email # - msisdn + # Derive the user's matrix ID from a type of 3PID used when registering. + # This overrides any matrix ID the user proposes when calling /register + # The 3PID type should be present in registrations_require_3pid to avoid + # users failing to register if they don't specify the right kind of 3pid. + # + # register_mxid_from_3pid: email + # Mandate that users are only allowed to associate certain formats of # 3PIDs with accounts on this server. # + # Use an Identity Server to establish which 3PIDs are allowed to register? + # Overrides allowed_local_3pids below. + # check_is_for_allowed_local_3pids: matrix.org + # + # If you are using an IS you can also check whether that IS registers + # pending invites for the given 3PID (and then allow it to sign up on + # the platform): + # + # allow_invited_3pids: False + # # allowed_local_3pids: # - medium: email # pattern: ".*@matrix\\.org" @@ -71,6 +103,11 @@ class RegistrationConfig(Config): # - medium: msisdn # pattern: "\\+44" + # If true, stop users from trying to change the 3PIDs associated with + # their accounts. + # + # disable_3pid_changes: False + # If set, allows registration by anyone who also has the shared # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" @@ -94,10 +131,25 @@ class RegistrationConfig(Config): - vector.im - riot.im + # If enabled, user IDs, display names and avatar URLs will be replicated + # to this server whenever they change. + # This is an experimental API currently implemented by sydent to support + # cross-homeserver user directories. + # replicate_user_profiles_to: example.com + + # If enabled, don't let users set their own display names/avatars + # other than for the very first time (unless they are a server admin). + # Useful when provisioning users based on the contents of a 3rd party + # directory and to avoid ambiguities. + # + # disable_set_displayname: False + # disable_set_avatar_url: False + # Users who register on this homeserver will automatically be joined # to these rooms #auto_join_rooms: # - "#example:example.com" + """ % locals() def add_arguments(self, parser): diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py index 38e8947843..9b0ae91289 100644 --- a/synapse/config/user_directory.py +++ b/synapse/config/user_directory.py @@ -23,11 +23,15 @@ class UserDirectoryConfig(Config): def read_config(self, config): self.user_directory_search_all_users = False + self.user_directory_defer_to_id_server = None user_directory_config = config.get("user_directory", None) if user_directory_config: self.user_directory_search_all_users = ( user_directory_config.get("search_all_users", False) ) + self.user_directory_defer_to_id_server = ( + user_directory_config.get("defer_to_id_server", None) + ) def default_config(self, config_dir_path, server_name, **kwargs): return """ @@ -41,4 +45,9 @@ class UserDirectoryConfig(Config): # #user_directory: # search_all_users: false + # + # If this is set, user search will be delegated to this ID server instead + # of synapse performing the search itself. + # This is an experimental API. + # defer_to_id_server: id.example.com """ diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index b3c5a9ee64..3a08208fd8 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -33,6 +33,7 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() self._identity_handler = hs.get_handlers().identity_handler + self._profile_handler = hs.get_profile_handler() self.user_directory_handler = hs.get_user_directory_handler() # Flag that indicates whether the process to part users from rooms is running @@ -87,6 +88,9 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_set_password_hash(user_id, None) + user = UserID.from_string(user_id) + yield self._profile_handler.set_active(user, False) + # Add the user to a table of users pending deactivation (ie. # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 859f6d2b2e..ff59957623 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,13 +16,16 @@ import logging -from twisted.internet import defer +from twisted.internet import defer, reactor from synapse.api.errors import AuthError, CodeMessageException, SynapseError from synapse.types import UserID, get_domain_from_id +from synapse.util.logcontext import run_in_background from ._base import BaseHandler +from signedjson.sign import sign_json + logger = logging.getLogger(__name__) @@ -29,6 +33,8 @@ class ProfileHandler(BaseHandler): PROFILE_UPDATE_MS = 60 * 1000 PROFILE_UPDATE_EVERY_MS = 24 * 60 * 60 * 1000 + PROFILE_REPLICATE_INTERVAL = 2 * 60 * 1000 + def __init__(self, hs): super(ProfileHandler, self).__init__(hs) @@ -39,11 +45,84 @@ class ProfileHandler(BaseHandler): self.user_directory_handler = hs.get_user_directory_handler() + self.http_client = hs.get_simple_http_client() + if hs.config.worker_app is None: self.clock.looping_call( self._update_remote_profile_cache, self.PROFILE_UPDATE_MS, ) + if len(self.hs.config.replicate_user_profiles_to) > 0: + reactor.callWhenRunning(self._assign_profile_replication_batches) + reactor.callWhenRunning(self._replicate_profiles) + # Add a looping call to replicate_profiles: this handles retries + # if the replication is unsuccessful when the user updated their + # profile. + self.clock.looping_call( + self._replicate_profiles, self.PROFILE_REPLICATE_INTERVAL + ) + + @defer.inlineCallbacks + def _assign_profile_replication_batches(self): + """If no profile replication has been done yet, allocate replication batch + numbers to each profile to start the replication process. + """ + logger.info("Assigning profile batch numbers...") + total = 0 + while True: + assigned = yield self.store.assign_profile_batch() + total += assigned + if assigned == 0: + break + logger.info("Assigned %d profile batch numbers", total) + + @defer.inlineCallbacks + def _replicate_profiles(self): + """If any profile data has been updated and not pushed to the replication targets, + replicate it. + """ + host_batches = yield self.store.get_replication_hosts() + latest_batch = yield self.store.get_latest_profile_replication_batch_number() + if latest_batch is None: + latest_batch = -1 + for repl_host in self.hs.config.replicate_user_profiles_to: + if repl_host not in host_batches: + host_batches[repl_host] = -1 + try: + for i in xrange(host_batches[repl_host] + 1, latest_batch + 1): + yield self._replicate_host_profile_batch(repl_host, i) + except Exception: + logger.exception( + "Exception while replicating to %s: aborting for now", repl_host, + ) + + @defer.inlineCallbacks + def _replicate_host_profile_batch(self, host, batchnum): + logger.info("Replicating profile batch %d to %s", batchnum, host) + batch_rows = yield self.store.get_profile_batch(batchnum) + batch = { + UserID(r["user_id"], self.hs.hostname).to_string(): ({ + "display_name": r["displayname"], + "avatar_url": r["avatar_url"], + } if r["active"] else None) for r in batch_rows + } + + url = "https://%s/_matrix/identity/api/v1/replicate_profiles" % (host,) + body = { + "batchnum": batchnum, + "batch": batch, + "origin_server": self.hs.hostname, + } + signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0]) + try: + yield self.http_client.post_json_get_json(url, signed_body) + yield self.store.update_replication_batch_for_host(host, batchnum) + logger.info("Sucessfully replicated profile batch %d to %s", batchnum, host) + except Exception: + # This will get retried when the looping call next comes around + logger.exception("Failed to replicate profile batch %d to %s", batchnum, host) + raise + @defer.inlineCallbacks def get_profile(self, user_id): target_user = UserID.from_string(user_id) @@ -130,19 +209,30 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def set_displayname(self, target_user, requester, new_displayname, by_admin=False): - """target_user is the user whose displayname is to be changed; - auth_user is the user attempting to make this change.""" + """target_user is the UserID whose displayname is to be changed; + requester is the authenticated user attempting to make this change.""" if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") - if not by_admin and target_user != requester.user: + if not by_admin and requester and target_user != requester.user: raise AuthError(400, "Cannot set another user's displayname") + if not by_admin and self.hs.config.disable_set_displayname: + profile = yield self.store.get_profileinfo(target_user.localpart) + if profile.display_name: + raise SynapseError(400, "Changing displayname is disabled on this server") + if new_displayname == '': new_displayname = None + if len(self.hs.config.replicate_user_profiles_to) > 0: + cur_batchnum = yield self.store.get_latest_profile_replication_batch_number() + new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1 + else: + new_batchnum = None + yield self.store.set_profile_displayname( - target_user.localpart, new_displayname + target_user.localpart, new_displayname, new_batchnum ) if self.hs.config.user_directory_search_all_users: @@ -151,7 +241,32 @@ class ProfileHandler(BaseHandler): target_user.to_string(), profile ) - yield self._update_join_states(requester, target_user) + if requester: + yield self._update_join_states(requester, target_user) + + # start a profile replication push + run_in_background(self._replicate_profiles) + + @defer.inlineCallbacks + def set_active(self, target_user, active): + """ + Sets the 'active' flag on a user profile. If set to false, the user account is + considered deactivated. + Note that unlike set_displayname and set_avatar_url, this does *not* perform + authorization checks! This is because the only place it's used currently is + in account deactivation where we've already done these checks anyway. + """ + if len(self.hs.config.replicate_user_profiles_to) > 0: + cur_batchnum = yield self.store.get_latest_profile_replication_batch_number() + new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1 + else: + new_batchnum = None + yield self.store.set_profile_active( + target_user.localpart, active, new_batchnum + ) + + # start a profile replication push + run_in_background(self._replicate_profiles) @defer.inlineCallbacks def get_avatar_url(self, target_user): @@ -191,8 +306,19 @@ class ProfileHandler(BaseHandler): if not by_admin and target_user != requester.user: raise AuthError(400, "Cannot set another user's avatar_url") + if not by_admin and self.hs.config.disable_set_avatar_url: + profile = yield self.store.get_profileinfo(target_user.localpart) + if profile.avatar_url: + raise SynapseError(400, "Changing avatar url is disabled on this server") + + if len(self.hs.config.replicate_user_profiles_to) > 0: + cur_batchnum = yield self.store.get_latest_profile_replication_batch_number() + new_batchnum = 0 if cur_batchnum is None else cur_batchnum + 1 + else: + new_batchnum = None + yield self.store.set_profile_avatar_url( - target_user.localpart, new_avatar_url + target_user.localpart, new_avatar_url, new_batchnum, ) if self.hs.config.user_directory_search_all_users: @@ -203,6 +329,9 @@ class ProfileHandler(BaseHandler): yield self._update_join_states(requester, target_user) + # start a profile replication push + run_in_background(self._replicate_profiles) + @defer.inlineCallbacks def on_profile_query(self, args): user = UserID.from_string(args["user_id"]) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 7caff0cbc8..8e9a82166f 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -124,6 +124,7 @@ class RegistrationHandler(BaseHandler): generate_token=True, guest_access_token=None, make_guest=False, + display_name=None, admin=False, ): """Registers a new client on the server. @@ -139,6 +140,7 @@ class RegistrationHandler(BaseHandler): since it offers no means of associating a device_id with the access_token. Instead you should call auth_handler.issue_access_token after registration. + display_name (str): The displayname to set for this user, if any Returns: A tuple of (user_id, access_token). Raises: @@ -175,13 +177,20 @@ class RegistrationHandler(BaseHandler): password_hash=password_hash, was_guest=was_guest, make_guest=make_guest, - create_profile_with_localpart=( - # If the user was a guest then they already have a profile - None if was_guest else user.localpart - ), admin=admin, ) + if display_name is None: + display_name = ( + # If the user was a guest then they already have a profile + None if was_guest else user.localpart + ) + + if display_name: + yield self.profile_handler.set_displayname( + user, None, display_name, by_admin=True, + ) + if self.hs.config.user_directory_search_all_users: profile = yield self.store.get_profileinfo(localpart) yield self.user_directory_handler.handle_local_profile_change( @@ -206,8 +215,12 @@ class RegistrationHandler(BaseHandler): token=token, password_hash=password_hash, make_guest=make_guest, - create_profile_with_localpart=user.localpart, ) + + yield self.profile_handler.set_displayname( + user, None, user.localpart, by_admin=True, + ) + except SynapseError: # if user id is taken, just generate another user = None @@ -251,8 +264,12 @@ class RegistrationHandler(BaseHandler): user_id=user_id, password_hash="", appservice_id=service_id, - create_profile_with_localpart=user.localpart, ) + + yield self.profile_handler.set_displayname( + user, None, user.localpart, by_admin=True, + ) + defer.returnValue(user_id) @defer.inlineCallbacks @@ -298,7 +315,10 @@ class RegistrationHandler(BaseHandler): user_id=user_id, token=token, password_hash=None, - create_profile_with_localpart=user.localpart, + ) + + yield self.profile_handler.set_displayname( + user, None, user.localpart, by_admin=True, ) except Exception as e: yield self.store.add_access_token_to_user(user_id, token) @@ -329,7 +349,9 @@ class RegistrationHandler(BaseHandler): logger.info("got threepid with medium '%s' and address '%s'", threepid['medium'], threepid['address']) - if not check_3pid_allowed(self.hs, threepid['medium'], threepid['address']): + if not ( + yield check_3pid_allowed(self.hs, threepid['medium'], threepid['address']) + ): raise RegistrationError( 403, "Third party identifier is not allowed" ) @@ -457,18 +479,15 @@ class RegistrationHandler(BaseHandler): user_id=user_id, token=token, password_hash=password_hash, - create_profile_with_localpart=user.localpart, ) + if displayname is not None: + yield self.profile_handler.set_displayname( + user, None, displayname, by_admin=True, + ) else: yield self._auth_handler.delete_access_tokens_for_user(user_id) yield self.store.add_access_token_to_user(user_id=user_id, token=token) - if displayname is not None: - logger.info("setting user display name: %s -> %s", user_id, displayname) - yield self.profile_handler.set_displayname( - user, requester, displayname, by_admin=True, - ) - defer.returnValue((user_id, token)) def auth_handler(self): diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f67512078b..78444efad2 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -43,12 +43,14 @@ class RoomCreationHandler(BaseHandler): "history_visibility": "shared", "original_invitees_have_ops": False, "guest_can_join": True, + "encryption_alg": "m.megolm.v1.aes-sha2", }, RoomCreationPreset.TRUSTED_PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, "history_visibility": "shared", "original_invitees_have_ops": True, "guest_can_join": True, + "encryption_alg": "m.megolm.v1.aes-sha2", }, RoomCreationPreset.PUBLIC_CHAT: { "join_rules": JoinRules.PUBLIC, @@ -394,6 +396,15 @@ class RoomCreationHandler(BaseHandler): content=content, ) + if "encryption_alg" in config: + send( + etype=EventTypes.Encryption, + state_key="", + content={ + 'algorithm': config["encryption_alg"], + } + ) + class RoomContextHandler(BaseHandler): @defer.inlineCallbacks diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c24e35362a..82c5fee759 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -613,7 +613,7 @@ class SyncHandler(object): res = yield self._generate_sync_entry_for_rooms( sync_result_builder, account_data_by_room ) - newly_joined_rooms, newly_joined_users, _, _ = res + newly_joined_rooms, newly_joined_or_invited_users, _, _ = res _, _, newly_left_rooms, newly_left_users = res block_all_presence_data = ( @@ -622,7 +622,7 @@ class SyncHandler(object): ) if not block_all_presence_data: yield self._generate_sync_entry_for_presence( - sync_result_builder, newly_joined_rooms, newly_joined_users + sync_result_builder, newly_joined_rooms, newly_joined_or_invited_users ) yield self._generate_sync_entry_for_to_device(sync_result_builder) @@ -630,7 +630,7 @@ class SyncHandler(object): device_lists = yield self._generate_sync_entry_for_device_list( sync_result_builder, newly_joined_rooms=newly_joined_rooms, - newly_joined_users=newly_joined_users, + newly_joined_or_invited_users=newly_joined_or_invited_users, newly_left_rooms=newly_left_rooms, newly_left_users=newly_left_users, ) @@ -706,7 +706,8 @@ class SyncHandler(object): @measure_func("_generate_sync_entry_for_device_list") @defer.inlineCallbacks def _generate_sync_entry_for_device_list(self, sync_result_builder, - newly_joined_rooms, newly_joined_users, + newly_joined_rooms, + newly_joined_or_invited_users, newly_left_rooms, newly_left_users): user_id = sync_result_builder.sync_config.user.to_string() since_token = sync_result_builder.since_token @@ -720,7 +721,7 @@ class SyncHandler(object): # share a room with? for room_id in newly_joined_rooms: joined_users = yield self.state.get_current_user_in_room(room_id) - newly_joined_users.update(joined_users) + newly_joined_or_invited_users.update(joined_users) for room_id in newly_left_rooms: left_users = yield self.state.get_current_user_in_room(room_id) @@ -728,7 +729,7 @@ class SyncHandler(object): # TODO: Check that these users are actually new, i.e. either they # weren't in the previous sync *or* they left and rejoined. - changed.update(newly_joined_users) + changed.update(newly_joined_or_invited_users) if not changed and not newly_left_users: defer.returnValue(DeviceLists( @@ -846,7 +847,7 @@ class SyncHandler(object): @defer.inlineCallbacks def _generate_sync_entry_for_presence(self, sync_result_builder, newly_joined_rooms, - newly_joined_users): + newly_joined_or_invited_users): """Generates the presence portion of the sync response. Populates the `sync_result_builder` with the result. @@ -854,8 +855,9 @@ class SyncHandler(object): sync_result_builder(SyncResultBuilder) newly_joined_rooms(list): List of rooms that the user has joined since the last sync (or empty if an initial sync) - newly_joined_users(list): List of users that have joined rooms - since the last sync (or empty if an initial sync) + newly_joined_or_invited_users(list): List of users that have joined + or been invited to rooms since the last sync (or empty if an initial + sync) """ now_token = sync_result_builder.now_token sync_config = sync_result_builder.sync_config @@ -881,7 +883,7 @@ class SyncHandler(object): "presence_key", presence_key ) - extra_users_ids = set(newly_joined_users) + extra_users_ids = set(newly_joined_or_invited_users) for room_id in newly_joined_rooms: users = yield self.state.get_current_user_in_room(room_id) extra_users_ids.update(users) @@ -913,7 +915,8 @@ class SyncHandler(object): Returns: Deferred(tuple): Returns a 4-tuple of - `(newly_joined_rooms, newly_joined_users, newly_left_rooms, newly_left_users)` + `(newly_joined_rooms, newly_joined_or_invited_users, + newly_left_rooms, newly_left_users)` """ user_id = sync_result_builder.sync_config.user.to_string() block_all_room_ephemeral = ( @@ -984,8 +987,8 @@ class SyncHandler(object): sync_result_builder.invited.extend(invited) - # Now we want to get any newly joined users - newly_joined_users = set() + # Now we want to get any newly joined or invited users + newly_joined_or_invited_users = set() newly_left_users = set() if since_token: for joined_sync in sync_result_builder.joined: @@ -994,19 +997,22 @@ class SyncHandler(object): ) for event in it: if event.type == EventTypes.Member: - if event.membership == Membership.JOIN: - newly_joined_users.add(event.state_key) + if ( + event.membership == Membership.JOIN or + event.membership == Membership.INVITE + ): + newly_joined_or_invited_users.add(event.state_key) else: prev_content = event.unsigned.get("prev_content", {}) prev_membership = prev_content.get("membership", None) if prev_membership == Membership.JOIN: newly_left_users.add(event.state_key) - newly_left_users -= newly_joined_users + newly_left_users -= newly_joined_or_invited_users defer.returnValue(( newly_joined_rooms, - newly_joined_users, + newly_joined_or_invited_users, newly_left_rooms, newly_left_users, )) @@ -1051,7 +1057,7 @@ class SyncHandler(object): where: room_entries is a list [RoomSyncResultBuilder] invited_rooms is a list [InvitedSyncResult] - newly_joined rooms is a list[str] of room ids + newly_joined_rooms is a list[str] of room ids newly_left_rooms is a list[str] of room ids """ user_id = sync_result_builder.sync_config.user.to_string() @@ -1086,7 +1092,7 @@ class SyncHandler(object): if room_id in sync_result_builder.joined_room_ids and non_joins: # Always include if the user (re)joined the room, especially # important so that device list changes are calculated correctly. - # If there are non join member events, but we are still in the room, + # If there are non-join member events, but we are still in the room, # then the user must have left and joined newly_joined_rooms.append(room_id) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index eeae466d82..40caba24a4 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -51,7 +51,7 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): 'id_server', 'client_secret', 'email', 'send_attempt' ]) - if not check_3pid_allowed(self.hs, "email", body['email']): + if not (yield check_3pid_allowed(self.hs, "email", body['email'])): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) @@ -87,7 +87,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - if not check_3pid_allowed(self.hs, "msisdn", msisdn): + if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) @@ -232,7 +232,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): ['id_server', 'client_secret', 'email', 'send_attempt'], ) - if not check_3pid_allowed(self.hs, "email", body['email']): + if not (yield check_3pid_allowed(self.hs, "email", body['email'])): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) @@ -267,7 +267,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - if not check_3pid_allowed(self.hs, "msisdn", msisdn): + if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) @@ -306,6 +306,9 @@ class ThreepidRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): + if self.hs.config.disable_3pid_changes: + raise SynapseError(400, "3PID changes disabled on this server") + body = parse_json_object_from_request(request) threePidCreds = body.get('threePidCreds') @@ -352,11 +355,15 @@ class ThreepidDeleteRestServlet(RestServlet): def __init__(self, hs): super(ThreepidDeleteRestServlet, self).__init__() + self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() @defer.inlineCallbacks def on_POST(self, request): + if self.hs.config.disable_3pid_changes: + raise SynapseError(400, "3PID changes disabled on this server") + body = parse_json_object_from_request(request) assert_params_in_dict(body, ['medium', 'address']) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index d6cf915d86..1b06e35dbd 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -23,7 +23,7 @@ from six import string_types from twisted.internet import defer import synapse -import synapse.types +from synapse import types from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.http.servlet import ( @@ -38,6 +38,10 @@ from synapse.util.threepids import check_3pid_allowed from ._base import client_v2_patterns, interactive_auth_handler +import re +from string import capwords + + # We ought to be using hmac.compare_digest() but on older pythons it doesn't # exist. It's a _really minor_ security flaw to use plain string comparison # because the timing attack is so obscured by all the other code here it's @@ -72,7 +76,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): 'id_server', 'client_secret', 'email', 'send_attempt' ]) - if not check_3pid_allowed(self.hs, "email", body['email']): + if not (yield check_3pid_allowed(self.hs, "email", body['email'])): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) @@ -112,7 +116,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): msisdn = phone_number_to_msisdn(body['country'], body['phone_number']) - if not check_3pid_allowed(self.hs, "msisdn", msisdn): + if not (yield check_3pid_allowed(self.hs, "msisdn", msisdn)): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) @@ -222,6 +226,8 @@ class RegisterRestServlet(RestServlet): raise SynapseError(400, "Invalid username") desired_username = body['username'] + desired_display_name = None + appservice = None if self.auth.has_access_token(request): appservice = yield self.auth.get_appservice_by_req(request) @@ -297,13 +303,6 @@ class RegisterRestServlet(RestServlet): session_id, "registered_user_id", None ) - if desired_username is not None: - yield self.registration_handler.check_username( - desired_username, - guest_access_token=guest_access_token, - assigned_user_id=registered_user_id, - ) - # Only give msisdn flows if the x_show_msisdn flag is given: # this is a hack to work around the fact that clients were shipped # that use fallback registration if they see any flows that they don't @@ -370,12 +369,87 @@ class RegisterRestServlet(RestServlet): medium = auth_result[login_type]['medium'] address = auth_result[login_type]['address'] - if not check_3pid_allowed(self.hs, medium, address): + if not (yield check_3pid_allowed(self.hs, medium, address)): raise SynapseError( 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, ) + if self.hs.config.register_mxid_from_3pid: + # override the desired_username based on the 3PID if any. + # reset it first to avoid folks picking their own username. + desired_username = None + + # we should have an auth_result at this point if we're going to progress + # to register the user (i.e. we haven't picked up a registered_user_id + # from our session store), in which case get ready and gen the + # desired_username + if auth_result: + if ( + self.hs.config.register_mxid_from_3pid == 'email' and + LoginType.EMAIL_IDENTITY in auth_result + ): + address = auth_result[LoginType.EMAIL_IDENTITY]['address'] + desired_username = types.strip_invalid_mxid_characters( + address.replace('@', '-').lower() + ) + + # find a unique mxid for the account, suffixing numbers + # if needed + while True: + try: + yield self.registration_handler.check_username( + desired_username, + guest_access_token=guest_access_token, + assigned_user_id=registered_user_id, + ) + # if we got this far we passed the check. + break + except SynapseError as e: + if e.errcode == Codes.USER_IN_USE: + m = re.match(r'^(.*?)(\d+)$', desired_username) + if m: + desired_username = m.group(1) + str( + int(m.group(2)) + 1 + ) + else: + desired_username += "1" + else: + # something else went wrong. + break + + # XXX: a nasty heuristic to turn an email address into + # a displayname, as part of register_mxid_from_3pid + parts = address.replace('.', ' ').split('@') + org_parts = parts[1].split(' ') + + if org_parts[-2] == "matrix" and org_parts[-1] == "org": + org = "Tchap Admin" + elif org_parts[-2] == "gouv" and org_parts[-1] == "fr": + org = org_parts[-3] if len(org_parts) > 2 else org_parts[-2] + else: + org = org_parts[-2] + + desired_display_name = ( + capwords(parts[0]) + " [" + capwords(org) + "]" + ) + elif ( + self.hs.config.register_mxid_from_3pid == 'msisdn' and + LoginType.MSISDN in auth_result + ): + desired_username = auth_result[LoginType.MSISDN]['address'] + else: + raise SynapseError( + 400, "Cannot derive mxid from 3pid; no recognised 3pid" + ) + + if desired_username is not None: + yield self.registration_handler.check_username( + desired_username, + guest_access_token=guest_access_token, + assigned_user_id=registered_user_id, + ) + if registered_user_id is not None: logger.info( "Already registered user ID %r for this session", @@ -388,10 +462,18 @@ class RegisterRestServlet(RestServlet): # NB: This may be from the auth handler and NOT from the POST assert_params_in_dict(params, ["password"]) - desired_username = params.get("username", None) + if not self.hs.config.register_mxid_from_3pid: + desired_username = params.get("username", None) + else: + # we keep the original desired_username derived from the 3pid above + pass + new_password = params.get("password", None) guest_access_token = params.get("guest_access_token", None) + # XXX: don't we need to validate these for length etc like we did on + # the ones from the JSON body earlier on in the method? + if desired_username is not None: desired_username = desired_username.lower() @@ -400,6 +482,7 @@ class RegisterRestServlet(RestServlet): password=new_password, guest_access_token=guest_access_token, generate_token=False, + display_name=desired_display_name, ) # remember that we've now registered that user account, and with diff --git a/synapse/rest/client/v2_alpha/user_directory.py b/synapse/rest/client/v2_alpha/user_directory.py index cac0624ba7..7d895cee1e 100644 --- a/synapse/rest/client/v2_alpha/user_directory.py +++ b/synapse/rest/client/v2_alpha/user_directory.py @@ -16,6 +16,7 @@ import logging from twisted.internet import defer +from signedjson.sign import sign_json from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet, parse_json_object_from_request @@ -37,6 +38,7 @@ class UserDirectorySearchRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() self.user_directory_handler = hs.get_user_directory_handler() + self.http_client = hs.get_simple_http_client() @defer.inlineCallbacks def on_POST(self, request): @@ -61,6 +63,14 @@ class UserDirectorySearchRestServlet(RestServlet): body = parse_json_object_from_request(request) + if self.hs.config.user_directory_defer_to_id_server: + signed_body = sign_json(body, self.hs.hostname, self.hs.config.signing_key[0]) + url = "http://%s/_matrix/identity/api/v1/user_directory/search" % ( + self.hs.config.user_directory_defer_to_id_server, + ) + resp = yield self.http_client.post_json_get_json(url, signed_body) + defer.returnValue((200, resp)) + limit = body.get("limit", 10) limit = min(limit, 50) diff --git a/synapse/rulecheck/__init__.py b/synapse/rulecheck/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/synapse/rulecheck/__init__.py diff --git a/synapse/rulecheck/domain_rule_checker.py b/synapse/rulecheck/domain_rule_checker.py new file mode 100644 index 0000000000..3caa6b34cb --- /dev/null +++ b/synapse/rulecheck/domain_rule_checker.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from synapse.config._base import ConfigError + +logger = logging.getLogger(__name__) + + +class DomainRuleChecker(object): + """ + A re-implementation of the SpamChecker that prevents users in one domain from + inviting users in other domains to rooms, based on a configuration. + + Takes a config in the format: + + spam_checker: + module: "rulecheck.DomainRuleChecker" + config: + domain_mapping: + "inviter_domain": [ "invitee_domain_permitted", "other_domain_permitted" ] + "other_inviter_domain": [ "invitee_domain_permitted" ] + default: False + } + + Don't forget to consider if you can invite users from your own domain. + """ + + def __init__(self, config): + self.domain_mapping = config["domain_mapping"] or {} + self.default = config["default"] + + def check_event_for_spam(self, event): + """Implements synapse.events.SpamChecker.check_event_for_spam + """ + return False + + def user_may_invite(self, inviter_userid, invitee_userid, room_id): + """Implements synapse.events.SpamChecker.user_may_invite + """ + inviter_domain = self._get_domain_from_id(inviter_userid) + invitee_domain = self._get_domain_from_id(invitee_userid) + + if inviter_domain not in self.domain_mapping: + return self.default + + return invitee_domain in self.domain_mapping[inviter_domain] + + def user_may_create_room(self, userid): + """Implements synapse.events.SpamChecker.user_may_create_room + """ + return True + + def user_may_create_room_alias(self, userid, room_alias): + """Implements synapse.events.SpamChecker.user_may_create_room_alias + """ + return True + + def user_may_publish_room(self, userid, room_id): + """Implements synapse.events.SpamChecker.user_may_publish_room + """ + return True + + @staticmethod + def parse_config(config): + """Implements synapse.events.SpamChecker.parse_config + """ + if "default" in config: + return config + else: + raise ConfigError("No default set for spam_config DomainRuleChecker") + + @staticmethod + def _get_domain_from_id(mxid): + """Parses a string and returns the domain part of the mxid. + + Args: + mxid (str): a valid mxid + + Returns: + str: the domain part of the mxid + + """ + idx = mxid.find(":") + if idx == -1: + raise Exception("Invalid ID: %r" % (mxid,)) + return mxid[idx + 1:] diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 60295da254..e6848c70a0 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +21,8 @@ from synapse.storage.roommember import ProfileInfo from ._base import SQLBaseStore +BATCH_SIZE = 100 + class ProfileWorkerStore(SQLBaseStore): @defer.inlineCallbacks @@ -62,6 +65,55 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_profile_avatar_url", ) + def get_latest_profile_replication_batch_number(self): + def f(txn): + txn.execute("SELECT MAX(batch) as maxbatch FROM profiles") + rows = self.cursor_to_dict(txn) + return rows[0]['maxbatch'] + return self.runInteraction( + "get_latest_profile_replication_batch_number", f, + ) + + def get_profile_batch(self, batchnum): + return self._simple_select_list( + table="profiles", + keyvalues={ + "batch": batchnum, + }, + retcols=("user_id", "displayname", "avatar_url", "active"), + desc="get_profile_batch", + ) + + def assign_profile_batch(self): + def f(txn): + sql = ( + "UPDATE profiles SET batch = " + "(SELECT COALESCE(MAX(batch), -1) + 1 FROM profiles) " + "WHERE user_id in (" + " SELECT user_id FROM profiles WHERE batch is NULL limit ?" + ")" + ) + txn.execute(sql, (BATCH_SIZE,)) + return txn.rowcount + return self.runInteraction("assign_profile_batch", f) + + def get_replication_hosts(self): + def f(txn): + txn.execute("SELECT host, last_synced_batch FROM profile_replication_status") + rows = self.cursor_to_dict(txn) + return {r['host']: r['last_synced_batch'] for r in rows} + return self.runInteraction("get_replication_hosts", f) + + def update_replication_batch_for_host(self, host, last_synced_batch): + return self._simple_upsert( + table="profile_replication_status", + keyvalues={"host": host}, + values={ + "last_synced_batch": last_synced_batch, + }, + desc="update_replication_batch_for_host", + ) + def get_from_remote_profile_cache(self, user_id): return self._simple_select_one( table="remote_profile_cache", @@ -73,27 +125,44 @@ class ProfileWorkerStore(SQLBaseStore): class ProfileStore(ProfileWorkerStore): - def create_profile(self, user_localpart): - return self._simple_insert( + def set_profile_displayname(self, user_localpart, new_displayname, batchnum): + return self._simple_upsert( table="profiles", - values={"user_id": user_localpart}, - desc="create_profile", + keyvalues={"user_id": user_localpart}, + values={ + "displayname": new_displayname, + "batch": batchnum, + }, + desc="set_profile_displayname", + lock=False # we can do this because user_id has a unique index ) - def set_profile_displayname(self, user_localpart, new_displayname): - return self._simple_update_one( + def set_profile_avatar_url(self, user_localpart, new_avatar_url, batchnum): + return self._simple_upsert( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname}, - desc="set_profile_displayname", + values={ + "avatar_url": new_avatar_url, + "batch": batchnum, + }, + desc="set_profile_avatar_url", + lock=False # we can do this because user_id has a unique index ) - def set_profile_avatar_url(self, user_localpart, new_avatar_url): - return self._simple_update_one( + def set_profile_active(self, user_localpart, active, batchnum): + values = { + "active": int(active), + "batch": batchnum, + } + if not active: + values["avatar_url"] = None + values["displayname"] = None + return self._simple_upsert( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"avatar_url": new_avatar_url}, - desc="set_profile_avatar_url", + values=values, + desc="set_profile_active", + lock=False # we can do this because user_id has a unique index ) def add_remote_profile_cache(self, user_id, displayname, avatar_url): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 07333f777d..50706519aa 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -141,7 +141,7 @@ class RegistrationStore(RegistrationWorkerStore, def register(self, user_id, token=None, password_hash=None, was_guest=False, make_guest=False, appservice_id=None, - create_profile_with_localpart=None, admin=False): + admin=False): """Attempts to register an account. Args: @@ -155,8 +155,6 @@ class RegistrationStore(RegistrationWorkerStore, make_guest (boolean): True if the the new user should be guest, false to add a regular user account. appservice_id (str): The ID of the appservice registering the user. - create_profile_with_localpart (str): Optionally create a profile for - the given localpart. Raises: StoreError if the user_id could not be registered. """ @@ -169,7 +167,6 @@ class RegistrationStore(RegistrationWorkerStore, was_guest, make_guest, appservice_id, - create_profile_with_localpart, admin ) @@ -182,7 +179,6 @@ class RegistrationStore(RegistrationWorkerStore, was_guest, make_guest, appservice_id, - create_profile_with_localpart, admin, ): now = int(self.clock.time()) @@ -247,14 +243,6 @@ class RegistrationStore(RegistrationWorkerStore, (next_id, user_id, token,) ) - if create_profile_with_localpart: - # set a default displayname serverside to avoid ugly race - # between auto-joins and clients trying to set displaynames - txn.execute( - "INSERT INTO profiles(user_id, displayname) VALUES (?,?)", - (create_profile_with_localpart, create_profile_with_localpart) - ) - self._invalidate_cache_and_stream( txn, self.get_user_by_id, (user_id,) ) diff --git a/synapse/storage/schema/delta/48/profiles_batch.sql b/synapse/storage/schema/delta/48/profiles_batch.sql new file mode 100644 index 0000000000..e744c02fe8 --- /dev/null +++ b/synapse/storage/schema/delta/48/profiles_batch.sql @@ -0,0 +1,36 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Add a batch number to track changes to profiles and the + * order they're made in so we can replicate user profiles + * to other hosts as they change + */ +ALTER TABLE profiles ADD COLUMN batch BIGINT DEFAULT NULL; + +/* + * Index on the batch number so we can get profiles + * by their batch + */ +CREATE INDEX profiles_batch_idx ON profiles(batch); + +/* + * A table to track what batch of user profiles has been + * synced to what profile replication target. + */ +CREATE TABLE profile_replication_status ( + host TEXT NOT NULL, + last_synced_batch BIGINT NOT NULL +); diff --git a/synapse/storage/schema/delta/50/profiles_deactivated_users.sql b/synapse/storage/schema/delta/50/profiles_deactivated_users.sql new file mode 100644 index 0000000000..c8893ecbe8 --- /dev/null +++ b/synapse/storage/schema/delta/50/profiles_deactivated_users.sql @@ -0,0 +1,23 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * A flag saying whether the user owning the profile has been deactivated + * This really belongs on the users table, not here, but the users table + * stores users by their full user_id and profiles stores them by localpart, + * so we can't easily join between the two tables. Plus, the batch number + * realy ought to represent data in this table that has changed. + */ +ALTER TABLE profiles ADD COLUMN active SMALLINT DEFAULT 1 NOT NULL; diff --git a/synapse/types.py b/synapse/types.py index 08f058f714..7d6cc7dba0 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -228,6 +228,18 @@ def contains_invalid_mxid_characters(localpart): return any(c not in mxid_localpart_allowed_characters for c in localpart) +def strip_invalid_mxid_characters(localpart): + """Removes any invalid characters from an mxid + + Args: + localpart (basestring): the localpart to be stripped + + Returns: + localpart (basestring): the localpart having been stripped + """ + return filter(lambda c: c in mxid_localpart_allowed_characters, localpart) + + class StreamToken( namedtuple("Token", ( "room_key", diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py index 75efa0117b..353d220bad 100644 --- a/synapse/util/threepids.py +++ b/synapse/util/threepids.py @@ -16,9 +16,12 @@ import logging import re +from twisted.internet import defer + logger = logging.getLogger(__name__) +@defer.inlineCallbacks def check_3pid_allowed(hs, medium, address): """Checks whether a given format of 3PID is allowed to be used on this HS @@ -28,9 +31,22 @@ def check_3pid_allowed(hs, medium, address): address (str): address within that medium (e.g. "wotan@matrix.org") msisdns need to first have been canonicalised Returns: - bool: whether the 3PID medium/address is allowed to be added to this HS + defered bool: whether the 3PID medium/address is allowed to be added to this HS """ + if hs.config.check_is_for_allowed_local_3pids: + data = yield hs.get_simple_http_client().get_json( + "https://%s%s" % ( + hs.config.check_is_for_allowed_local_3pids, + "/_matrix/identity/api/v1/info" + ), + {'medium': medium, 'address': address} + ) + if hs.config.allow_invited_3pids and data.get('invited'): + defer.returnValue(True) + else: + defer.returnValue(data['hs'] == hs.config.server_name) + if hs.config.allowed_local_3pids: for constraint in hs.config.allowed_local_3pids: logger.debug( @@ -41,8 +57,8 @@ def check_3pid_allowed(hs, medium, address): medium == constraint['medium'] and re.match(constraint['pattern'], address) ): - return True + defer.returnValue(True) else: - return True + defer.returnValue(True) - return False + defer.returnValue(False) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index dc17918a3d..ead3b4030b 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -68,14 +68,12 @@ class ProfileTestCase(unittest.TestCase): self.bob = UserID.from_string("@4567:test") self.alice = UserID.from_string("@alice:remote") - yield self.store.create_profile(self.frank.localpart) - self.handler = hs.get_profile_handler() @defer.inlineCallbacks def test_get_my_name(self): yield self.store.set_profile_displayname( - self.frank.localpart, "Frank" + self.frank.localpart, "Frank", 1, ) displayname = yield self.handler.get_displayname(self.frank) @@ -123,8 +121,7 @@ class ProfileTestCase(unittest.TestCase): @defer.inlineCallbacks def test_incoming_fed_query(self): - yield self.store.create_profile("caroline") - yield self.store.set_profile_displayname("caroline", "Caroline") + yield self.store.set_profile_displayname("caroline", "Caroline", 1) response = yield self.query_handlers["profile"]( {"user_id": "@caroline:test", "field": "displayname"} @@ -135,7 +132,7 @@ class ProfileTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_my_avatar(self): yield self.store.set_profile_avatar_url( - self.frank.localpart, "http://my.server/me.png" + self.frank.localpart, "http://my.server/me.png", 1, ) avatar_url = yield self.handler.get_avatar_url(self.frank) diff --git a/tests/rulecheck/__init__.py b/tests/rulecheck/__init__.py new file mode 100644 index 0000000000..a354d38ca8 --- /dev/null +++ b/tests/rulecheck/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/rulecheck/test_domainrulecheck.py b/tests/rulecheck/test_domainrulecheck.py new file mode 100644 index 0000000000..a24fb53766 --- /dev/null +++ b/tests/rulecheck/test_domainrulecheck.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest + +from synapse.config._base import ConfigError +from synapse.rulecheck.domain_rule_checker import DomainRuleChecker + + +class DomainRuleCheckerTestCase(unittest.TestCase): + + def test_allowed(self): + config = { + "default": False, + "domain_mapping": { + "source_one": ["target_one", "target_two"], + "source_two": ["target_two"] + } + } + check = DomainRuleChecker(config) + self.assertTrue(check.user_may_invite("test:source_one", + "test:target_one", "room")) + self.assertTrue(check.user_may_invite("test:source_one", + "test:target_two", "room")) + self.assertTrue(check.user_may_invite("test:source_two", + "test:target_two", "room")) + + def test_disallowed(self): + config = { + "default": True, + "domain_mapping": { + "source_one": ["target_one", "target_two"], + "source_two": ["target_two"], + "source_four": [] + } + } + check = DomainRuleChecker(config) + self.assertFalse(check.user_may_invite("test:source_one", + "test:target_three", "room")) + self.assertFalse(check.user_may_invite("test:source_two", + "test:target_three", "room")) + self.assertFalse(check.user_may_invite("test:source_two", + "test:target_one", "room")) + self.assertFalse(check.user_may_invite("test:source_four", + "test:target_one", "room")) + + def test_default_allow(self): + config = { + "default": True, + "domain_mapping": { + "source_one": ["target_one", "target_two"], + "source_two": ["target_two"] + } + } + check = DomainRuleChecker(config) + self.assertTrue(check.user_may_invite("test:source_three", + "test:target_one", "room")) + + def test_default_deny(self): + config = { + "default": False, + "domain_mapping": { + "source_one": ["target_one", "target_two"], + "source_two": ["target_two"] + } + } + check = DomainRuleChecker(config) + self.assertFalse(check.user_may_invite("test:source_three", + "test:target_one", "room")) + + def test_config_parse(self): + config = { + "default": False, + "domain_mapping": { + "source_one": ["target_one", "target_two"], + "source_two": ["target_two"] + } + } + self.assertEquals(config, DomainRuleChecker.parse_config(config)) + + def test_config_parse_failure(self): + config = { + "domain_mapping": { + "source_one": ["target_one", "target_two"], + "source_two": ["target_two"] + } + } + self.assertRaises(ConfigError, DomainRuleChecker.parse_config, config) diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index 2c95e5e95a..003076ec53 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -35,12 +35,8 @@ class ProfileStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_displayname(self): - yield self.store.create_profile( - self.u_frank.localpart - ) - yield self.store.set_profile_displayname( - self.u_frank.localpart, "Frank" + self.u_frank.localpart, "Frank", 1, ) self.assertEquals( @@ -50,12 +46,8 @@ class ProfileStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_avatar_url(self): - yield self.store.create_profile( - self.u_frank.localpart - ) - yield self.store.set_profile_avatar_url( - self.u_frank.localpart, "http://my.site/here" + self.u_frank.localpart, "http://my.site/here", 1, ) self.assertEquals( diff --git a/tests/utils.py b/tests/utils.py index e488238bb3..0dfd7792fb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -69,6 +69,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None config.federation_rc_concurrent = 10 config.filter_timeline_limit = 5000 config.user_directory_search_all_users = False + config.replicate_user_profiles_to = [] config.user_consent_server_notice_content = None config.block_events_without_consent_error = None |