diff options
| -rw-r--r-- | CHANGES.rst | 37 | ||||
| -rw-r--r-- | docs/consent_tracking.md | 160 | ||||
| -rw-r--r-- | docs/manhole.md | 43 | ||||
| -rw-r--r-- | docs/privacy_policy_templates/README.md | 23 | ||||
| -rw-r--r-- | docs/privacy_policy_templates/en/1.0.html | 6 | ||||
| -rw-r--r-- | docs/server_notices.md | 71 | ||||
| -rw-r--r-- | synapse/__init__.py | 2 | ||||
| -rw-r--r-- | synapse/api/errors.py | 31 | ||||
| -rw-r--r-- | synapse/api/urls.py | 50 | ||||
| -rwxr-xr-x | synapse/app/homeserver.py | 2 | ||||
| -rw-r--r-- | synapse/config/consent_config.py | 17 | ||||
| -rw-r--r-- | synapse/config/server_notices_config.py | 15 | ||||
| -rw-r--r-- | synapse/handlers/deactivate_account.py | 2 | ||||
| -rw-r--r-- | synapse/handlers/federation.py | 10 | ||||
| -rw-r--r-- | synapse/handlers/message.py | 91 | ||||
| -rw-r--r-- | synapse/handlers/room.py | 4 | ||||
| -rw-r--r-- | synapse/handlers/room_member.py | 1 | ||||
| -rw-r--r-- | synapse/rest/consent/consent_resource.py | 22 | ||||
| -rw-r--r-- | synapse/server_notices/consent_server_notices.py | 41 | ||||
| -rw-r--r-- | synapse/server_notices/server_notices_manager.py | 23 | ||||
| -rw-r--r-- | synapse/server_notices/worker_server_notices_sender.py | 2 | ||||
| -rw-r--r-- | synapse/storage/stream.py | 15 | ||||
| -rw-r--r-- | tests/utils.py | 1 |
23 files changed, 611 insertions, 58 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 851cb9b2d7..096fa4794a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,40 @@ +Changes in synapse v0.30.0-rc1 (2018-05-23) +========================================== + +This version includes support for Privacy Notice agreement collection. More +details live at docs/privacy_policy_templates/README.md + + +GDPR Support: + +* ConsentResource to gather policy consent from users (PR #3213) +* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) +* Infrastructure for a server notices room (PR #3232) +* Send users a server notice about consent (PR #3236) +* Reject attempts to send event before privacy consent is given (PR #3257) +* Add a 'has_consented' template var to consent forms (PR #3262) +* Fix dependency on jinja2 (PR #3263) + +Features: + +* Cohort analytics (PR #3163, #3241, #3251) +* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! +* Add in flight request metrics (PR #3252) + +Changes: + +* Remove unused `update_external_syncs` (PR #3233) +* Use stream rather depth ordering for push actions (PR #3212) +* Make purge_history operate on tokens (PR #3221) +* Don't support limitless pagination (PR #3265) + +Bug Fixes: + +* Fix logcontext resource usage tracking (PR #3258) +* Fix error in handling receipts (PR #3235) +* Stop the transaction cache caching failures (PR #3255) + + Changes in synapse v0.29.1 (2018-05-17) ========================================== Changes: diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md new file mode 100644 index 0000000000..064eae82f7 --- /dev/null +++ b/docs/consent_tracking.md @@ -0,0 +1,160 @@ +Support in Synapse for tracking agreement to server terms and conditions +======================================================================== + +Synapse 0.30 introduces support for tracking whether users have agreed to the +terms and conditions set by the administrator of a server - and blocking access +to the server until they have. + +There are several parts to this functionality; each requires some specific +configuration in `homeserver.yaml` to be enabled. + +Note that various parts of the configuation and this document refer to the +"privacy policy": agreement with a privacy policy is one particular use of this +feature, but of course adminstrators can specify other terms and conditions +unrelated to "privacy" per se. + +Collecting policy agreement from a user +--------------------------------------- + +Synapse can be configured to serve the user a simple policy form with an +"accept" button. Clicking "Accept" records the user's acceptance in the +database and shows a success page. + +To enable this, first create templates for the policy and success pages. +These should be stored on the local filesystem. + +These templates use the [Jinja2](http://jinja.pocoo.org) templating language, +and [docs/privacy_policy_templates](privacy_policy_templates) gives +examples of the sort of thing that can be done. + +Note that the templates must be stored under a name giving the language of the +template - currently this must always be `en` (for "English"); +internationalisation support is intended for the future. + +The template for the policy itself should be versioned and named according to +the version: for example `1.0.html`. The version of the policy which the user +has agreed to is stored in the database. + +Once the templates are in place, make the following changes to `homeserver.yaml`: + + 1. Add a `user_consent` section, which should look like: + + ```yaml + user_consent: + template_dir: privacy_policy_templates + version: 1.0 + ``` + + `template_dir` points to the directory containing the policy + templates. `version` defines the version of the policy which will be served + to the user. In the example above, Synapse will serve + `privacy_policy_templates/en/1.0.html`. + + + 2. Add a `form_secret` setting at the top level: + + + ```yaml + form_secret: "<unique secret>" + ``` + + This should be set to an arbitrary secret string (try `pwgen -y 30` to + generate suitable secrets). + + More on what this is used for below. + + 3. Add `consent` wherever the `client` resource is currently enabled in the + `listeners` configuration. For example: + + ```yaml + listeners: + - port: 8008 + resources: + - names: + - client + - consent + ``` + + +Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this +should be a matter of `pip install Jinja2`. On debian, try `apt-get install +python-jinja2`. + +Once this is complete, and the server has been restarted, try visiting +`https://<server>/_matrix/consent`. If correctly configured, this should give +an error "Missing string query parameter 'u'". It is now possible to manually +construct URIs where users can give their consent. + +### Constructing the consent URI + +It may be useful to manually construct the "consent URI" for a given user - for +instance, in order to send them an email asking them to consent. To do this, +take the base `https://<server>/_matrix/consent` URL and add the following +query parameters: + + * `u`: the user id of the user. This can either be a full MXID + (`@user:server.com`) or just the localpart (`user`). + + * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is + possible to calculate this on the commandline with something like: + + ```bash + echo -n '<user>' | openssl sha256 -hmac '<form_secret>' + ``` + + This should result in a URI which looks something like: + `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`. + + +Sending users a server notice asking them to agree to the policy +---------------------------------------------------------------- + +It is possible to configure Synapse to send a [server +notice](server_notices.md) to anybody who has not yet agreed to the current +version of the policy. To do so: + + * ensure that the consent resource is configured, as in the previous section + + * ensure that server notices are configured, as in [server_notices.md](server_notices.md). + + * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For + example: + + ```yaml + user_consent: + server_notice_content: + msgtype: m.text + body: >- + Please give your consent to the privacy policy at %(consent_uri)s. + ``` + + Synapse automatically replaces the placeholder `%(consent_uri)s` with the + consent uri for that user. + + * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base + URI that clients use to connect to the server. (It is used to construct + `consent_uri` in the server notice.) + + +Blocking users from using the server until they agree to the policy +------------------------------------------------------------------- + +Synapse can be configured to block any attempts to join rooms or send messages +until the user has given their agreement to the policy. (Joining the server +notices room is exempted from this). + +To enable this, add `block_events_error` under `user_consent`. For example: + +```yaml +user_consent: + block_events_error: >- + You can't send any messages until you consent to the privacy policy at + %(consent_uri)s. +``` + +Synapse automatically replaces the placeholder `%(consent_uri)s` with the +consent uri for that user. + +ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base +URI that clients use to connect to the server. (It is used to construct +`consent_uri` in the error.) diff --git a/docs/manhole.md b/docs/manhole.md new file mode 100644 index 0000000000..7375f5ad46 --- /dev/null +++ b/docs/manhole.md @@ -0,0 +1,43 @@ +Using the synapse manhole +========================= + +The "manhole" allows server administrators to access a Python shell on a running +Synapse installation. This is a very powerful mechanism for administration and +debugging. + +To enable it, first uncomment the `manhole` listener configuration in +`homeserver.yaml`: + +```yaml +listeners: + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` + +(`bind_addresses` in the above is important: it ensures that access to the +manhole is only possible for local users). + +Note that this will give administrative access to synapse to **all users** with +shell access to the server. It should therefore **not** be enabled in +environments where untrusted users have shell access. + +Then restart synapse, and point an ssh client at port 9000 on localhost, using +the username `matrix`: + +```bash +ssh -p9000 matrix@localhost +``` + +The password is `rabbithole`. + +This gives a Python REPL in which `hs` gives access to the +`synapse.server.HomeServer` object - which in turn gives access to many other +parts of the process. + +As a simple example, retrieving an event from the database: + +``` +>>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org') +<Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>> +``` diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md deleted file mode 100644 index 8e91c516b3..0000000000 --- a/docs/privacy_policy_templates/README.md +++ /dev/null @@ -1,23 +0,0 @@ -If enabling the 'consent' resource in synapse, you will need some templates -for the HTML to be served to the user. This directory contains very simple -examples of the sort of thing that can be done. - -You'll need to add this sort of thing to your homeserver.yaml: - -``` -form_secret: <unique but arbitrary secret> - -user_consent: - template_dir: docs/privacy_policy_templates - default_version: 1.0 -``` - -You should then be able to enable the `consent` resource under a `listener` -entry. For example: - -``` -listeners: - - port: 8008 - resources: - - names: [client, consent] -``` diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html index ab8666f0c3..55c5e4b612 100644 --- a/docs/privacy_policy_templates/en/1.0.html +++ b/docs/privacy_policy_templates/en/1.0.html @@ -4,6 +4,11 @@ <title>Matrix.org Privacy policy</title> </head> <body> + {% if has_consented %} + <p> + Your base already belong to us. + </p> + {% else %} <p> All your base are belong to us. </p> @@ -13,5 +18,6 @@ <input type="hidden" name="h" value="{{userhmac}}"/> <input type="submit" value="Sure thing!"/> </form> + {% endif %} </body> </html> diff --git a/docs/server_notices.md b/docs/server_notices.md new file mode 100644 index 0000000000..221553b24d --- /dev/null +++ b/docs/server_notices.md @@ -0,0 +1,71 @@ +Server Notices +============== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server polices(see +[consent_tracking.md](consent_tracking.md)), however the intention is that +they may also find a use for features such as "Message of the day". + +This is a feature specific to Synapse, but it uses standard Matrix +communication mechanisms, so should work with any Matrix client. + +User experience +--------------- + +When the user is first sent a server notice, they will get an invitation to a +room (typically called 'Server Notices', though this is configurable in +`homeserver.yaml`). They will be **unable to reject** this invitation - +attempts to do so will receive an error. + +Once they accept the invitation, they will see the notice message in the room +history; it will appear to have come from the 'server notices user' (see +below). + +The user is prevented from sending any messages in this room by the power +levels. They also cannot leave it. + +Synapse configuration +--------------------- + +Server notices come from a specific user id on the server. Server +administrators are free to choose the user id - something like `server` is +suggested, meaning the notices will come from +`@server:<your_server_name>`. Once the Server Notices user is configured, that +user id becomes a special, privileged user, so administrators should ensure +that **it is not already allocated**. + +In order to support server notices, it is necessary to add some configuration +to the `homeserver.yaml` file. In particular, you should add a `server_notices` +section, which should look like this: + +```yaml +server_notices: + system_mxid_localpart: server + system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: "Server Notices" +``` + +The only compulsory setting is `system_mxid_localpart`, which defines the user +id of the Server Notices user, as above. `room_name` defines the name of the +room which will be created. + +`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the +displayname and avatar of the Server Notices user. + +Sending notices +--------------- + +As of the current version of synapse, there is no convenient interface for +sending notices (other than the automated ones sent as part of consent +tracking). + +In the meantime, it is possible to test this feature using the manhole. Having +gone into the manhole as described in [manhole.md](manhole.md), a notice can be +sent with something like: + +``` +>>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'}) +``` diff --git a/synapse/__init__.py b/synapse/__init__.py index b11a1d3ea3..353387f154 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.1" +__version__ = "0.30.0-rc1" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index a9ff5576f3..e6ad3768f0 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -19,6 +19,7 @@ import logging import simplejson as json from six import iteritems +from six.moves import http_client logger = logging.getLogger(__name__) @@ -51,6 +52,8 @@ class Codes(object): THREEPID_DENIED = "M_THREEPID_DENIED" INVALID_USERNAME = "M_INVALID_USERNAME" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" + CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" + CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" class CodeMessageException(RuntimeError): @@ -138,6 +141,32 @@ class SynapseError(CodeMessageException): return res +class ConsentNotGivenError(SynapseError): + """The error returned to the client when the user has not consented to the + privacy policy. + """ + def __init__(self, msg, consent_uri): + """Constructs a ConsentNotGivenError + + Args: + msg (str): The human-readable error message + consent_url (str): The URL where the user can give their consent + """ + super(ConsentNotGivenError, self).__init__( + code=http_client.FORBIDDEN, + msg=msg, + errcode=Codes.CONSENT_NOT_GIVEN + ) + self._consent_uri = consent_uri + + def error_dict(self): + return cs_error( + self.msg, + self.errcode, + consent_uri=self._consent_uri + ) + + class RegistrationError(SynapseError): """An error raised when a registration event fails.""" pass @@ -292,7 +321,7 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs): Args: msg (str): The error message. - code (int): The error code. + code (str): The error code. kwargs : Additional keys to add to the response. Returns: A dict representing the error response JSON. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 91a33a3402..bb46b5da8a 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +15,12 @@ # limitations under the License. """Contains the URL paths to prefix various aspects of the server with. """ +from hashlib import sha256 +import hmac + +from six.moves.urllib.parse import urlencode + +from synapse.config import ConfigError CLIENT_PREFIX = "/_matrix/client/api/v1" CLIENT_V2_ALPHA_PREFIX = "/_matrix/client/v2_alpha" @@ -25,3 +32,46 @@ SERVER_KEY_PREFIX = "/_matrix/key/v1" SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" MEDIA_PREFIX = "/_matrix/media/r0" LEGACY_MEDIA_PREFIX = "/_matrix/media/v1" + + +class ConsentURIBuilder(object): + def __init__(self, hs_config): + """ + Args: + hs_config (synapse.config.homeserver.HomeServerConfig): + """ + if hs_config.form_secret is None: + raise ConfigError( + "form_secret not set in config", + ) + if hs_config.public_baseurl is None: + raise ConfigError( + "public_baseurl not set in config", + ) + + self._hmac_secret = hs_config.form_secret.encode("utf-8") + self._public_baseurl = hs_config.public_baseurl + + def build_user_consent_uri(self, user_id): + """Build a URI which we can give to the user to do their privacy + policy consent + + Args: + user_id (str): mxid or username of user + + Returns + (str) the URI where the user can do consent + """ + mac = hmac.new( + key=self._hmac_secret, + msg=user_id, + digestmod=sha256, + ).hexdigest() + consent_uri = "%s_matrix/consent?%s" % ( + self._public_baseurl, + urlencode({ + "u": user_id, + "h": mac + }), + ) + return consent_uri diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bceb21a8d5..caccbaa814 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -41,7 +41,6 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource -from synapse.rest.consent.consent_resource import ConsentResource from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -186,6 +185,7 @@ class SynapseHomeServer(HomeServer): }) if name == "consent": + from synapse.rest.consent.consent_resource import ConsentResource consent_resource = ConsentResource(self) if compress: consent_resource = gz_wrap(consent_resource) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 8698b2993f..ddcd305f4c 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -34,13 +34,22 @@ DEFAULT_CONFIG = """\ # asking them to consent to the privacy policy. The 'server_notices' section # must also be configured for this to work. # +# 'block_events_error', if set, will block any attempts to send events +# until the user consents to the privacy policy. The value of the setting is +# used as the text of the error. +# # user_consent: # template_dir: res/templates/privacy # version: 1.0 # server_notice_content: # msgtype: m.text -# body: | -# Pls do consent kthx +# body: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# block_events_error: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# """ @@ -51,6 +60,7 @@ class ConsentConfig(Config): self.user_consent_version = None self.user_consent_template_dir = None self.user_consent_server_notice_content = None + self.block_events_without_consent_error = None def read_config(self, config): consent_config = config.get("user_consent") @@ -61,6 +71,9 @@ class ConsentConfig(Config): self.user_consent_server_notice_content = consent_config.get( "server_notice_content", ) + self.block_events_without_consent_error = consent_config.get( + "block_events_error", + ) def default_config(self, **kwargs): return DEFAULT_CONFIG diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py index ccef8d2ec5..be1d1f762c 100644 --- a/synapse/config/server_notices_config.py +++ b/synapse/config/server_notices_config.py @@ -26,12 +26,13 @@ DEFAULT_CONFIG = """\ # setting, which defines the id of the user which will be used to send the # notices. # -# It's also possible to override the room name, or the display name of the -# "notices" user. +# It's also possible to override the room name, the display name of the +# "notices" user, and the avatar for the user. # # server_notices: # system_mxid_localpart: notices # system_mxid_display_name: "Server Notices" +# system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" # room_name: "Server Notices" """ @@ -48,6 +49,10 @@ class ServerNoticesConfig(Config): The display name to use for the server notices user. None if server notices are not enabled. + server_notices_mxid_avatar_url (str|None): + The display name to use for the server notices user. + None if server notices are not enabled. + server_notices_room_name (str|None): The name to use for the server notices room. None if server notices are not enabled. @@ -56,6 +61,7 @@ class ServerNoticesConfig(Config): super(ServerNoticesConfig, self).__init__() self.server_notices_mxid = None self.server_notices_mxid_display_name = None + self.server_notices_mxid_avatar_url = None self.server_notices_room_name = None def read_config(self, config): @@ -68,7 +74,10 @@ class ServerNoticesConfig(Config): mxid_localpart, self.server_name, ).to_string() self.server_notices_mxid_display_name = c.get( - 'system_mxid_display_name', 'Server Notices', + 'system_mxid_display_name', None, + ) + self.server_notices_mxid_avatar_url = c.get( + 'system_mxid_avatar_url', None, ) # todo: i18n self.server_notices_room_name = c.get('room_name', "Server Notices") diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 4eb18775e8..d58ea6c650 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -61,7 +61,7 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) - # Add the user to a table of users penpding deactivation (ie. + # Add the user to a table of users pending deactivation (ie. # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 786c200378..2c72beff2e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -479,18 +479,18 @@ class FederationHandler(BaseHandler): # to get all state ids that we're interested in. event_map = yield self.store.get_events([ e_id - for key_to_eid in event_to_state_ids.values() - for key, e_id in key_to_eid.items() + for key_to_eid in event_to_state_ids.itervalues() + for key, e_id in key_to_eid.iteritems() if key[0] != EventTypes.Member or check_match(key[1]) ]) event_to_state = { e_id: { key: event_map[inner_e_id] - for key, inner_e_id in key_to_eid.items() + for key, inner_e_id in key_to_eid.iteritems() if inner_e_id in event_map } - for e_id, key_to_eid in event_to_state_ids.items() + for e_id, key_to_eid in event_to_state_ids.iteritems() } def redact_disallowed(event, state): @@ -505,7 +505,7 @@ class FederationHandler(BaseHandler): # membership states for the requesting server to determine # if the server is either in the room or has been invited # into the room. - for ev in state.values(): + for ev in state.itervalues(): if ev.type != EventTypes.Member: continue try: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 8343b5839d..c32b9bcae4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -20,10 +20,15 @@ import sys from canonicaljson import encode_canonical_json import six from twisted.internet import defer, reactor +from twisted.internet.defer import succeed from twisted.python.failure import Failure from synapse.api.constants import EventTypes, Membership, MAX_DEPTH -from synapse.api.errors import AuthError, Codes, SynapseError +from synapse.api.errors import ( + AuthError, Codes, SynapseError, + ConsentNotGivenError, +) +from synapse.api.urls import ConsentURIBuilder from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -431,6 +436,9 @@ class EventCreationHandler(object): self.spam_checker = hs.get_spam_checker() + if self.config.block_events_without_consent_error is not None: + self._consent_uri_builder = ConsentURIBuilder(self.config) + @defer.inlineCallbacks def create_event(self, requester, event_dict, token_id=None, txn_id=None, prev_events_and_hashes=None): @@ -482,6 +490,10 @@ class EventCreationHandler(object): target, e ) + is_exempt = yield self._is_exempt_from_privacy_policy(builder) + if not is_exempt: + yield self.assert_accepted_privacy_policy(requester) + if token_id is not None: builder.internal_metadata.token_id = token_id @@ -496,6 +508,83 @@ class EventCreationHandler(object): defer.returnValue((event, context)) + def _is_exempt_from_privacy_policy(self, builder): + """"Determine if an event to be sent is exempt from having to consent + to the privacy policy + + Args: + builder (synapse.events.builder.EventBuilder): event being created + + Returns: + Deferred[bool]: true if the event can be sent without the user + consenting + """ + # the only thing the user can do is join the server notices room. + if builder.type == EventTypes.Member: + membership = builder.content.get("membership", None) + if membership == Membership.JOIN: + return self._is_server_notices_room(builder.room_id) + return succeed(False) + + @defer.inlineCallbacks + def _is_server_notices_room(self, room_id): + if self.config.server_notices_mxid is None: + defer.returnValue(False) + user_ids = yield self.store.get_users_in_room(room_id) + defer.returnValue(self.config.server_notices_mxid in user_ids) + + @defer.inlineCallbacks + def assert_accepted_privacy_policy(self, requester): + """Check if a user has accepted the privacy policy + + Called when the given user is about to do something that requires + privacy consent. We see if the user is exempt and otherwise check that + they have given consent. If they have not, a ConsentNotGiven error is + raised. + + Args: + requester (synapse.types.Requester): + The user making the request + + Returns: + Deferred[None]: returns normally if the user has consented or is + exempt + + Raises: + ConsentNotGivenError: if the user has not given consent yet + """ + if self.config.block_events_without_consent_error is None: + return + + # exempt AS users from needing consent + if requester.app_service is not None: + return + + user_id = requester.user.to_string() + + # exempt the system notices user + if ( + self.config.server_notices_mxid is not None and + user_id == self.config.server_notices_mxid + ): + return + + u = yield self.store.get_user_by_id(user_id) + assert u is not None + if u["consent_version"] == self.config.user_consent_version: + return + + consent_uri = self._consent_uri_builder.build_user_consent_uri( + requester.user.localpart, + ) + msg = self.config.block_events_without_consent_error % { + 'consent_uri': consent_uri, + } + raise ConsentNotGivenError( + msg=msg, + consent_uri=consent_uri, + ) + @defer.inlineCallbacks def send_nonmember_event(self, requester, event, context, ratelimit=True): """ diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5e9fa95a2d..b5850db42f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -126,6 +126,10 @@ class RoomCreationHandler(BaseHandler): except Exception: raise SynapseError(400, "Invalid user_id: %s" % (i,)) + yield self.event_creation_handler.assert_accepted_privacy_policy( + requester, + ) + invite_3pid_list = config.get("invite_3pid", []) visibility = config.get("visibility", None) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5263f09aaa..82adfc8fdf 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -306,6 +306,7 @@ class RoomMemberHandler(object): raise SynapseError( http_client.FORBIDDEN, "You cannot leave this room", + errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, ) if effective_membership_state == Membership.INVITE: diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 306385601c..724911d1e6 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -95,8 +95,8 @@ class ConsentResource(Resource): # this is required by the request_handler wrapper self.clock = hs.get_clock() - self._default_consent_verison = hs.config.user_consent_version - if self._default_consent_verison is None: + self._default_consent_version = hs.config.user_consent_version + if self._default_consent_version is None: raise ConfigError( "Consent resource is enabled but user_consent section is " "missing in config file.", @@ -114,7 +114,10 @@ class ConsentResource(Resource): ) loader = jinja2.FileSystemLoader(consent_template_directory) - self._jinja_env = jinja2.Environment(loader=loader) + self._jinja_env = jinja2.Environment( + loader=loader, + autoescape=jinja2.select_autoescape(['html', 'htm', 'xml']), + ) if hs.config.form_secret is None: raise ConfigError( @@ -129,6 +132,7 @@ class ConsentResource(Resource): return NOT_DONE_YET @wrap_html_request_handler + @defer.inlineCallbacks def _async_render_GET(self, request): """ Args: @@ -136,16 +140,26 @@ class ConsentResource(Resource): """ version = parse_string(request, "v", - default=self._default_consent_verison) + default=self._default_consent_version) username = parse_string(request, "u", required=True) userhmac = parse_string(request, "h", required=True) self._check_hash(username, userhmac) + if username.startswith('@'): + qualified_user_id = username + else: + qualified_user_id = UserID(username, self.hs.hostname).to_string() + + u = yield self.store.get_user_by_id(qualified_user_id) + if u is None: + raise NotFoundError("Unknown user") + try: self._render_template( request, "%s.html" % (version,), user=username, userhmac=userhmac, version=version, + has_consented=(u["consent_version"] == version), ) except TemplateNotFound: raise NotFoundError("Unknown policy version") diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index 440f6b1cd4..a709802856 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -14,10 +14,13 @@ # limitations under the License. import logging +from six import (iteritems, string_types) from twisted.internet import defer from synapse.api.errors import SynapseError +from synapse.api.urls import ConsentURIBuilder from synapse.config import ConfigError +from synapse.types import get_localpart_from_id logger = logging.getLogger(__name__) @@ -52,6 +55,8 @@ class ConsentServerNotices(object): "key.", ) + self._consent_uri_builder = ConsentURIBuilder(hs.config) + @defer.inlineCallbacks def maybe_send_server_notice_to_user(self, user_id): """Check if we need to send a notice to this user, and does so if so @@ -81,10 +86,18 @@ class ConsentServerNotices(object): # we've already sent a notice to the user return - # need to send a message + # need to send a message. try: + consent_uri = self._consent_uri_builder.build_user_consent_uri( + get_localpart_from_id(user_id), + ) + content = copy_with_str_subst( + self._server_notice_content, { + 'consent_uri': consent_uri, + }, + ) yield self._server_notices_manager.send_notice( - user_id, self._server_notice_content, + user_id, content, ) yield self._store.user_set_consent_server_notice_sent( user_id, self._current_consent_version, @@ -93,3 +106,27 @@ class ConsentServerNotices(object): logger.error("Error sending server notice about user consent: %s", e) finally: self._users_in_progress.remove(user_id) + + +def copy_with_str_subst(x, substitutions): + """Deep-copy a structure, carrying out string substitions on any strings + + Args: + x (object): structure to be copied + substitutions (object): substitutions to be made - passed into the + string '%' operator + + Returns: + copy of x + """ + if isinstance(x, string_types): + return x % substitutions + if isinstance(x, dict): + return { + k: copy_with_str_subst(v, substitutions) for (k, v) in iteritems(x) + } + if isinstance(x, (list, tuple)): + return [copy_with_str_subst(y) for y in x] + + # assume it's uninterested and can be shallow-copied. + return x diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index f535b9c9da..a26deace53 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ class ServerNoticesManager(object): self._config = hs.config self._room_creation_handler = hs.get_room_creation_handler() self._event_creation_handler = hs.get_event_creation_handler() + self._is_mine_id = hs.is_mine_id def is_enabled(self): """Checks if server notices are enabled on this server. @@ -55,7 +56,7 @@ class ServerNoticesManager(object): event_content (dict): content of event to send Returns: - Deferrred[None] + Deferred[None] """ room_id = yield self.get_notice_room_for_user(user_id) @@ -89,6 +90,9 @@ class ServerNoticesManager(object): if not self.is_enabled(): raise Exception("Server notices not enabled") + assert self._is_mine_id(user_id), \ + "Cannot send server notices to remote users" + rooms = yield self._store.get_rooms_for_user_where_membership_is( user_id, [Membership.INVITE, Membership.JOIN], ) @@ -109,6 +113,19 @@ class ServerNoticesManager(object): # apparently no existing notice room: create a new one logger.info("Creating server notices room for %s", user_id) + # see if we want to override the profile info for the server user. + # note that if we want to override either the display name or the + # avatar, we have to use both. + join_profile = None + if ( + self._config.server_notices_mxid_display_name is not None or + self._config.server_notices_mxid_avatar_url is not None + ): + join_profile = { + "displayname": self._config.server_notices_mxid_display_name, + "avatar_url": self._config.server_notices_mxid_avatar_url, + } + requester = create_requester(system_mxid) info = yield self._room_creation_handler.create_room( requester, @@ -121,9 +138,7 @@ class ServerNoticesManager(object): "invite": (user_id,) }, ratelimit=False, - creator_join_profile={ - "displayname": self._config.server_notices_mxid_display_name, - }, + creator_join_profile=join_profile, ) room_id = info['room_id'] diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py index 25abb4ccf5..4a133026c3 100644 --- a/synapse/server_notices/worker_server_notices_sender.py +++ b/synapse/server_notices/worker_server_notices_sender.py @@ -32,7 +32,7 @@ class WorkerServerNoticesSender(object): Returns: Deferred """ - return defer.succeed() + return defer.succeed(None) def on_user_ip(self, user_id): """Called on the master when a worker process saw a client request. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index ea24710ad8..fb463c525a 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -684,8 +684,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. - limit (int): The maximum number of events to return. Zero or less - means no limit. + limit (int): The maximum number of events to return. event_filter (Filter|None): If provided filters the events to those that match the filter. @@ -694,6 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): as a list of _EventDictReturn and a token that points to the end of the result set. """ + + assert int(limit) >= 0 + # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -723,22 +725,17 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): bounds += " AND " + filter_clause args.extend(filter_args) - if int(limit) > 0: - args.append(int(limit)) - limit_str = " LIMIT ?" - else: - limit_str = "" + args.append(int(limit)) sql = ( "SELECT event_id, topological_ordering, stream_ordering" " FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," - " stream_ordering %(order)s %(limit)s" + " stream_ordering %(order)s LIMIT ?" ) % { "bounds": bounds, "order": order, - "limit": limit_str } txn.execute(sql, args) diff --git a/tests/utils.py b/tests/utils.py index 9626e6fd78..262c4a5714 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -64,6 +64,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.filter_timeline_limit = 5000 config.user_directory_search_all_users = False config.user_consent_server_notice_content = None + config.block_events_without_consent_error = None # disable user directory updates, because they get done in the # background, which upsets the test runner. |