diff options
24 files changed, 485 insertions, 74 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 851cb9b2d7..0569b581db 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,59 @@ +Changes in synapse v0.30.0 (2018-05-24) +========================================== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server policies (see ``docs/consent_tracking.md``), +however the intention is that they may also find a use for features such +as "Message of the day". + +This feature is specific to Synapse, but uses standard Matrix communication mechanisms, +so should work with any Matrix client. For more details see ``docs/server_notices.md`` + +Further Server Notices/Consent Tracking Support: + +* Allow overriding the server_notices user's avatar (PR #3273) +* Use the localpart in the consent uri (PR #3272) +* Support for putting %(consent_uri)s in messages (PR #3271) +* Block attempts to send server notices to remote users (PR #3270) +* Docs on consent bits (PR #3268) + + + +Changes in synapse v0.30.0-rc1 (2018-05-23) +========================================== + +Server Notices/Consent Tracking Support: + +* ConsentResource to gather policy consent from users (PR #3213) +* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) +* Infrastructure for a server notices room (PR #3232) +* Send users a server notice about consent (PR #3236) +* Reject attempts to send event before privacy consent is given (PR #3257) +* Add a 'has_consented' template var to consent forms (PR #3262) +* Fix dependency on jinja2 (PR #3263) + +Features: + +* Cohort analytics (PR #3163, #3241, #3251) +* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! +* Add in flight request metrics (PR #3252) + +Changes: + +* Remove unused `update_external_syncs` (PR #3233) +* Use stream rather depth ordering for push actions (PR #3212) +* Make purge_history operate on tokens (PR #3221) +* Don't support limitless pagination (PR #3265) + +Bug Fixes: + +* Fix logcontext resource usage tracking (PR #3258) +* Fix error in handling receipts (PR #3235) +* Stop the transaction cache caching failures (PR #3255) + + Changes in synapse v0.29.1 (2018-05-17) ========================================== Changes: diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md new file mode 100644 index 0000000000..064eae82f7 --- /dev/null +++ b/docs/consent_tracking.md @@ -0,0 +1,160 @@ +Support in Synapse for tracking agreement to server terms and conditions +======================================================================== + +Synapse 0.30 introduces support for tracking whether users have agreed to the +terms and conditions set by the administrator of a server - and blocking access +to the server until they have. + +There are several parts to this functionality; each requires some specific +configuration in `homeserver.yaml` to be enabled. + +Note that various parts of the configuation and this document refer to the +"privacy policy": agreement with a privacy policy is one particular use of this +feature, but of course adminstrators can specify other terms and conditions +unrelated to "privacy" per se. + +Collecting policy agreement from a user +--------------------------------------- + +Synapse can be configured to serve the user a simple policy form with an +"accept" button. Clicking "Accept" records the user's acceptance in the +database and shows a success page. + +To enable this, first create templates for the policy and success pages. +These should be stored on the local filesystem. + +These templates use the [Jinja2](http://jinja.pocoo.org) templating language, +and [docs/privacy_policy_templates](privacy_policy_templates) gives +examples of the sort of thing that can be done. + +Note that the templates must be stored under a name giving the language of the +template - currently this must always be `en` (for "English"); +internationalisation support is intended for the future. + +The template for the policy itself should be versioned and named according to +the version: for example `1.0.html`. The version of the policy which the user +has agreed to is stored in the database. + +Once the templates are in place, make the following changes to `homeserver.yaml`: + + 1. Add a `user_consent` section, which should look like: + + ```yaml + user_consent: + template_dir: privacy_policy_templates + version: 1.0 + ``` + + `template_dir` points to the directory containing the policy + templates. `version` defines the version of the policy which will be served + to the user. In the example above, Synapse will serve + `privacy_policy_templates/en/1.0.html`. + + + 2. Add a `form_secret` setting at the top level: + + + ```yaml + form_secret: "<unique secret>" + ``` + + This should be set to an arbitrary secret string (try `pwgen -y 30` to + generate suitable secrets). + + More on what this is used for below. + + 3. Add `consent` wherever the `client` resource is currently enabled in the + `listeners` configuration. For example: + + ```yaml + listeners: + - port: 8008 + resources: + - names: + - client + - consent + ``` + + +Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this +should be a matter of `pip install Jinja2`. On debian, try `apt-get install +python-jinja2`. + +Once this is complete, and the server has been restarted, try visiting +`https://<server>/_matrix/consent`. If correctly configured, this should give +an error "Missing string query parameter 'u'". It is now possible to manually +construct URIs where users can give their consent. + +### Constructing the consent URI + +It may be useful to manually construct the "consent URI" for a given user - for +instance, in order to send them an email asking them to consent. To do this, +take the base `https://<server>/_matrix/consent` URL and add the following +query parameters: + + * `u`: the user id of the user. This can either be a full MXID + (`@user:server.com`) or just the localpart (`user`). + + * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is + possible to calculate this on the commandline with something like: + + ```bash + echo -n '<user>' | openssl sha256 -hmac '<form_secret>' + ``` + + This should result in a URI which looks something like: + `https://<server>/_matrix/consent?u=<user>&h=68a152465a4d...`. + + +Sending users a server notice asking them to agree to the policy +---------------------------------------------------------------- + +It is possible to configure Synapse to send a [server +notice](server_notices.md) to anybody who has not yet agreed to the current +version of the policy. To do so: + + * ensure that the consent resource is configured, as in the previous section + + * ensure that server notices are configured, as in [server_notices.md](server_notices.md). + + * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For + example: + + ```yaml + user_consent: + server_notice_content: + msgtype: m.text + body: >- + Please give your consent to the privacy policy at %(consent_uri)s. + ``` + + Synapse automatically replaces the placeholder `%(consent_uri)s` with the + consent uri for that user. + + * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base + URI that clients use to connect to the server. (It is used to construct + `consent_uri` in the server notice.) + + +Blocking users from using the server until they agree to the policy +------------------------------------------------------------------- + +Synapse can be configured to block any attempts to join rooms or send messages +until the user has given their agreement to the policy. (Joining the server +notices room is exempted from this). + +To enable this, add `block_events_error` under `user_consent`. For example: + +```yaml +user_consent: + block_events_error: >- + You can't send any messages until you consent to the privacy policy at + %(consent_uri)s. +``` + +Synapse automatically replaces the placeholder `%(consent_uri)s` with the +consent uri for that user. + +ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base +URI that clients use to connect to the server. (It is used to construct +`consent_uri` in the error.) diff --git a/docs/manhole.md b/docs/manhole.md new file mode 100644 index 0000000000..7375f5ad46 --- /dev/null +++ b/docs/manhole.md @@ -0,0 +1,43 @@ +Using the synapse manhole +========================= + +The "manhole" allows server administrators to access a Python shell on a running +Synapse installation. This is a very powerful mechanism for administration and +debugging. + +To enable it, first uncomment the `manhole` listener configuration in +`homeserver.yaml`: + +```yaml +listeners: + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` + +(`bind_addresses` in the above is important: it ensures that access to the +manhole is only possible for local users). + +Note that this will give administrative access to synapse to **all users** with +shell access to the server. It should therefore **not** be enabled in +environments where untrusted users have shell access. + +Then restart synapse, and point an ssh client at port 9000 on localhost, using +the username `matrix`: + +```bash +ssh -p9000 matrix@localhost +``` + +The password is `rabbithole`. + +This gives a Python REPL in which `hs` gives access to the +`synapse.server.HomeServer` object - which in turn gives access to many other +parts of the process. + +As a simple example, retrieving an event from the database: + +``` +>>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org') +<Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>> +``` diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md deleted file mode 100644 index a3e6fc0986..0000000000 --- a/docs/privacy_policy_templates/README.md +++ /dev/null @@ -1,23 +0,0 @@ -If enabling the 'consent' resource in synapse, you will need some templates -for the HTML to be served to the user. This directory contains very simple -examples of the sort of thing that can be done. - -You'll need to add this sort of thing to your homeserver.yaml: - -``` -form_secret: <unique but arbitrary secret> - -user_consent: - template_dir: docs/privacy_policy_templates - version: 1.0 -``` - -You should then be able to enable the `consent` resource under a `listener` -entry. For example: - -``` -listeners: - - port: 8008 - resources: - - names: [client, consent] -``` diff --git a/docs/server_notices.md b/docs/server_notices.md new file mode 100644 index 0000000000..221553b24d --- /dev/null +++ b/docs/server_notices.md @@ -0,0 +1,71 @@ +Server Notices +============== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server polices(see +[consent_tracking.md](consent_tracking.md)), however the intention is that +they may also find a use for features such as "Message of the day". + +This is a feature specific to Synapse, but it uses standard Matrix +communication mechanisms, so should work with any Matrix client. + +User experience +--------------- + +When the user is first sent a server notice, they will get an invitation to a +room (typically called 'Server Notices', though this is configurable in +`homeserver.yaml`). They will be **unable to reject** this invitation - +attempts to do so will receive an error. + +Once they accept the invitation, they will see the notice message in the room +history; it will appear to have come from the 'server notices user' (see +below). + +The user is prevented from sending any messages in this room by the power +levels. They also cannot leave it. + +Synapse configuration +--------------------- + +Server notices come from a specific user id on the server. Server +administrators are free to choose the user id - something like `server` is +suggested, meaning the notices will come from +`@server:<your_server_name>`. Once the Server Notices user is configured, that +user id becomes a special, privileged user, so administrators should ensure +that **it is not already allocated**. + +In order to support server notices, it is necessary to add some configuration +to the `homeserver.yaml` file. In particular, you should add a `server_notices` +section, which should look like this: + +```yaml +server_notices: + system_mxid_localpart: server + system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" + room_name: "Server Notices" +``` + +The only compulsory setting is `system_mxid_localpart`, which defines the user +id of the Server Notices user, as above. `room_name` defines the name of the +room which will be created. + +`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the +displayname and avatar of the Server Notices user. + +Sending notices +--------------- + +As of the current version of synapse, there is no convenient interface for +sending notices (other than the automated ones sent as part of consent +tracking). + +In the meantime, it is possible to test this feature using the manhole. Having +gone into the manhole as described in [manhole.md](manhole.md), a notice can be +sent with something like: + +``` +>>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'}) +``` diff --git a/synapse/__init__.py b/synapse/__init__.py index b11a1d3ea3..5bada5e290 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.1" +__version__ = "0.30.0" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index aca804280c..e6ad3768f0 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -53,6 +53,7 @@ class Codes(object): INVALID_USERNAME = "M_INVALID_USERNAME" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" + CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" class CodeMessageException(RuntimeError): diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 44c4711e6c..ddcd305f4c 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -43,10 +43,13 @@ DEFAULT_CONFIG = """\ # version: 1.0 # server_notice_content: # msgtype: m.text -# body: | -# Pls do consent kthx -# block_events_error: | -# You can't send any messages until you consent to the privacy policy. +# body: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# block_events_error: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# """ diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py index ccef8d2ec5..be1d1f762c 100644 --- a/synapse/config/server_notices_config.py +++ b/synapse/config/server_notices_config.py @@ -26,12 +26,13 @@ DEFAULT_CONFIG = """\ # setting, which defines the id of the user which will be used to send the # notices. # -# It's also possible to override the room name, or the display name of the -# "notices" user. +# It's also possible to override the room name, the display name of the +# "notices" user, and the avatar for the user. # # server_notices: # system_mxid_localpart: notices # system_mxid_display_name: "Server Notices" +# system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" # room_name: "Server Notices" """ @@ -48,6 +49,10 @@ class ServerNoticesConfig(Config): The display name to use for the server notices user. None if server notices are not enabled. + server_notices_mxid_avatar_url (str|None): + The display name to use for the server notices user. + None if server notices are not enabled. + server_notices_room_name (str|None): The name to use for the server notices room. None if server notices are not enabled. @@ -56,6 +61,7 @@ class ServerNoticesConfig(Config): super(ServerNoticesConfig, self).__init__() self.server_notices_mxid = None self.server_notices_mxid_display_name = None + self.server_notices_mxid_avatar_url = None self.server_notices_room_name = None def read_config(self, config): @@ -68,7 +74,10 @@ class ServerNoticesConfig(Config): mxid_localpart, self.server_name, ).to_string() self.server_notices_mxid_display_name = c.get( - 'system_mxid_display_name', 'Server Notices', + 'system_mxid_display_name', None, + ) + self.server_notices_mxid_avatar_url = c.get( + 'system_mxid_avatar_url', None, ) # todo: i18n self.server_notices_room_name = c.get('room_name', "Server Notices") diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 824f4a42e3..29ae086786 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -20,6 +20,8 @@ from frozendict import frozendict import re +from six import string_types + # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' # (?<!stuff) matches if the current position in the string is not preceded # by a match for 'stuff'. @@ -277,7 +279,7 @@ def serialize_event(e, time_now_ms, as_client_event=True, if only_event_fields: if (not isinstance(only_event_fields, list) or - not all(isinstance(f, basestring) for f in only_event_fields)): + not all(isinstance(f, string_types) for f in only_event_fields)): raise TypeError("only_event_fields must be a list of strings") d = only_fields(d, only_event_fields) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 2f4c8a1018..e0e5bf818c 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -17,6 +17,8 @@ from synapse.types import EventID, RoomID, UserID from synapse.api.errors import SynapseError from synapse.api.constants import EventTypes, Membership +from six import string_types + class EventValidator(object): @@ -49,7 +51,7 @@ class EventValidator(object): strings.append("state_key") for s in strings: - if not isinstance(getattr(event, s), basestring): + if not isinstance(getattr(event, s), string_types): raise SynapseError(400, "Not '%s' a string type" % (s,)) if event.type == EventTypes.Member: @@ -88,5 +90,5 @@ class EventValidator(object): for s in keys: if s not in d: raise SynapseError(400, "'%s' not in content" % (s,)) - if not isinstance(d[s], basestring): + if not isinstance(d[s], string_types): raise SynapseError(400, "Not '%s' a string type" % (s,)) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 2d95b04e0c..62d20ad130 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -20,6 +20,8 @@ from synapse.api.errors import SynapseError from synapse.types import GroupID, RoomID, UserID, get_domain_from_id from twisted.internet import defer +from six import string_types + logger = logging.getLogger(__name__) @@ -431,7 +433,7 @@ class GroupsServerHandler(object): "long_description"): if keyname in content: value = content[keyname] - if not isinstance(value, basestring): + if not isinstance(value, string_types): raise SynapseError(400, "%r value is not a string" % (keyname,)) profile[keyname] = value diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index f92f953a79..ce6e2b14fe 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -32,6 +32,7 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() self._identity_handler = hs.get_handlers().identity_handler + self.user_directory_handler = hs.get_user_directory_handler() # Flag that indicates whether the process to part users from rooms is running self._user_parter_running = False @@ -88,6 +89,9 @@ class DeactivateAccountHandler(BaseHandler): # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) + # delete from user directory + yield self.user_directory_handler.handle_user_deactivated(user_id) + # Now start the process that goes through that list and # parts users from rooms (if it isn't already running) self._start_user_parting() diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba3ede8024..2c72beff2e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -479,18 +479,18 @@ class FederationHandler(BaseHandler): # to get all state ids that we're interested in. event_map = yield self.store.get_events([ e_id - for key_to_eid in event_to_state_ids.values() - for key, e_id in key_to_eid.items() + for key_to_eid in event_to_state_ids.itervalues() + for key, e_id in key_to_eid.iteritems() if key[0] != EventTypes.Member or check_match(key[1]) ]) event_to_state = { e_id: { key: event_map[inner_e_id] - for key, inner_e_id in key_to_eid.items() + for key, inner_e_id in key_to_eid.iteritems() if inner_e_id in event_map } - for e_id, key_to_eid in event_to_state_ids.items() + for e_id, key_to_eid in event_to_state_ids.iteritems() } def redact_disallowed(event, state): @@ -505,7 +505,7 @@ class FederationHandler(BaseHandler): # membership states for the requesting server to determine # if the server is either in the room or has been invited # into the room. - for ev in state.values(): + for ev in state.itervalues(): if ev.type != EventTypes.Member: continue try: @@ -751,9 +751,19 @@ class FederationHandler(BaseHandler): curr_state = yield self.state_handler.get_current_state(room_id) def get_domains_from_state(state): + """Get joined domains from state + + Args: + state (dict[tuple, FrozenEvent]): State map from type/state + key to event. + + Returns: + list[tuple[str, int]]: Returns a list of servers with the + lowest depth of their joins. Sorted by lowest depth first. + """ joined_users = [ (state_key, int(event.depth)) - for (e_type, state_key), event in state.items() + for (e_type, state_key), event in state.iteritems() if e_type == EventTypes.Member and event.membership == Membership.JOIN ] @@ -770,7 +780,7 @@ class FederationHandler(BaseHandler): except Exception: pass - return sorted(joined_domains.items(), key=lambda d: d[1]) + return sorted(joined_domains.iteritems(), key=lambda d: d[1]) curr_domains = get_domains_from_state(curr_state) @@ -787,7 +797,7 @@ class FederationHandler(BaseHandler): yield self.backfill( dom, room_id, limit=100, - extremities=[e for e in extremities.keys()] + extremities=extremities, ) # If this succeeded then we probably already have the # appropriate stuff. @@ -833,7 +843,7 @@ class FederationHandler(BaseHandler): tried_domains = set(likely_domains) tried_domains.add(self.server_name) - event_ids = list(extremities.keys()) + event_ids = list(extremities.iterkeys()) logger.debug("calling resolve_state_groups in _maybe_backfill") resolve = logcontext.preserve_fn( @@ -843,31 +853,34 @@ class FederationHandler(BaseHandler): [resolve(room_id, [e]) for e in event_ids], consumeErrors=True, )) + + # dict[str, dict[tuple, str]], a map from event_id to state map of + # event_ids. states = dict(zip(event_ids, [s.state for s in states])) state_map = yield self.store.get_events( - [e_id for ids in states.values() for e_id in ids], + [e_id for ids in states.itervalues() for e_id in ids.itervalues()], get_prev_content=False ) states = { key: { k: state_map[e_id] - for k, e_id in state_dict.items() + for k, e_id in state_dict.iteritems() if e_id in state_map - } for key, state_dict in states.items() + } for key, state_dict in states.iteritems() } for e_id, _ in sorted_extremeties_tuple: likely_domains = get_domains_from_state(states[e_id]) success = yield try_backfill([ - dom for dom in likely_domains + dom for dom, _ in likely_domains if dom not in tried_domains ]) if success: defer.returnValue(True) - tried_domains.update(likely_domains) + tried_domains.update(dom for dom, _ in likely_domains) defer.returnValue(False) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c3adbc6c95..c32b9bcae4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -574,9 +574,14 @@ class EventCreationHandler(object): if u["consent_version"] == self.config.user_consent_version: return - consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id) + consent_uri = self._consent_uri_builder.build_user_consent_uri( + requester.user.localpart, + ) + msg = self.config.block_events_without_consent_error % { + 'consent_uri': consent_uri, + } raise ConsentNotGivenError( - msg=self.config.block_events_without_consent_error, + msg=msg, consent_uri=consent_uri, ) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5263f09aaa..82adfc8fdf 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -306,6 +306,7 @@ class RoomMemberHandler(object): raise SynapseError( http_client.FORBIDDEN, "You cannot leave this room", + errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, ) if effective_membership_state == Membership.INVITE: diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 714f0195c8..2f73d664f9 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -123,6 +123,13 @@ class UserDirectoryHandler(object): ) @defer.inlineCallbacks + def handle_user_deactivated(self, user_id): + """Called when a user ID is deactivated + """ + yield self.store.remove_from_user_dir(user_id) + yield self.store.remove_from_user_in_public_room(user_id) + + @defer.inlineCallbacks def _unsafe_process(self): # If self.pos is None then means we haven't fetched it from DB if self.pos is None: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 21eaf77dc4..c7f919498c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -42,6 +42,8 @@ import random import sys import urllib from six.moves.urllib import parse as urlparse +from six import string_types + logger = logging.getLogger(__name__) outbound_logger = logging.getLogger("synapse.http.outbound") @@ -558,7 +560,7 @@ class MatrixFederationHttpClient(object): encoded_args = {} for k, vs in args.items(): - if isinstance(vs, basestring): + if isinstance(vs, string_types): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] @@ -673,7 +675,7 @@ def check_content_type_is_json(headers): RuntimeError if the """ - c_type = headers.getRawHeaders("Content-Type") + c_type = headers.getRawHeaders(b"Content-Type") if c_type is None: raise RuntimeError( "No Content-Type header" @@ -690,7 +692,7 @@ def check_content_type_is_json(headers): def encode_query_args(args): encoded_args = {} for k, vs in args.items(): - if isinstance(vs, basestring): + if isinstance(vs, string_types): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 3601f2d365..d55efde8cc 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -21,6 +21,8 @@ from synapse.types import UserID from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache from synapse.util.caches.lrucache import LruCache +from six import string_types + logger = logging.getLogger(__name__) @@ -238,7 +240,7 @@ def _flatten_dict(d, prefix=[], result=None): if result is None: result = {} for key, value in d.items(): - if isinstance(value, basestring): + if isinstance(value, string_types): result[".".join(prefix + [key])] = value.lower() elif hasattr(value, "items"): _flatten_dict(value, prefix=(prefix + [key]), result=result) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 4a73813c58..647994bd53 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -23,6 +23,8 @@ from synapse.handlers.presence import format_user_presence_state from synapse.http.servlet import parse_json_object_from_request from .base import ClientV1RestServlet, client_path_patterns +from six import string_types + import logging logger = logging.getLogger(__name__) @@ -71,7 +73,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet): if "status_msg" in content: state["status_msg"] = content.pop("status_msg") - if not isinstance(state["status_msg"], basestring): + if not isinstance(state["status_msg"], string_types): raise SynapseError(400, "status_msg must be a string.") if content: @@ -129,7 +131,7 @@ class PresenceListRestServlet(ClientV1RestServlet): if "invite" in content: for u in content["invite"]: - if not isinstance(u, basestring): + if not isinstance(u, string_types): raise SynapseError(400, "Bad invite value.") if len(u) == 0: continue @@ -140,7 +142,7 @@ class PresenceListRestServlet(ClientV1RestServlet): if "drop" in content: for u in content["drop"]: - if not isinstance(u, basestring): + if not isinstance(u, string_types): raise SynapseError(400, "Bad drop value.") if len(u) == 0: continue diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index 440f6b1cd4..a709802856 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -14,10 +14,13 @@ # limitations under the License. import logging +from six import (iteritems, string_types) from twisted.internet import defer from synapse.api.errors import SynapseError +from synapse.api.urls import ConsentURIBuilder from synapse.config import ConfigError +from synapse.types import get_localpart_from_id logger = logging.getLogger(__name__) @@ -52,6 +55,8 @@ class ConsentServerNotices(object): "key.", ) + self._consent_uri_builder = ConsentURIBuilder(hs.config) + @defer.inlineCallbacks def maybe_send_server_notice_to_user(self, user_id): """Check if we need to send a notice to this user, and does so if so @@ -81,10 +86,18 @@ class ConsentServerNotices(object): # we've already sent a notice to the user return - # need to send a message + # need to send a message. try: + consent_uri = self._consent_uri_builder.build_user_consent_uri( + get_localpart_from_id(user_id), + ) + content = copy_with_str_subst( + self._server_notice_content, { + 'consent_uri': consent_uri, + }, + ) yield self._server_notices_manager.send_notice( - user_id, self._server_notice_content, + user_id, content, ) yield self._store.user_set_consent_server_notice_sent( user_id, self._current_consent_version, @@ -93,3 +106,27 @@ class ConsentServerNotices(object): logger.error("Error sending server notice about user consent: %s", e) finally: self._users_in_progress.remove(user_id) + + +def copy_with_str_subst(x, substitutions): + """Deep-copy a structure, carrying out string substitions on any strings + + Args: + x (object): structure to be copied + substitutions (object): substitutions to be made - passed into the + string '%' operator + + Returns: + copy of x + """ + if isinstance(x, string_types): + return x % substitutions + if isinstance(x, dict): + return { + k: copy_with_str_subst(v, substitutions) for (k, v) in iteritems(x) + } + if isinstance(x, (list, tuple)): + return [copy_with_str_subst(y) for y in x] + + # assume it's uninterested and can be shallow-copied. + return x diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index f535b9c9da..a26deace53 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ class ServerNoticesManager(object): self._config = hs.config self._room_creation_handler = hs.get_room_creation_handler() self._event_creation_handler = hs.get_event_creation_handler() + self._is_mine_id = hs.is_mine_id def is_enabled(self): """Checks if server notices are enabled on this server. @@ -55,7 +56,7 @@ class ServerNoticesManager(object): event_content (dict): content of event to send Returns: - Deferrred[None] + Deferred[None] """ room_id = yield self.get_notice_room_for_user(user_id) @@ -89,6 +90,9 @@ class ServerNoticesManager(object): if not self.is_enabled(): raise Exception("Server notices not enabled") + assert self._is_mine_id(user_id), \ + "Cannot send server notices to remote users" + rooms = yield self._store.get_rooms_for_user_where_membership_is( user_id, [Membership.INVITE, Membership.JOIN], ) @@ -109,6 +113,19 @@ class ServerNoticesManager(object): # apparently no existing notice room: create a new one logger.info("Creating server notices room for %s", user_id) + # see if we want to override the profile info for the server user. + # note that if we want to override either the display name or the + # avatar, we have to use both. + join_profile = None + if ( + self._config.server_notices_mxid_display_name is not None or + self._config.server_notices_mxid_avatar_url is not None + ): + join_profile = { + "displayname": self._config.server_notices_mxid_display_name, + "avatar_url": self._config.server_notices_mxid_avatar_url, + } + requester = create_requester(system_mxid) info = yield self._room_creation_handler.create_room( requester, @@ -121,9 +138,7 @@ class ServerNoticesManager(object): "invite": (user_id,) }, ratelimit=False, - creator_join_profile={ - "displayname": self._config.server_notices_mxid_display_name, - }, + creator_join_profile=join_profile, ) room_id = info['room_id'] diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py index 25abb4ccf5..4a133026c3 100644 --- a/synapse/server_notices/worker_server_notices_sender.py +++ b/synapse/server_notices/worker_server_notices_sender.py @@ -32,7 +32,7 @@ class WorkerServerNoticesSender(object): Returns: Deferred """ - return defer.succeed() + return defer.succeed(None) def on_user_ip(self, user_id): """Called on the master when a worker process saw a client request. diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index ea24710ad8..fb463c525a 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -684,8 +684,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. - limit (int): The maximum number of events to return. Zero or less - means no limit. + limit (int): The maximum number of events to return. event_filter (Filter|None): If provided filters the events to those that match the filter. @@ -694,6 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): as a list of _EventDictReturn and a token that points to the end of the result set. """ + + assert int(limit) >= 0 + # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -723,22 +725,17 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): bounds += " AND " + filter_clause args.extend(filter_args) - if int(limit) > 0: - args.append(int(limit)) - limit_str = " LIMIT ?" - else: - limit_str = "" + args.append(int(limit)) sql = ( "SELECT event_id, topological_ordering, stream_ordering" " FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," - " stream_ordering %(order)s %(limit)s" + " stream_ordering %(order)s LIMIT ?" ) % { "bounds": bounds, "order": order, - "limit": limit_str } txn.execute(sql, args) |