Merge branch 'release-v0.30.0' into develop

8 files changed, 128 insertions, 19 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 851cb9b2d7..096fa4794a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,40 @@
+Changes in synapse v0.30.0-rc1 (2018-05-23)
+==========================================
+
+This version includes support for Privacy Notice agreement collection. More
+details live at docs/privacy_policy_templates/README.md
+
+
+GDPR Support:
+
+* ConsentResource to gather policy consent from users (PR #3213)
+* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225)
+* Infrastructure for a server notices room (PR #3232)
+* Send users a server notice about consent (PR #3236)
+* Reject attempts to send event before privacy consent is given (PR #3257)
+* Add a 'has_consented' template var to consent forms (PR #3262)
+* Fix dependency on jinja2 (PR #3263)
+
+Features:
+
+* Cohort analytics (PR #3163, #3241, #3251)
+* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman!
+* Add in flight request metrics (PR #3252)
+
+Changes:
+
+* Remove unused `update_external_syncs` (PR #3233)
+* Use stream rather depth ordering for push actions (PR #3212)
+* Make purge_history operate on tokens (PR #3221)
+* Don't support limitless pagination (PR #3265)
+
+Bug Fixes:
+
+* Fix logcontext resource usage tracking (PR #3258)
+* Fix error in handling receipts (PR #3235)
+* Stop the transaction cache caching failures (PR #3255)
+
+
 Changes in synapse v0.29.1 (2018-05-17)
 ==========================================
 Changes:
diff --git a/docs/server_notices.md b/docs/server_notices.md
index 8e18e3d95d..221553b24d 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -44,13 +44,16 @@ section, which should look like this:
 server_notices:
    system_mxid_localpart: server
    system_mxid_display_name: "Server Notices"
+   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
    room_name: "Server Notices"
 ```
 
 The only compulsory setting is `system_mxid_localpart`, which defines the user
-id of the server notices user, as above. `system_mxid_display_name` and
-`room_name` define the displayname of the system notices user, and of
-the notices room, respectively.
+id of the Server Notices user, as above. `room_name` defines the name of the
+room which will be created.
+
+`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
+displayname and avatar of the Server Notices user.
 
 Sending notices
 ---------------
diff --git a/synapse/__init__.py b/synapse/__init__.py
index b11a1d3ea3..353387f154 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.29.1"
+__version__ = "0.30.0-rc1"
diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 44c4711e6c..ddcd305f4c 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -43,10 +43,13 @@ DEFAULT_CONFIG = """\
 #   version: 1.0
 #   server_notice_content:
 #     msgtype: m.text
-#     body: |
-#       Pls do consent kthx
-#   block_events_error: |
-#     You can't send any messages until you consent to the privacy policy.
+#     body: >-
+#       To continue using this homeserver you must review and agree to the
+#       terms and conditions at %(consent_uri)s
+#   block_events_error: >-
+#     To continue using this homeserver you must review and agree to the
+#     terms and conditions at %(consent_uri)s
+#
 """
 
 
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
index ccef8d2ec5..be1d1f762c 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices_config.py
@@ -26,12 +26,13 @@ DEFAULT_CONFIG = """\
 # setting, which defines the id of the user which will be used to send the
 # notices.
 #
-# It's also possible to override the room name, or the display name of the
-# "notices" user.
+# It's also possible to override the room name, the display name of the
+# "notices" user, and the avatar for the user.
 #
 # server_notices:
 #   system_mxid_localpart: notices
 #   system_mxid_display_name: "Server Notices"
+#   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
 #   room_name: "Server Notices"
 """
 
@@ -48,6 +49,10 @@ class ServerNoticesConfig(Config):
             The display name to use for the server notices user.
             None if server notices are not enabled.
 
+        server_notices_mxid_avatar_url (str|None):
+            The display name to use for the server notices user.
+            None if server notices are not enabled.
+
         server_notices_room_name (str|None):
             The name to use for the server notices room.
             None if server notices are not enabled.
@@ -56,6 +61,7 @@ class ServerNoticesConfig(Config):
         super(ServerNoticesConfig, self).__init__()
         self.server_notices_mxid = None
         self.server_notices_mxid_display_name = None
+        self.server_notices_mxid_avatar_url = None
         self.server_notices_room_name = None
 
     def read_config(self, config):
@@ -68,7 +74,10 @@ class ServerNoticesConfig(Config):
             mxid_localpart, self.server_name,
         ).to_string()
         self.server_notices_mxid_display_name = c.get(
-            'system_mxid_display_name', 'Server Notices',
+            'system_mxid_display_name', None,
+        )
+        self.server_notices_mxid_avatar_url = c.get(
+            'system_mxid_avatar_url', None,
         )
         # todo: i18n
         self.server_notices_room_name = c.get('room_name', "Server Notices")
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index c3adbc6c95..c32b9bcae4 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -574,9 +574,14 @@ class EventCreationHandler(object):
         if u["consent_version"] == self.config.user_consent_version:
             return
 
-        consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id)
+        consent_uri = self._consent_uri_builder.build_user_consent_uri(
+            requester.user.localpart,
+        )
+        msg = self.config.block_events_without_consent_error % {
+            'consent_uri': consent_uri,
+        }
         raise ConsentNotGivenError(
-            msg=self.config.block_events_without_consent_error,
+            msg=msg,
             consent_uri=consent_uri,
         )
 
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index 440f6b1cd4..a709802856 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -14,10 +14,13 @@
 # limitations under the License.
 import logging
 
+from six import (iteritems, string_types)
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
+from synapse.api.urls import ConsentURIBuilder
 from synapse.config import ConfigError
+from synapse.types import get_localpart_from_id
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +55,8 @@ class ConsentServerNotices(object):
                     "key.",
                 )
 
+            self._consent_uri_builder = ConsentURIBuilder(hs.config)
+
     @defer.inlineCallbacks
     def maybe_send_server_notice_to_user(self, user_id):
         """Check if we need to send a notice to this user, and does so if so
@@ -81,10 +86,18 @@ class ConsentServerNotices(object):
                 # we've already sent a notice to the user
                 return
 
-            # need to send a message
+            # need to send a message.
             try:
+                consent_uri = self._consent_uri_builder.build_user_consent_uri(
+                    get_localpart_from_id(user_id),
+                )
+                content = copy_with_str_subst(
+                    self._server_notice_content, {
+                        'consent_uri': consent_uri,
+                    },
+                )
                 yield self._server_notices_manager.send_notice(
-                    user_id, self._server_notice_content,
+                    user_id, content,
                 )
                 yield self._store.user_set_consent_server_notice_sent(
                     user_id, self._current_consent_version,
@@ -93,3 +106,27 @@ class ConsentServerNotices(object):
                 logger.error("Error sending server notice about user consent: %s", e)
         finally:
             self._users_in_progress.remove(user_id)
+
+
+def copy_with_str_subst(x, substitutions):
+    """Deep-copy a structure, carrying out string substitions on any strings
+
+    Args:
+        x (object): structure to be copied
+        substitutions (object): substitutions to be made - passed into the
+            string '%' operator
+
+    Returns:
+        copy of x
+    """
+    if isinstance(x, string_types):
+        return x % substitutions
+    if isinstance(x, dict):
+        return {
+            k: copy_with_str_subst(v, substitutions) for (k, v) in iteritems(x)
+        }
+    if isinstance(x, (list, tuple)):
+        return [copy_with_str_subst(y) for y in x]
+
+    # assume it's uninterested and can be shallow-copied.
+    return x
diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py
index f535b9c9da..a26deace53 100644
--- a/synapse/server_notices/server_notices_manager.py
+++ b/synapse/server_notices/server_notices_manager.py
@@ -35,6 +35,7 @@ class ServerNoticesManager(object):
         self._config = hs.config
         self._room_creation_handler = hs.get_room_creation_handler()
         self._event_creation_handler = hs.get_event_creation_handler()
+        self._is_mine_id = hs.is_mine_id
 
     def is_enabled(self):
         """Checks if server notices are enabled on this server.
@@ -55,7 +56,7 @@ class ServerNoticesManager(object):
             event_content (dict): content of event to send
 
         Returns:
-            Deferrred[None]
+            Deferred[None]
         """
         room_id = yield self.get_notice_room_for_user(user_id)
 
@@ -89,6 +90,9 @@ class ServerNoticesManager(object):
         if not self.is_enabled():
             raise Exception("Server notices not enabled")
 
+        assert self._is_mine_id(user_id), \
+            "Cannot send server notices to remote users"
+
         rooms = yield self._store.get_rooms_for_user_where_membership_is(
             user_id, [Membership.INVITE, Membership.JOIN],
         )
@@ -109,6 +113,19 @@ class ServerNoticesManager(object):
         # apparently no existing notice room: create a new one
         logger.info("Creating server notices room for %s", user_id)
 
+        # see if we want to override the profile info for the server user.
+        # note that if we want to override either the display name or the
+        # avatar, we have to use both.
+        join_profile = None
+        if (
+            self._config.server_notices_mxid_display_name is not None or
+            self._config.server_notices_mxid_avatar_url is not None
+        ):
+            join_profile = {
+                "displayname": self._config.server_notices_mxid_display_name,
+                "avatar_url": self._config.server_notices_mxid_avatar_url,
+            }
+
         requester = create_requester(system_mxid)
         info = yield self._room_creation_handler.create_room(
             requester,
@@ -121,9 +138,7 @@ class ServerNoticesManager(object):
                 "invite": (user_id,)
             },
             ratelimit=False,
-            creator_join_profile={
-                "displayname": self._config.server_notices_mxid_display_name,
-            },
+            creator_join_profile=join_profile,
         )
         room_id = info['room_id']