From 47815edcfae73c5b938f8354853a09c0b80ef27e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 11 May 2018 00:17:11 +0100
Subject: ConsentResource to gather policy consent from users

Hopefully there are enough comments and docs in this that it makes sense on its
own.
---
 synapse/rest/consent/__init__.py         |   0
 synapse/rest/consent/consent_resource.py | 210 +++++++++++++++++++++++++++++++
 2 files changed, 210 insertions(+)
 create mode 100644 synapse/rest/consent/__init__.py
 create mode 100644 synapse/rest/consent/consent_resource.py

(limited to 'synapse/rest')

diff --git a/synapse/rest/consent/__init__.py b/synapse/rest/consent/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
new file mode 100644
index 0000000000..d791302278
--- /dev/null
+++ b/synapse/rest/consent/consent_resource.py
@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from hashlib import sha256
+import hmac
+import logging
+from os import path
+from six.moves import http_client
+
+import jinja2
+from jinja2 import TemplateNotFound
+from twisted.internet import defer
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+
+from synapse.api.errors import NotFoundError, SynapseError, StoreError
+from synapse.config import ConfigError
+from synapse.http.server import (
+    finish_request,
+    wrap_html_request_handler,
+)
+from synapse.http.servlet import parse_string
+from synapse.types import UserID
+
+
+# language to use for the templates. TODO: figure this out from Accept-Language
+TEMPLATE_LANGUAGE = "en"
+
+logger = logging.getLogger(__name__)
+
+# use hmac.compare_digest if we have it (python 2.7.7), else just use equality
+if hasattr(hmac, "compare_digest"):
+    compare_digest = hmac.compare_digest
+else:
+    def compare_digest(a, b):
+        return a == b
+
+
+class ConsentResource(Resource):
+    """A twisted Resource to display a privacy policy and gather consent to it
+
+    When accessed via GET, returns the privacy policy via a template.
+
+    When accessed via POST, records the user's consent in the database and
+    displays a success page.
+
+    The config should include a template_dir setting which contains templates
+    for the HTML. The directory should contain one subdirectory per language
+    (eg, 'en', 'fr'), and each language directory should contain the policy
+    document (named as '<version>.html') and a success page (success.html).
+
+    Both forms take a set of parameters from the browser. For the POST form,
+    these are normally sent as form parameters (but may be query-params); for
+    GET requests they must be query params. These are:
+
+        u: the complete mxid, or the localpart of the user giving their
+           consent. Required for both GET (where it is used as an input to the
+           template) and for POST (where it is used to find the row in the db
+           to update).
+
+        h: hmac_sha256(secret, u), where 'secret' is the privacy_secret in the
+           config file. If it doesn't match, the request is 403ed.
+
+        v: the version of the privacy policy being agreed to.
+
+           For GET: optional, and defaults to whatever was set in the config
+           file. Used to choose the version of the policy to pick from the
+           templates directory.
+
+           For POST: required; gives the value to be recorded in the database
+           against the user.
+    """
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): homeserver
+        """
+        Resource.__init__(self)
+
+        self.hs = hs
+        self.store = hs.get_datastore()
+
+        # this is required by the request_handler wrapper
+        self.clock = hs.get_clock()
+
+        consent_config = hs.config.consent_config
+        if consent_config is None:
+            raise ConfigError(
+                "Consent resource is enabled but user_consent section is "
+                "missing in config file.",
+            )
+
+        # daemonize changes the cwd to /, so make the path absolute now.
+        consent_template_directory = path.abspath(
+            consent_config["template_dir"],
+        )
+        if not path.isdir(consent_template_directory):
+            raise ConfigError(
+                "Could not find template directory '%s'" % (
+                    consent_template_directory,
+                ),
+            )
+
+        loader = jinja2.FileSystemLoader(consent_template_directory)
+        self._jinja_env = jinja2.Environment(loader=loader)
+
+        self._default_consent_verison = consent_config["default_version"]
+
+        if hs.config.form_secret is None:
+            raise ConfigError(
+                "Consent resource is enabled but form_secret is not set in "
+                "config file. It should be set to an arbitrary secret string.",
+            )
+
+        self._hmac_secret = hs.config.form_secret.encode("utf-8")
+
+    def render_GET(self, request):
+        self._async_render_GET(request)
+        return NOT_DONE_YET
+
+    @wrap_html_request_handler
+    def _async_render_GET(self, request):
+        """
+        Args:
+            request (twisted.web.http.Request):
+        """
+
+        version = parse_string(request, "v",
+                               default=self._default_consent_verison)
+        username = parse_string(request, "u", required=True)
+        userhmac = parse_string(request, "h", required=True)
+
+        self._check_hash(username, userhmac)
+
+        try:
+            self._render_template(
+                request, "%s.html" % (version,),
+                user=username, userhmac=userhmac, version=version,
+            )
+        except TemplateNotFound:
+            raise NotFoundError("Unknown policy version")
+
+    def render_POST(self, request):
+        self._async_render_POST(request)
+        return NOT_DONE_YET
+
+    @wrap_html_request_handler
+    @defer.inlineCallbacks
+    def _async_render_POST(self, request):
+        """
+        Args:
+            request (twisted.web.http.Request):
+        """
+        version = parse_string(request, "v", required=True)
+        username = parse_string(request, "u", required=True)
+        userhmac = parse_string(request, "h", required=True)
+
+        self._check_hash(username, userhmac)
+
+        if username.startswith('@'):
+            qualified_user_id = username
+        else:
+            qualified_user_id = UserID(username, self.hs.hostname).to_string()
+
+        try:
+            yield self.store.user_set_consent_version(qualified_user_id, version)
+        except StoreError as e:
+            if e.code != 404:
+                raise
+            raise NotFoundError("Unknown user")
+
+        try:
+            self._render_template(request, "success.html")
+        except TemplateNotFound:
+            raise NotFoundError("success.html not found")
+
+    def _render_template(self, request, template_name, **template_args):
+        # get_template checks for ".." so we don't need to worry too much
+        # about path traversal here.
+        template_html = self._jinja_env.get_template(
+            path.join(TEMPLATE_LANGUAGE, template_name)
+        )
+        html_bytes = template_html.render(**template_args).encode("utf8")
+
+        request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+        request.setHeader(b"Content-Length", b"%i" % len(html_bytes))
+        request.write(html_bytes)
+        finish_request(request)
+
+    def _check_hash(self, userid, userhmac):
+        want_mac = hmac.new(
+            key=self._hmac_secret,
+            msg=userid,
+            digestmod=sha256,
+        ).hexdigest()
+
+        if not compare_digest(want_mac, userhmac):
+            raise SynapseError(http_client.FORBIDDEN, "HMAC incorrect")
-- 
cgit 1.5.1


From 5f27ed75ad804ab9b5287f0deb8fd24b9a3a6232 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 15 May 2018 16:06:30 +0100
Subject: Make purge_history operate on tokens

As we're soon going to change how topological_ordering works
---
 synapse/handlers/message.py     | 12 ++++++------
 synapse/rest/client/v1/admin.py | 17 ++++++++++-------
 synapse/storage/events.py       | 17 +++++++++--------
 3 files changed, 25 insertions(+), 21 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index b793fc4df7..8343b5839d 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -86,14 +86,14 @@ class MessageHandler(BaseHandler):
         # map from purge id to PurgeStatus
         self._purges_by_id = {}
 
-    def start_purge_history(self, room_id, topological_ordering,
+    def start_purge_history(self, room_id, token,
                             delete_local_events=False):
         """Start off a history purge on a room.
 
         Args:
             room_id (str): The room to purge from
 
-            topological_ordering (int): minimum topo ordering to preserve
+            token (str): topological token to delete events before
             delete_local_events (bool): True to delete local events as well as
                 remote ones
 
@@ -115,19 +115,19 @@ class MessageHandler(BaseHandler):
         self._purges_by_id[purge_id] = PurgeStatus()
         run_in_background(
             self._purge_history,
-            purge_id, room_id, topological_ordering, delete_local_events,
+            purge_id, room_id, token, delete_local_events,
         )
         return purge_id
 
     @defer.inlineCallbacks
-    def _purge_history(self, purge_id, room_id, topological_ordering,
+    def _purge_history(self, purge_id, room_id, token,
                        delete_local_events):
         """Carry out a history purge on a room.
 
         Args:
             purge_id (str): The id for this purge
             room_id (str): The room to purge from
-            topological_ordering (int): minimum topo ordering to preserve
+            token (str): topological token to delete events before
             delete_local_events (bool): True to delete local events as well as
                 remote ones
 
@@ -138,7 +138,7 @@ class MessageHandler(BaseHandler):
         try:
             with (yield self.pagination_lock.write(room_id)):
                 yield self.store.purge_history(
-                    room_id, topological_ordering, delete_local_events,
+                    room_id, token, delete_local_events,
                 )
             logger.info("[purge] complete")
             self._purges_by_id[purge_id].status = PurgeStatus.STATUS_COMPLETE
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index efd5c9873d..282ce6be42 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -151,10 +151,11 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
             if event.room_id != room_id:
                 raise SynapseError(400, "Event is for wrong room.")
 
-            depth = event.depth
+            token = yield self.store.get_topological_token_for_event(event_id)
+
             logger.info(
-                "[purge] purging up to depth %i (event_id %s)",
-                depth, event_id,
+                "[purge] purging up to token %s (event_id %s)",
+                token, event_id,
             )
         elif 'purge_up_to_ts' in body:
             ts = body['purge_up_to_ts']
@@ -174,7 +175,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
                 )
             )
             if room_event_after_stream_ordering:
-                (_, depth, _) = room_event_after_stream_ordering
+                token = yield self.store.get_topological_token_for_event(
+                    room_event_after_stream_ordering,
+                )
             else:
                 logger.warn(
                     "[purge] purging events not possible: No event found "
@@ -187,9 +190,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
                     errcode=Codes.NOT_FOUND,
                 )
             logger.info(
-                "[purge] purging up to depth %i (received_ts %i => "
+                "[purge] purging up to token %d (received_ts %i => "
                 "stream_ordering %i)",
-                depth, ts, stream_ordering,
+                token, ts, stream_ordering,
             )
         else:
             raise SynapseError(
@@ -199,7 +202,7 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
             )
 
         purge_id = yield self.handlers.message_handler.start_purge_history(
-            room_id, depth,
+            room_id, token,
             delete_local_events=delete_local_events,
         )
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index b4ae6664f0..f65e18c1ee 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -33,7 +33,7 @@ from synapse.util.metrics import Measure
 from synapse.api.constants import EventTypes
 from synapse.api.errors import SynapseError
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
-from synapse.types import get_domain_from_id
+from synapse.types import get_domain_from_id, RoomStreamToken
 import synapse.metrics
 
 # these are only included to make the type annotations work
@@ -1803,15 +1803,14 @@ class EventsStore(EventsWorkerStore):
         return self.runInteraction("get_all_new_events", get_all_new_events_txn)
 
     def purge_history(
-        self, room_id, topological_ordering, delete_local_events,
+        self, room_id, token, delete_local_events,
     ):
         """Deletes room history before a certain point
 
         Args:
             room_id (str):
 
-            topological_ordering (int):
-                minimum topo ordering to preserve
+            token (str): A topological token to delete events before
 
             delete_local_events (bool):
                 if True, we will delete local events as well as remote ones
@@ -1821,12 +1820,12 @@ class EventsStore(EventsWorkerStore):
 
         return self.runInteraction(
             "purge_history",
-            self._purge_history_txn, room_id, topological_ordering,
+            self._purge_history_txn, room_id, token,
             delete_local_events,
         )
 
     def _purge_history_txn(
-        self, txn, room_id, topological_ordering, delete_local_events,
+        self, txn, room_id, token, delete_local_events,
     ):
         # Tables that should be pruned:
         #     event_auth
@@ -1856,6 +1855,8 @@ class EventsStore(EventsWorkerStore):
         # furthermore, we might already have the table from a previous (failed)
         # purge attempt, so let's drop the table first.
 
+        token = RoomStreamToken.parse(token)
+
         txn.execute("DROP TABLE IF EXISTS events_to_purge")
 
         txn.execute(
@@ -1888,7 +1889,7 @@ class EventsStore(EventsWorkerStore):
         rows = txn.fetchall()
         max_depth = max(row[0] for row in rows)
 
-        if max_depth <= topological_ordering:
+        if max_depth <= token.topological:
             # We need to ensure we don't delete all the events from the datanase
             # otherwise we wouldn't be able to send any events (due to not
             # having any backwards extremeties)
@@ -1904,7 +1905,7 @@ class EventsStore(EventsWorkerStore):
             should_delete_expr += " AND event_id NOT LIKE ?"
             should_delete_params += ("%:" + self.hs.hostname, )
 
-        should_delete_params += (room_id, topological_ordering)
+        should_delete_params += (room_id, token.topological)
 
         txn.execute(
             "INSERT INTO events_to_purge"
-- 
cgit 1.5.1


From c46367d0d755e810745cae6d0212bdaa12de7645 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 09:01:09 +0100
Subject: Move RoomCreationHandler out of synapse.handlers.Handlers

Handlers is deprecated nowadays, so let's move this out before I add a new
dependency on it.

Also fix the docstrings on create_room.
---
 synapse/handlers/__init__.py    | 5 +----
 synapse/handlers/room.py        | 8 ++++++--
 synapse/rest/client/v1/admin.py | 4 ++--
 synapse/rest/client/v1/room.py  | 5 ++---
 synapse/server.py               | 5 +++++
 synapse/server.pyi              | 3 +++
 6 files changed, 19 insertions(+), 11 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 8f8fd82eb0..d358842b3e 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -14,9 +14,7 @@
 # limitations under the License.
 
 from .register import RegistrationHandler
-from .room import (
-    RoomCreationHandler, RoomContextHandler,
-)
+from .room import RoomContextHandler
 from .message import MessageHandler
 from .federation import FederationHandler
 from .directory import DirectoryHandler
@@ -47,7 +45,6 @@ class Handlers(object):
     def __init__(self, hs):
         self.registration_handler = RegistrationHandler(hs)
         self.message_handler = MessageHandler(hs)
-        self.room_creation_handler = RoomCreationHandler(hs)
         self.federation_handler = FederationHandler(hs)
         self.directory_handler = DirectoryHandler(hs)
         self.admin_handler = AdminHandler(hs)
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 8df8fcbbad..e36426de5a 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -72,10 +72,14 @@ class RoomCreationHandler(BaseHandler):
         """ Creates a new room.
 
         Args:
-            requester (Requester): The user who requested the room creation.
+            requester (synapse.types.Requester):
+                The user who requested the room creation.
             config (dict) : A dict of configuration options.
+            ratelimit (bool): set to False to disable the rate limiter
         Returns:
-            The new room ID.
+            Deferred[dict]:
+                a dict containing the keys `room_id` and, if an alias was
+                requested, `room_alias`.
         Raises:
             SynapseError if the room ID couldn't be stored, or something went
             horribly wrong.
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index efd5c9873d..646a95c9fb 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -273,8 +273,8 @@ class ShutdownRoomRestServlet(ClientV1RestServlet):
     def __init__(self, hs):
         super(ShutdownRoomRestServlet, self).__init__(hs)
         self.store = hs.get_datastore()
-        self.handlers = hs.get_handlers()
         self.state = hs.get_state_handler()
+        self._room_creation_handler = hs.get_room_creation_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
         self.room_member_handler = hs.get_room_member_handler()
 
@@ -296,7 +296,7 @@ class ShutdownRoomRestServlet(ClientV1RestServlet):
         message = content.get("message", self.DEFAULT_MESSAGE)
         room_name = content.get("room_name", "Content Violation Notification")
 
-        info = yield self.handlers.room_creation_handler.create_room(
+        info = yield self._room_creation_handler.create_room(
             room_creator_requester,
             config={
                 "preset": "public_chat",
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index fcf9c9ab44..0b984987ed 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -41,7 +41,7 @@ class RoomCreateRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(RoomCreateRestServlet, self).__init__(hs)
-        self.handlers = hs.get_handlers()
+        self._room_creation_handler = hs.get_room_creation_handler()
 
     def register(self, http_server):
         PATTERNS = "/createRoom"
@@ -64,8 +64,7 @@ class RoomCreateRestServlet(ClientV1RestServlet):
     def on_POST(self, request):
         requester = yield self.auth.get_user_by_req(request)
 
-        handler = self.handlers.room_creation_handler
-        info = yield handler.create_room(
+        info = yield self._room_creation_handler.create_room(
             requester, self.get_room_config(request)
         )
 
diff --git a/synapse/server.py b/synapse/server.py
index 21cde5b6fc..9e6f3584b2 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -46,6 +46,7 @@ from synapse.handlers.devicemessage import DeviceMessageHandler
 from synapse.handlers.device import DeviceHandler
 from synapse.handlers.e2e_keys import E2eKeysHandler
 from synapse.handlers.presence import PresenceHandler
+from synapse.handlers.room import RoomCreationHandler
 from synapse.handlers.room_list import RoomListHandler
 from synapse.handlers.room_member import RoomMemberMasterHandler
 from synapse.handlers.room_member_worker import RoomMemberWorkerHandler
@@ -109,6 +110,7 @@ class HomeServer(object):
         'federation_server',
         'handlers',
         'auth',
+        'room_creation_handler',
         'state_handler',
         'state_resolution_handler',
         'presence_handler',
@@ -227,6 +229,9 @@ class HomeServer(object):
     def build_simple_http_client(self):
         return SimpleHttpClient(self)
 
+    def build_room_creation_handler(self):
+        return RoomCreationHandler(self)
+
     def build_state_handler(self):
         return StateHandler(self)
 
diff --git a/synapse/server.pyi b/synapse/server.pyi
index c3a9a3847b..aeda093f27 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -40,6 +40,9 @@ class HomeServer(object):
     def get_deactivate_account_handler(self) -> synapse.handlers.deactivate_account.DeactivateAccountHandler:
         pass
 
+    def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler:
+        pass
+
     def get_set_password_handler(self) -> synapse.handlers.set_password.SetPasswordHandler:
         pass
 
-- 
cgit 1.5.1


From 6d6e7288fe0200353740e03e6218a2781342f3e4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 21 May 2018 16:49:59 +0100
Subject: Stop the transaction cache caching failures

The transaction cache has some code which tries to stop it caching failures,
but if the callback function failed straight away, then things would happen
backwards and we'd end up with the failure stuck in the cache.
---
 synapse/rest/client/transactions.py    | 22 ++++++++------
 tests/rest/client/test_transactions.py | 54 ++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 9 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index fceca2edeb..93ce0f5348 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -87,19 +87,23 @@ class HttpTransactionCache(object):
 
         deferred = fn(*args, **kwargs)
 
-        # if the request fails with a Twisted failure, remove it
-        # from the transaction map. This is done to ensure that we don't
-        # cache transient errors like rate-limiting errors, etc.
+        observable = ObservableDeferred(deferred, consumeErrors=False)
+        self.transactions[txn_key] = (observable, self.clock.time_msec())
+
+        # if the request fails with an exception, remove it from the
+        # transaction map. This is done to ensure that we don't cache
+        # transient errors like rate-limiting errors, etc.
+        #
+        # (make sure we add this errback *after* adding the key above, in case
+        # the deferred has already failed and is running errbacks
+        # synchronously)
         def remove_from_map(err):
             self.transactions.pop(txn_key, None)
-            return err
+            # we deliberately do not propagate the error any further, as we
+            # expect the observers to have reported it.
+
         deferred.addErrback(remove_from_map)
 
-        # We don't add any other errbacks to the raw deferred, so we ask
-        # ObservableDeferred to swallow the error. This is fine as the error will
-        # still be reported to the observers.
-        observable = ObservableDeferred(deferred, consumeErrors=True)
-        self.transactions[txn_key] = (observable, self.clock.time_msec())
         return observable.observe()
 
     def _cleanup(self):
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index d7cea30260..b650a7772b 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -2,6 +2,8 @@ from synapse.rest.client.transactions import HttpTransactionCache
 from synapse.rest.client.transactions import CLEANUP_PERIOD_MS
 from twisted.internet import defer
 from mock import Mock, call
+
+from synapse.util.logcontext import LoggingContext
 from tests import unittest
 from tests.utils import MockClock
 
@@ -39,6 +41,58 @@ class HttpTransactionCacheTestCase(unittest.TestCase):
         # expect only a single call to do the work
         cb.assert_called_once_with("some_arg", keyword="arg", changing_args=0)
 
+    @defer.inlineCallbacks
+    def test_does_not_cache_exceptions(self):
+        """Checks that, if the callback throws an exception, it is called again
+        for the next request.
+        """
+        called = [False]
+
+        def cb():
+            if called[0]:
+                # return a valid result the second time
+                return defer.succeed(self.mock_http_response)
+
+            called[0] = True
+            raise Exception("boo")
+
+        with LoggingContext("test") as test_context:
+            try:
+                yield self.cache.fetch_or_execute(self.mock_key, cb)
+            except Exception as e:
+                self.assertEqual(e.message, "boo")
+            self.assertIs(LoggingContext.current_context(), test_context)
+
+            res = yield self.cache.fetch_or_execute(self.mock_key, cb)
+            self.assertEqual(res, self.mock_http_response)
+            self.assertIs(LoggingContext.current_context(), test_context)
+
+    @defer.inlineCallbacks
+    def test_does_not_cache_failures(self):
+        """Checks that, if the callback returns a failure, it is called again
+        for the next request.
+        """
+        called = [False]
+
+        def cb():
+            if called[0]:
+                # return a valid result the second time
+                return defer.succeed(self.mock_http_response)
+
+            called[0] = True
+            return defer.fail(Exception("boo"))
+
+        with LoggingContext("test") as test_context:
+            try:
+                yield self.cache.fetch_or_execute(self.mock_key, cb)
+            except Exception as e:
+                self.assertEqual(e.message, "boo")
+            self.assertIs(LoggingContext.current_context(), test_context)
+
+            res = yield self.cache.fetch_or_execute(self.mock_key, cb)
+            self.assertEqual(res, self.mock_http_response)
+            self.assertIs(LoggingContext.current_context(), test_context)
+
     @defer.inlineCallbacks
     def test_cleans_up(self):
         cb = Mock(
-- 
cgit 1.5.1


From 6e1cb54a05df3dd614acb022a665458cc1c9698f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 21 May 2018 16:58:20 +0100
Subject: Fix logcontext leak in HttpTransactionCache

ONE DAY I WILL PURGE THE WORLD OF THIS EVIL
---
 synapse/rest/client/transactions.py    | 46 ++++++++++++++++------------------
 tests/rest/client/test_transactions.py | 21 ++++++++++++++++
 2 files changed, 42 insertions(+), 25 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py
index 93ce0f5348..20fa6678ef 100644
--- a/synapse/rest/client/transactions.py
+++ b/synapse/rest/client/transactions.py
@@ -19,6 +19,7 @@ import logging
 
 from synapse.api.auth import get_access_token_from_request
 from synapse.util.async import ObservableDeferred
+from synapse.util.logcontext import make_deferred_yieldable, run_in_background
 
 logger = logging.getLogger(__name__)
 
@@ -80,31 +81,26 @@ class HttpTransactionCache(object):
         Returns:
             Deferred which resolves to a tuple of (response_code, response_dict).
         """
-        try:
-            return self.transactions[txn_key][0].observe()
-        except (KeyError, IndexError):
-            pass  # execute the function instead.
-
-        deferred = fn(*args, **kwargs)
-
-        observable = ObservableDeferred(deferred, consumeErrors=False)
-        self.transactions[txn_key] = (observable, self.clock.time_msec())
-
-        # if the request fails with an exception, remove it from the
-        # transaction map. This is done to ensure that we don't cache
-        # transient errors like rate-limiting errors, etc.
-        #
-        # (make sure we add this errback *after* adding the key above, in case
-        # the deferred has already failed and is running errbacks
-        # synchronously)
-        def remove_from_map(err):
-            self.transactions.pop(txn_key, None)
-            # we deliberately do not propagate the error any further, as we
-            # expect the observers to have reported it.
-
-        deferred.addErrback(remove_from_map)
-
-        return observable.observe()
+        if txn_key in self.transactions:
+            observable = self.transactions[txn_key][0]
+        else:
+            # execute the function instead.
+            deferred = run_in_background(fn, *args, **kwargs)
+
+            observable = ObservableDeferred(deferred)
+            self.transactions[txn_key] = (observable, self.clock.time_msec())
+
+            # if the request fails with an exception, remove it
+            # from the transaction map. This is done to ensure that we don't
+            # cache transient errors like rate-limiting errors, etc.
+            def remove_from_map(err):
+                self.transactions.pop(txn_key, None)
+                # we deliberately do not propagate the error any further, as we
+                # expect the observers to have reported it.
+
+            deferred.addErrback(remove_from_map)
+
+        return make_deferred_yieldable(observable.observe())
 
     def _cleanup(self):
         now = self.clock.time_msec()
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index b650a7772b..b5bc2fa255 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -3,6 +3,7 @@ from synapse.rest.client.transactions import CLEANUP_PERIOD_MS
 from twisted.internet import defer
 from mock import Mock, call
 
+from synapse.util import async
 from synapse.util.logcontext import LoggingContext
 from tests import unittest
 from tests.utils import MockClock
@@ -41,6 +42,26 @@ class HttpTransactionCacheTestCase(unittest.TestCase):
         # expect only a single call to do the work
         cb.assert_called_once_with("some_arg", keyword="arg", changing_args=0)
 
+    @defer.inlineCallbacks
+    def test_logcontexts_with_async_result(self):
+        @defer.inlineCallbacks
+        def cb():
+            yield async.sleep(0)
+            defer.returnValue("yay")
+
+        @defer.inlineCallbacks
+        def test():
+            with LoggingContext("c") as c1:
+                res = yield self.cache.fetch_or_execute(self.mock_key, cb)
+                self.assertIs(LoggingContext.current_context(), c1)
+                self.assertEqual(res, "yay")
+
+        # run the test twice in parallel
+        d = defer.gatherResults([test(), test()])
+        self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel)
+        yield d
+        self.assertIs(LoggingContext.current_context(), LoggingContext.sentinel)
+
     @defer.inlineCallbacks
     def test_does_not_cache_exceptions(self):
         """Checks that, if the callback throws an exception, it is called again
-- 
cgit 1.5.1


From d14d7b8fdc67f8f784c69605f95fe136e032b5d4 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 17 May 2018 12:09:18 +0100
Subject: Rename 'version' param on user consent config

we're going to use it for the version we require too.
---
 synapse/config/consent_config.py         | 10 ++++++----
 synapse/rest/consent/consent_resource.py |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 675fce0911..45856b9e8a 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -18,19 +18,21 @@ from ._base import Config
 DEFAULT_CONFIG = """\
 # User Consent configuration
 #
-# uncomment and configure if enabling the 'consent' resource under 'listeners'.
+# Parts of this section are required if enabling the 'consent' resource under
+# 'listeners', in particular 'template_dir' and 'version'.
 #
 # 'template_dir' gives the location of the templates for the HTML forms.
 # This directory should contain one subdirectory per language (eg, 'en', 'fr'),
 # and each language directory should contain the policy document (named as
 # '<version>.html') and a success page (success.html).
 #
-# 'default_version' gives the version of the policy document to serve up if
-# there is no 'v' parameter.
+# 'version' specifies the 'current' version of the policy document. It defines
+# the version to be served by the consent resource if there is no 'v'
+# parameter.
 #
 # user_consent:
 #   template_dir: res/templates/privacy
-#   default_version: 1.0
+#   version: 1.0
 """
 
 
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index d791302278..b0536ee375 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -116,7 +116,7 @@ class ConsentResource(Resource):
         loader = jinja2.FileSystemLoader(consent_template_directory)
         self._jinja_env = jinja2.Environment(loader=loader)
 
-        self._default_consent_verison = consent_config["default_version"]
+        self._default_consent_verison = consent_config["version"]
 
         if hs.config.form_secret is None:
             raise ConfigError(
-- 
cgit 1.5.1


From d5dca9a04fbac7fea0822eaf2fd513b13449184c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 18 May 2018 15:41:40 +0100
Subject: Move consent config parsing into ConsentConfig

turns out we need to reuse this, so it's better in the config class.
---
 synapse/config/consent_config.py                 | 16 +++++++++++++++-
 synapse/rest/consent/consent_resource.py         |  8 +++-----
 synapse/server_notices/consent_server_notices.py | 10 ++--------
 tests/utils.py                                   |  2 +-
 4 files changed, 21 insertions(+), 15 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index a6fbc5a058..8698b2993f 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -45,8 +45,22 @@ DEFAULT_CONFIG = """\
 
 
 class ConsentConfig(Config):
+    def __init__(self):
+        super(ConsentConfig, self).__init__()
+
+        self.user_consent_version = None
+        self.user_consent_template_dir = None
+        self.user_consent_server_notice_content = None
+
     def read_config(self, config):
-        self.consent_config = config.get("user_consent")
+        consent_config = config.get("user_consent")
+        if consent_config is None:
+            return
+        self.user_consent_version = str(consent_config["version"])
+        self.user_consent_template_dir = consent_config["template_dir"]
+        self.user_consent_server_notice_content = consent_config.get(
+            "server_notice_content",
+        )
 
     def default_config(self, **kwargs):
         return DEFAULT_CONFIG
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index b0536ee375..306385601c 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -95,8 +95,8 @@ class ConsentResource(Resource):
         # this is required by the request_handler wrapper
         self.clock = hs.get_clock()
 
-        consent_config = hs.config.consent_config
-        if consent_config is None:
+        self._default_consent_verison = hs.config.user_consent_version
+        if self._default_consent_verison is None:
             raise ConfigError(
                 "Consent resource is enabled but user_consent section is "
                 "missing in config file.",
@@ -104,7 +104,7 @@ class ConsentResource(Resource):
 
         # daemonize changes the cwd to /, so make the path absolute now.
         consent_template_directory = path.abspath(
-            consent_config["template_dir"],
+            hs.config.user_consent_template_dir,
         )
         if not path.isdir(consent_template_directory):
             raise ConfigError(
@@ -116,8 +116,6 @@ class ConsentResource(Resource):
         loader = jinja2.FileSystemLoader(consent_template_directory)
         self._jinja_env = jinja2.Environment(loader=loader)
 
-        self._default_consent_verison = consent_config["version"]
-
         if hs.config.form_secret is None:
             raise ConfigError(
                 "Consent resource is enabled but form_secret is not set in "
diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py
index e9098aef27..440f6b1cd4 100644
--- a/synapse/server_notices/consent_server_notices.py
+++ b/synapse/server_notices/consent_server_notices.py
@@ -35,16 +35,10 @@ class ConsentServerNotices(object):
         self._server_notices_manager = hs.get_server_notices_manager()
         self._store = hs.get_datastore()
 
-        self._current_consent_version = None
-        self._server_notice_content = None
         self._users_in_progress = set()
 
-        consent_config = hs.config.consent_config
-        if consent_config is not None:
-            self._current_consent_version = str(consent_config["version"])
-            self._server_notice_content = consent_config.get(
-                "server_notice_content"
-            )
+        self._current_consent_version = hs.config.user_consent_version
+        self._server_notice_content = hs.config.user_consent_server_notice_content
 
         if self._server_notice_content is not None:
             if not self._server_notices_manager.is_enabled():
diff --git a/tests/utils.py b/tests/utils.py
index 63d8e9c640..9626e6fd78 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -63,7 +63,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.federation_rc_concurrent = 10
         config.filter_timeline_limit = 5000
         config.user_directory_search_all_users = False
-        config.consent_config = None
+        config.user_consent_server_notice_content = None
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
-- 
cgit 1.5.1


From 8810685df936ba60e0e9d36c9371c20f305c9126 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 10:57:56 +0100
Subject: Stub out ServerNoticesSender on the workers

... and have the sync endpoints call it directly rather than obsure indirection
via PresenceHandler
---
 synapse/handlers/events.py                         |  5 +++
 synapse/handlers/presence.py                       |  4 --
 synapse/rest/client/v2_alpha/sync.py               |  4 ++
 synapse/server.py                                  |  7 ++++
 synapse/server_notices/server_notices_sender.py    |  8 ++--
 .../server_notices/worker_server_notices_sender.py | 46 ++++++++++++++++++++++
 6 files changed, 66 insertions(+), 8 deletions(-)
 create mode 100644 synapse/server_notices/worker_server_notices_sender.py

(limited to 'synapse/rest')

diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index d3685fb12a..8bc642675f 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -48,6 +48,7 @@ class EventStreamHandler(BaseHandler):
 
         self.notifier = hs.get_notifier()
         self.state = hs.get_state_handler()
+        self._server_notices_sender = hs.get_server_notices_sender()
 
     @defer.inlineCallbacks
     @log_function
@@ -58,6 +59,10 @@ class EventStreamHandler(BaseHandler):
 
         If `only_keys` is not None, events from keys will be sent down.
         """
+
+        # send any outstanding server notices to the user.
+        yield self._server_notices_sender.on_user_syncing(auth_user_id)
+
         auth_user = UserID.from_string(auth_user_id)
         presence_handler = self.hs.get_presence_handler()
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index adc816f747..500a131874 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -100,7 +100,6 @@ class PresenceHandler(object):
         self.notifier = hs.get_notifier()
         self.federation = hs.get_federation_sender()
         self.state = hs.get_state_handler()
-        self._server_notices_sender = hs.get_server_notices_sender()
 
         federation_registry = hs.get_federation_registry()
 
@@ -433,9 +432,6 @@ class PresenceHandler(object):
                     last_user_sync_ts=self.clock.time_msec(),
                 )])
 
-        # send any outstanding server notices to the user.
-        yield self._server_notices_sender.on_user_syncing(user_id)
-
         @defer.inlineCallbacks
         def _end():
             try:
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index eb91c0b293..a291cffbf1 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -85,6 +85,7 @@ class SyncRestServlet(RestServlet):
         self.clock = hs.get_clock()
         self.filtering = hs.get_filtering()
         self.presence_handler = hs.get_presence_handler()
+        self._server_notices_sender = hs.get_server_notices_sender()
 
     @defer.inlineCallbacks
     def on_GET(self, request):
@@ -149,6 +150,9 @@ class SyncRestServlet(RestServlet):
         else:
             since_token = None
 
+        # send any outstanding server notices to the user.
+        yield self._server_notices_sender.on_user_syncing(user.to_string())
+
         affect_presence = set_presence != PresenceState.OFFLINE
 
         if affect_presence:
diff --git a/synapse/server.py b/synapse/server.py
index e7c733f2d4..58dbf78437 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -74,6 +74,9 @@ from synapse.rest.media.v1.media_repository import (
 )
 from synapse.server_notices.server_notices_manager import ServerNoticesManager
 from synapse.server_notices.server_notices_sender import ServerNoticesSender
+from synapse.server_notices.worker_server_notices_sender import (
+    WorkerServerNoticesSender,
+)
 from synapse.state import StateHandler, StateResolutionHandler
 from synapse.storage import DataStore
 from synapse.streams.events import EventSources
@@ -403,9 +406,13 @@ class HomeServer(object):
         return FederationHandlerRegistry()
 
     def build_server_notices_manager(self):
+        if self.config.worker_app:
+            raise Exception("Workers cannot send server notices")
         return ServerNoticesManager(self)
 
     def build_server_notices_sender(self):
+        if self.config.worker_app:
+            return WorkerServerNoticesSender(self)
         return ServerNoticesSender(self)
 
     def remove_pusher(self, app_id, push_key, user_id):
diff --git a/synapse/server_notices/server_notices_sender.py b/synapse/server_notices/server_notices_sender.py
index 9eade85851..5d23965f34 100644
--- a/synapse/server_notices/server_notices_sender.py
+++ b/synapse/server_notices/server_notices_sender.py
@@ -31,9 +31,6 @@ class ServerNoticesSender(object):
     def on_user_syncing(self, user_id):
         """Called when the user performs a sync operation.
 
-        This is only called when /sync (or /events) is called on the synapse
-        master. In a deployment with synchrotrons, on_user_ip is called
-
         Args:
             user_id (str): mxid of user who synced
 
@@ -45,7 +42,7 @@ class ServerNoticesSender(object):
         )
 
     def on_user_ip(self, user_id):
-        """Called when a worker process saw a client request.
+        """Called on the master when a worker process saw a client request.
 
         Args:
             user_id (str): mxid
@@ -53,6 +50,9 @@ class ServerNoticesSender(object):
         Returns:
             Deferred
         """
+        # The synchrotrons use a stubbed version of ServerNoticesSender, so
+        # we check for notices to send to the user in on_user_ip as well as
+        # in on_user_syncing
         return self._consent_server_notices.maybe_send_server_notice_to_user(
             user_id,
         )
diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py
new file mode 100644
index 0000000000..25abb4ccf5
--- /dev/null
+++ b/synapse/server_notices/worker_server_notices_sender.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+
+
+class WorkerServerNoticesSender(object):
+    """Stub impl of ServerNoticesSender which does nothing"""
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer):
+        """
+
+    def on_user_syncing(self, user_id):
+        """Called when the user performs a sync operation.
+
+        Args:
+            user_id (str): mxid of user who synced
+
+        Returns:
+            Deferred
+        """
+        return defer.succeed()
+
+    def on_user_ip(self, user_id):
+        """Called on the master when a worker process saw a client request.
+
+        Args:
+            user_id (str): mxid
+
+        Returns:
+            Deferred
+        """
+        raise AssertionError("on_user_ip unexpectedly called on worker")
-- 
cgit 1.5.1


From 669400e22f514efbd846cad1f2943a48d79276b7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 14:18:53 +0100
Subject: Enable auto-escaping for the consent templates

... to reduce the risk of somebody introducing an html injection attack...
---
 synapse/rest/consent/consent_resource.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index 306385601c..e6a6dcbefa 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -114,7 +114,10 @@ class ConsentResource(Resource):
             )
 
         loader = jinja2.FileSystemLoader(consent_template_directory)
-        self._jinja_env = jinja2.Environment(loader=loader)
+        self._jinja_env = jinja2.Environment(
+            loader=loader,
+            autoescape=jinja2.select_autoescape(['html', 'htm', 'xml']),
+        )
 
         if hs.config.form_secret is None:
             raise ConfigError(
-- 
cgit 1.5.1


From 7b36d06a69942653f6c6fbb6dbf341a452002237 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 22 May 2018 14:50:22 +0100
Subject: Add a 'has_consented' template var to consent forms

fixes #3260
---
 docs/privacy_policy_templates/README.md   |  2 +-
 docs/privacy_policy_templates/en/1.0.html |  6 ++++++
 synapse/rest/consent/consent_resource.py  | 17 ++++++++++++++---
 3 files changed, 21 insertions(+), 4 deletions(-)

(limited to 'synapse/rest')

diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md
index 8e91c516b3..a3e6fc0986 100644
--- a/docs/privacy_policy_templates/README.md
+++ b/docs/privacy_policy_templates/README.md
@@ -9,7 +9,7 @@ form_secret: <unique but arbitrary secret>
 
 user_consent:
   template_dir: docs/privacy_policy_templates
-  default_version: 1.0
+  version: 1.0
 ```
 
 You should then be able to enable the `consent` resource under a `listener`
diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html
index ab8666f0c3..55c5e4b612 100644
--- a/docs/privacy_policy_templates/en/1.0.html
+++ b/docs/privacy_policy_templates/en/1.0.html
@@ -4,6 +4,11 @@
     <title>Matrix.org Privacy policy</title>
   </head>
   <body>
+  {% if has_consented %}
+    <p>
+      Your base already belong to us.
+    </p>
+  {% else %}
     <p>
       All your base are belong to us.
     </p>
@@ -13,5 +18,6 @@
       <input type="hidden" name="h" value="{{userhmac}}"/>
       <input type="submit" value="Sure thing!"/>
     </form>
+  {% endif %}
   </body>
 </html>
diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py
index e6a6dcbefa..724911d1e6 100644
--- a/synapse/rest/consent/consent_resource.py
+++ b/synapse/rest/consent/consent_resource.py
@@ -95,8 +95,8 @@ class ConsentResource(Resource):
         # this is required by the request_handler wrapper
         self.clock = hs.get_clock()
 
-        self._default_consent_verison = hs.config.user_consent_version
-        if self._default_consent_verison is None:
+        self._default_consent_version = hs.config.user_consent_version
+        if self._default_consent_version is None:
             raise ConfigError(
                 "Consent resource is enabled but user_consent section is "
                 "missing in config file.",
@@ -132,6 +132,7 @@ class ConsentResource(Resource):
         return NOT_DONE_YET
 
     @wrap_html_request_handler
+    @defer.inlineCallbacks
     def _async_render_GET(self, request):
         """
         Args:
@@ -139,16 +140,26 @@ class ConsentResource(Resource):
         """
 
         version = parse_string(request, "v",
-                               default=self._default_consent_verison)
+                               default=self._default_consent_version)
         username = parse_string(request, "u", required=True)
         userhmac = parse_string(request, "h", required=True)
 
         self._check_hash(username, userhmac)
 
+        if username.startswith('@'):
+            qualified_user_id = username
+        else:
+            qualified_user_id = UserID(username, self.hs.hostname).to_string()
+
+        u = yield self.store.get_user_by_id(qualified_user_id)
+        if u is None:
+            raise NotFoundError("Unknown user")
+
         try:
             self._render_template(
                 request, "%s.html" % (version,),
                 user=username, userhmac=userhmac, version=version,
+                has_consented=(u["consent_version"] == version),
             )
         except TemplateNotFound:
             raise NotFoundError("Unknown policy version")
-- 
cgit 1.5.1