From ec638a16022d29cbab32ac8ddb4cddb7f99d6495 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 15 Apr 2019 18:40:15 +0100
Subject: Only handle GET requests for /push_rules

---
 synapse/rest/client/v1/push_rule.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index c654f9b5f0..7e9d95de26 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -39,9 +39,13 @@ class PushRuleRestServlet(ClientV1RestServlet):
         super(PushRuleRestServlet, self).__init__(hs)
         self.store = hs.get_datastore()
         self.notifier = hs.get_notifier()
+        self._is_worker = hs.config.worker_app is not None
 
     @defer.inlineCallbacks
     def on_PUT(self, request):
+        if self._is_worker:
+            raise Exception("Cannot handle PUT /push_rules on worker")
+
         spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath])
         try:
             priority_class = _priority_class_from_spec(spec)
@@ -103,6 +107,9 @@ class PushRuleRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_DELETE(self, request):
+        if self._is_worker:
+            raise Exception("Cannot handle DELETE /push_rules on worker")
+
         spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath])
 
         requester = yield self.auth.get_user_by_req(request)
-- 
cgit 1.5.1


From ad010f630616fd0211706293e49fed38d037ceb5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 16 Apr 2019 17:42:50 +0100
Subject: Remove usage of request.postpath

This is an undocumented variable in twisted, and relies on the servlet
being mounted in the right way.

This also breaks getting push rules on workers.
---
 synapse/rest/client/v1/push_rule.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index 7e9d95de26..506ec95ddd 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -31,7 +31,7 @@ from .base import ClientV1RestServlet, client_path_patterns
 
 
 class PushRuleRestServlet(ClientV1RestServlet):
-    PATTERNS = client_path_patterns("/pushrules/.*$")
+    PATTERNS = client_path_patterns("/(?P<path>pushrules/.*)$")
     SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR = (
         "Unrecognised request: You probably wanted a trailing slash")
 
@@ -42,11 +42,11 @@ class PushRuleRestServlet(ClientV1RestServlet):
         self._is_worker = hs.config.worker_app is not None
 
     @defer.inlineCallbacks
-    def on_PUT(self, request):
+    def on_PUT(self, request, path):
         if self._is_worker:
             raise Exception("Cannot handle PUT /push_rules on worker")
 
-        spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath])
+        spec = _rule_spec_from_path([x for x in path.split("/")])
         try:
             priority_class = _priority_class_from_spec(spec)
         except InvalidRuleException as e:
@@ -106,11 +106,11 @@ class PushRuleRestServlet(ClientV1RestServlet):
         defer.returnValue((200, {}))
 
     @defer.inlineCallbacks
-    def on_DELETE(self, request):
+    def on_DELETE(self, request, path):
         if self._is_worker:
             raise Exception("Cannot handle DELETE /push_rules on worker")
 
-        spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath])
+        spec = _rule_spec_from_path([x for x in path.split("/")])
 
         requester = yield self.auth.get_user_by_req(request)
         user_id = requester.user.to_string()
@@ -130,7 +130,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
                 raise
 
     @defer.inlineCallbacks
-    def on_GET(self, request):
+    def on_GET(self, request, path):
         requester = yield self.auth.get_user_by_req(request)
         user_id = requester.user.to_string()
 
@@ -141,7 +141,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
 
         rules = format_push_rules_for_user(requester.user, rules)
 
-        path = [x.decode('utf8') for x in request.postpath][1:]
+        path = [x for x in path.split("/")][1:]
 
         if path == []:
             # we're a reference impl: pedantry is our job.
@@ -157,7 +157,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
         else:
             raise UnrecognizedRequestError()
 
-    def on_OPTIONS(self, _):
+    def on_OPTIONS(self, request, path):
         return 200, {}
 
     def notify_user(self, user_id):
-- 
cgit 1.5.1


From 600ec04739a3fd7a2697a837f6e232c970bd97d3 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Wed, 17 Apr 2019 12:01:59 +0100
Subject: Make sure we're not registering the same 3pid twice

---
 changelog.d/5071.bugfix                  |  1 +
 synapse/rest/client/v2_alpha/register.py | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 changelog.d/5071.bugfix

(limited to 'synapse/rest')

diff --git a/changelog.d/5071.bugfix b/changelog.d/5071.bugfix
new file mode 100644
index 0000000000..ddf7ab5fa8
--- /dev/null
+++ b/changelog.d/5071.bugfix
@@ -0,0 +1 @@
+Make sure we're not registering the same 3pid twice on registration.
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 6d235262c8..dc3e265bcd 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -391,6 +391,13 @@ class RegisterRestServlet(RestServlet):
         # the user-facing checks will probably already have happened in
         # /register/email/requestToken when we requested a 3pid, but that's not
         # guaranteed.
+        #
+        # Also check that we're not trying to register a 3pid that's already
+        # been registered.
+        #
+        # This has probably happened in /register/email/requestToken as well,
+        # but if a user hits this endpoint twice then clicks on each link from
+        # the two activation emails, they would register the same 3pid twice.
 
         if auth_result:
             for login_type in [LoginType.EMAIL_IDENTITY, LoginType.MSISDN]:
@@ -406,6 +413,17 @@ class RegisterRestServlet(RestServlet):
                             Codes.THREEPID_DENIED,
                         )
 
+                    existingUid = yield self.store.get_user_id_by_threepid(
+                        medium, address,
+                    )
+
+                    if existingUid is not None:
+                        raise SynapseError(
+                            400,
+                            "%s is already in use" % medium,
+                            Codes.THREEPID_IN_USE,
+                        )
+
         if registered_user_id is not None:
             logger.info(
                 "Already registered user ID %r for this session",
-- 
cgit 1.5.1


From 20f0617e87924c929f0db0c06d30de0c8d15081c Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Wed, 10 Apr 2019 17:58:47 +0100
Subject: Send out emails with links to extend an account's validity period

---
 changelog.d/5047.feature                           |   1 +
 docs/sample_config.yaml                            |  29 ++-
 synapse/api/auth.py                                |   5 +-
 synapse/config/emailconfig.py                      |   7 +-
 synapse/config/registration.py                     |  52 ++++-
 synapse/handlers/account_validity.py               | 228 +++++++++++++++++++++
 synapse/push/mailer.py                             |  14 +-
 synapse/push/pusher.py                             |   6 +-
 synapse/res/templates/mail-expiry.css              |   4 +
 synapse/res/templates/notice_expiry.html           |  43 ++++
 synapse/res/templates/notice_expiry.txt            |   7 +
 synapse/rest/__init__.py                           |   2 +
 synapse/rest/client/v2_alpha/account_validity.py   |  62 ++++++
 synapse/server.py                                  |   5 +
 synapse/storage/registration.py                    | 168 +++++++++++++--
 .../storage/schema/delta/54/account_validity.sql   |   9 +-
 tests/rest/client/v2_alpha/test_register.py        | 100 ++++++++-
 17 files changed, 699 insertions(+), 43 deletions(-)
 create mode 100644 changelog.d/5047.feature
 create mode 100644 synapse/handlers/account_validity.py
 create mode 100644 synapse/res/templates/mail-expiry.css
 create mode 100644 synapse/res/templates/notice_expiry.html
 create mode 100644 synapse/res/templates/notice_expiry.txt
 create mode 100644 synapse/rest/client/v2_alpha/account_validity.py

(limited to 'synapse/rest')

diff --git a/changelog.d/5047.feature b/changelog.d/5047.feature
new file mode 100644
index 0000000000..12766a82a7
--- /dev/null
+++ b/changelog.d/5047.feature
@@ -0,0 +1 @@
+Add time-based account expiration.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index 5594c8b9af..8bbd437239 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -643,11 +643,31 @@ uploads_path: "DATADIR/uploads"
 #
 #enable_registration: false
 
-# Optional account validity parameter. This allows for, e.g., accounts to
-# be denied any request after a given period.
+# Optional account validity configuration. This allows for accounts to be denied
+# any request after a given period.
+#
+# ``enabled`` defines whether the account validity feature is enabled. Defaults
+# to False.
+#
+# ``period`` allows setting the period after which an account is valid
+# after its registration. When renewing the account, its validity period
+# will be extended by this amount of time. This parameter is required when using
+# the account validity feature.
+#
+# ``renew_at`` is the amount of time before an account's expiry date at which
+# Synapse will send an email to the account's email address with a renewal link.
+# This needs the ``email`` and ``public_baseurl`` configuration sections to be
+# filled.
+#
+# ``renew_email_subject`` is the subject of the email sent out with the renewal
+# link. ``%(app)s`` can be used as a placeholder for the ``app_name`` parameter
+# from the ``email`` section.
 #
 #account_validity:
+#  enabled: True
 #  period: 6w
+#  renew_at: 1w
+#  renew_email_subject: "Renew your %(app)s account"
 
 # The user must provide all of the below types of 3PID when registering.
 #
@@ -890,7 +910,7 @@ password_config:
 
 
 
-# Enable sending emails for notification events
+# Enable sending emails for notification events or expiry notices
 # Defining a custom URL for Riot is only needed if email notifications
 # should contain links to a self-hosted installation of Riot; when set
 # the "app_name" setting is ignored.
@@ -912,6 +932,9 @@ password_config:
 #   #template_dir: res/templates
 #   notif_template_html: notif_mail.html
 #   notif_template_text: notif_mail.txt
+#   # Templates for account expiry notices.
+#   expiry_template_html: notice_expiry.html
+#   expiry_template_text: notice_expiry.txt
 #   notif_for_new_users: True
 #   riot_base_url: "http://localhost/riot"
 
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 976e0dd18b..4482962510 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -230,8 +230,9 @@ class Auth(object):
 
             # Deny the request if the user account has expired.
             if self._account_validity.enabled:
-                expiration_ts = yield self.store.get_expiration_ts_for_user(user)
-                if self.clock.time_msec() >= expiration_ts:
+                user_id = user.to_string()
+                expiration_ts = yield self.store.get_expiration_ts_for_user(user_id)
+                if expiration_ts and self.clock.time_msec() >= expiration_ts:
                     raise AuthError(
                         403,
                         "User account has expired",
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index 93d70cff14..60827be72f 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -71,6 +71,8 @@ class EmailConfig(Config):
             self.email_notif_from = email_config["notif_from"]
             self.email_notif_template_html = email_config["notif_template_html"]
             self.email_notif_template_text = email_config["notif_template_text"]
+            self.email_expiry_template_html = email_config["expiry_template_html"]
+            self.email_expiry_template_text = email_config["expiry_template_text"]
 
             template_dir = email_config.get("template_dir")
             # we need an absolute path, because we change directory after starting (and
@@ -120,7 +122,7 @@ class EmailConfig(Config):
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
-        # Enable sending emails for notification events
+        # Enable sending emails for notification events or expiry notices
         # Defining a custom URL for Riot is only needed if email notifications
         # should contain links to a self-hosted installation of Riot; when set
         # the "app_name" setting is ignored.
@@ -142,6 +144,9 @@ class EmailConfig(Config):
         #   #template_dir: res/templates
         #   notif_template_html: notif_mail.html
         #   notif_template_text: notif_mail.txt
+        #   # Templates for account expiry notices.
+        #   expiry_template_html: notice_expiry.html
+        #   expiry_template_text: notice_expiry.txt
         #   notif_for_new_users: True
         #   riot_base_url: "http://localhost/riot"
         """
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index b7a7b4f1cf..129c208204 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -21,12 +21,26 @@ from synapse.util.stringutils import random_string_with_symbols
 
 
 class AccountValidityConfig(Config):
-    def __init__(self, config):
-        self.enabled = (len(config) > 0)
+    def __init__(self, config, synapse_config):
+        self.enabled = config.get("enabled", False)
+        self.renew_by_email_enabled = ("renew_at" in config)
 
-        period = config.get("period", None)
-        if period:
-            self.period = self.parse_duration(period)
+        if self.enabled:
+            if "period" in config:
+                self.period = self.parse_duration(config["period"])
+            else:
+                raise ConfigError("'period' is required when using account validity")
+
+            if "renew_at" in config:
+                self.renew_at = self.parse_duration(config["renew_at"])
+
+            if "renew_email_subject" in config:
+                self.renew_email_subject = config["renew_email_subject"]
+            else:
+                self.renew_email_subject = "Renew your %(app)s account"
+
+        if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
+            raise ConfigError("Can't send renewal emails without 'public_baseurl'")
 
 
 class RegistrationConfig(Config):
@@ -40,7 +54,9 @@ class RegistrationConfig(Config):
                 strtobool(str(config["disable_registration"]))
             )
 
-        self.account_validity = AccountValidityConfig(config.get("account_validity", {}))
+        self.account_validity = AccountValidityConfig(
+            config.get("account_validity", {}), config,
+        )
 
         self.registrations_require_3pid = config.get("registrations_require_3pid", [])
         self.allowed_local_3pids = config.get("allowed_local_3pids", [])
@@ -86,11 +102,31 @@ class RegistrationConfig(Config):
         #
         #enable_registration: false
 
-        # Optional account validity parameter. This allows for, e.g., accounts to
-        # be denied any request after a given period.
+        # Optional account validity configuration. This allows for accounts to be denied
+        # any request after a given period.
+        #
+        # ``enabled`` defines whether the account validity feature is enabled. Defaults
+        # to False.
+        #
+        # ``period`` allows setting the period after which an account is valid
+        # after its registration. When renewing the account, its validity period
+        # will be extended by this amount of time. This parameter is required when using
+        # the account validity feature.
+        #
+        # ``renew_at`` is the amount of time before an account's expiry date at which
+        # Synapse will send an email to the account's email address with a renewal link.
+        # This needs the ``email`` and ``public_baseurl`` configuration sections to be
+        # filled.
+        #
+        # ``renew_email_subject`` is the subject of the email sent out with the renewal
+        # link. ``%%(app)s`` can be used as a placeholder for the ``app_name`` parameter
+        # from the ``email`` section.
         #
         #account_validity:
+        #  enabled: True
         #  period: 6w
+        #  renew_at: 1w
+        #  renew_email_subject: "Renew your %%(app)s account"
 
         # The user must provide all of the below types of 3PID when registering.
         #
diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py
new file mode 100644
index 0000000000..e82049e42d
--- /dev/null
+++ b/synapse/handlers/account_validity.py
@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import email.mime.multipart
+import email.utils
+import logging
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+from twisted.internet import defer
+
+from synapse.api.errors import StoreError
+from synapse.types import UserID
+from synapse.util import stringutils
+from synapse.util.logcontext import make_deferred_yieldable
+
+try:
+    from synapse.push.mailer import load_jinja2_templates
+except ImportError:
+    load_jinja2_templates = None
+
+logger = logging.getLogger(__name__)
+
+
+class AccountValidityHandler(object):
+    def __init__(self, hs):
+        self.hs = hs
+        self.store = self.hs.get_datastore()
+        self.sendmail = self.hs.get_sendmail()
+        self.clock = self.hs.get_clock()
+
+        self._account_validity = self.hs.config.account_validity
+
+        if self._account_validity.renew_by_email_enabled and load_jinja2_templates:
+            # Don't do email-specific configuration if renewal by email is disabled.
+            try:
+                app_name = self.hs.config.email_app_name
+
+                self._subject = self._account_validity.renew_email_subject % {
+                    "app": app_name,
+                }
+
+                self._from_string = self.hs.config.email_notif_from % {
+                    "app": app_name,
+                }
+            except Exception:
+                # If substitution failed, fall back to the bare strings.
+                self._subject = self._account_validity.renew_email_subject
+                self._from_string = self.hs.config.email_notif_from
+
+            self._raw_from = email.utils.parseaddr(self._from_string)[1]
+
+            self._template_html, self._template_text = load_jinja2_templates(
+                config=self.hs.config,
+                template_html_name=self.hs.config.email_expiry_template_html,
+                template_text_name=self.hs.config.email_expiry_template_text,
+            )
+
+            # Check the renewal emails to send and send them every 30min.
+            self.clock.looping_call(
+                self.send_renewal_emails,
+                30 * 60 * 1000,
+            )
+
+    @defer.inlineCallbacks
+    def send_renewal_emails(self):
+        """Gets the list of users whose account is expiring in the amount of time
+        configured in the ``renew_at`` parameter from the ``account_validity``
+        configuration, and sends renewal emails to all of these users as long as they
+        have an email 3PID attached to their account.
+        """
+        expiring_users = yield self.store.get_users_expiring_soon()
+
+        if expiring_users:
+            for user in expiring_users:
+                yield self._send_renewal_email(
+                    user_id=user["user_id"],
+                    expiration_ts=user["expiration_ts_ms"],
+                )
+
+    @defer.inlineCallbacks
+    def _send_renewal_email(self, user_id, expiration_ts):
+        """Sends out a renewal email to every email address attached to the given user
+        with a unique link allowing them to renew their account.
+
+        Args:
+            user_id (str): ID of the user to send email(s) to.
+            expiration_ts (int): Timestamp in milliseconds for the expiration date of
+                this user's account (used in the email templates).
+        """
+        addresses = yield self._get_email_addresses_for_user(user_id)
+
+        # Stop right here if the user doesn't have at least one email address.
+        # In this case, they will have to ask their server admin to renew their
+        # account manually.
+        if not addresses:
+            return
+
+        try:
+            user_display_name = yield self.store.get_profile_displayname(
+                UserID.from_string(user_id).localpart
+            )
+            if user_display_name is None:
+                user_display_name = user_id
+        except StoreError:
+            user_display_name = user_id
+
+        renewal_token = yield self._get_renewal_token(user_id)
+        url = "%s_matrix/client/unstable/account_validity/renew?token=%s" % (
+            self.hs.config.public_baseurl,
+            renewal_token,
+        )
+
+        template_vars = {
+            "display_name": user_display_name,
+            "expiration_ts": expiration_ts,
+            "url": url,
+        }
+
+        html_text = self._template_html.render(**template_vars)
+        html_part = MIMEText(html_text, "html", "utf8")
+
+        plain_text = self._template_text.render(**template_vars)
+        text_part = MIMEText(plain_text, "plain", "utf8")
+
+        for address in addresses:
+            raw_to = email.utils.parseaddr(address)[1]
+
+            multipart_msg = MIMEMultipart('alternative')
+            multipart_msg['Subject'] = self._subject
+            multipart_msg['From'] = self._from_string
+            multipart_msg['To'] = address
+            multipart_msg['Date'] = email.utils.formatdate()
+            multipart_msg['Message-ID'] = email.utils.make_msgid()
+            multipart_msg.attach(text_part)
+            multipart_msg.attach(html_part)
+
+            logger.info("Sending renewal email to %s", address)
+
+            yield make_deferred_yieldable(self.sendmail(
+                self.hs.config.email_smtp_host,
+                self._raw_from, raw_to, multipart_msg.as_string().encode('utf8'),
+                reactor=self.hs.get_reactor(),
+                port=self.hs.config.email_smtp_port,
+                requireAuthentication=self.hs.config.email_smtp_user is not None,
+                username=self.hs.config.email_smtp_user,
+                password=self.hs.config.email_smtp_pass,
+                requireTransportSecurity=self.hs.config.require_transport_security
+            ))
+
+        yield self.store.set_renewal_mail_status(
+            user_id=user_id,
+            email_sent=True,
+        )
+
+    @defer.inlineCallbacks
+    def _get_email_addresses_for_user(self, user_id):
+        """Retrieve the list of email addresses attached to a user's account.
+
+        Args:
+            user_id (str): ID of the user to lookup email addresses for.
+
+        Returns:
+            defer.Deferred[list[str]]: Email addresses for this account.
+        """
+        threepids = yield self.store.user_get_threepids(user_id)
+
+        addresses = []
+        for threepid in threepids:
+            if threepid["medium"] == "email":
+                addresses.append(threepid["address"])
+
+        defer.returnValue(addresses)
+
+    @defer.inlineCallbacks
+    def _get_renewal_token(self, user_id):
+        """Generates a 32-byte long random string that will be inserted into the
+        user's renewal email's unique link, then saves it into the database.
+
+        Args:
+            user_id (str): ID of the user to generate a string for.
+
+        Returns:
+            defer.Deferred[str]: The generated string.
+
+        Raises:
+            StoreError(500): Couldn't generate a unique string after 5 attempts.
+        """
+        attempts = 0
+        while attempts < 5:
+            try:
+                renewal_token = stringutils.random_string(32)
+                yield self.store.set_renewal_token_for_user(user_id, renewal_token)
+                defer.returnValue(renewal_token)
+            except StoreError:
+                attempts += 1
+        raise StoreError(500, "Couldn't generate a unique string as refresh string.")
+
+    @defer.inlineCallbacks
+    def renew_account(self, renewal_token):
+        """Renews the account attached to a given renewal token by pushing back the
+        expiration date by the current validity period in the server's configuration.
+
+        Args:
+            renewal_token (str): Token sent with the renewal request.
+        """
+        user_id = yield self.store.get_user_from_renewal_token(renewal_token)
+
+        logger.debug("Renewing an account for user %s", user_id)
+
+        new_expiration_date = self.clock.time_msec() + self._account_validity.period
+
+        yield self.store.renew_account_for_user(
+            user_id=user_id,
+            new_expiration_ts=new_expiration_date,
+        )
diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py
index 1eb5be0957..c269bcf4a4 100644
--- a/synapse/push/mailer.py
+++ b/synapse/push/mailer.py
@@ -521,11 +521,11 @@ def format_ts_filter(value, format):
     return time.strftime(format, time.localtime(value / 1000))
 
 
-def load_jinja2_templates(config):
+def load_jinja2_templates(config, template_html_name, template_text_name):
     """Load the jinja2 email templates from disk
 
     Returns:
-        (notif_template_html, notif_template_text)
+        (template_html, template_text)
     """
     logger.info("loading email templates from '%s'", config.email_template_dir)
     loader = jinja2.FileSystemLoader(config.email_template_dir)
@@ -533,14 +533,10 @@ def load_jinja2_templates(config):
     env.filters["format_ts"] = format_ts_filter
     env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
 
-    notif_template_html = env.get_template(
-        config.email_notif_template_html
-    )
-    notif_template_text = env.get_template(
-        config.email_notif_template_text
-    )
+    template_html = env.get_template(template_html_name)
+    template_text = env.get_template(template_text_name)
 
-    return notif_template_html, notif_template_text
+    return template_html, template_text
 
 
 def _create_mxc_to_http_filter(config):
diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py
index b33f2a357b..14bc7823cf 100644
--- a/synapse/push/pusher.py
+++ b/synapse/push/pusher.py
@@ -44,7 +44,11 @@ class PusherFactory(object):
         if hs.config.email_enable_notifs:
             self.mailers = {}  # app_name -> Mailer
 
-            templates = load_jinja2_templates(hs.config)
+            templates = load_jinja2_templates(
+                config=hs.config,
+                template_html_name=hs.config.email_notif_template_html,
+                template_text_name=hs.config.email_notif_template_text,
+            )
             self.notif_template_html, self.notif_template_text = templates
 
             self.pusher_types["email"] = self._create_email_pusher
diff --git a/synapse/res/templates/mail-expiry.css b/synapse/res/templates/mail-expiry.css
new file mode 100644
index 0000000000..3dea486467
--- /dev/null
+++ b/synapse/res/templates/mail-expiry.css
@@ -0,0 +1,4 @@
+.noticetext {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
diff --git a/synapse/res/templates/notice_expiry.html b/synapse/res/templates/notice_expiry.html
new file mode 100644
index 0000000000..f0d7c66e1b
--- /dev/null
+++ b/synapse/res/templates/notice_expiry.html
@@ -0,0 +1,43 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <style type="text/css">
+            {% include 'mail.css' without context %}
+            {% include "mail-%s.css" % app_name ignore missing without context %}
+            {% include 'mail-expiry.css' without context %}
+        </style>
+    </head>
+    <body>
+        <table id="page">
+            <tr>
+                <td> </td>
+                <td id="inner">
+                    <table class="header">
+                        <tr>
+                            <td>
+                                <div class="salutation">Hi {{ display_name }},</div>
+                            </td>
+                            <td class="logo">
+                                {% if app_name == "Riot" %}
+                                    <img src="http://riot.im/img/external/riot-logo-email.png" width="83" height="83" alt="[Riot]"/>
+                                {% elif app_name == "Vector" %}
+                                    <img src="http://matrix.org/img/vector-logo-email.png" width="64" height="83" alt="[Vector]"/>
+                                {% else %}
+                                    <img src="http://matrix.org/img/matrix-120x51.png" width="120" height="51" alt="[matrix]"/>
+                                {% endif %}
+                            </td>
+                        </tr>
+                        <tr>
+                          <td colspan="2">
+                            <div class="noticetext">Your account will expire on {{ expiration_ts|format_ts("%d-%m-%Y") }}. This means that you will lose access to your account after this date.</div>
+                            <div class="noticetext">To extend the validity of your account, please click on the link bellow (or copy and paste it into a new browser tab):</div>
+                            <div class="noticetext"><a href="{{ url }}">{{ url }}</a></div>
+                          </td>
+                        </tr>
+                    </table>
+                </td>
+                <td> </td>
+            </tr>
+        </table>
+    </body>
+</html>
diff --git a/synapse/res/templates/notice_expiry.txt b/synapse/res/templates/notice_expiry.txt
new file mode 100644
index 0000000000..41f1c4279c
--- /dev/null
+++ b/synapse/res/templates/notice_expiry.txt
@@ -0,0 +1,7 @@
+Hi {{ display_name }},
+
+Your account will expire on {{ expiration_ts|format_ts("%d-%m-%Y") }}. This means that you will lose access to your account after this date.
+
+To extend the validity of your account, please click on the link bellow (or copy and paste it to a new browser tab):
+
+{{ url }}
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 91f5247d52..a66885d349 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -33,6 +33,7 @@ from synapse.rest.client.v1 import (
 from synapse.rest.client.v2_alpha import (
     account,
     account_data,
+    account_validity,
     auth,
     capabilities,
     devices,
@@ -109,3 +110,4 @@ class ClientRestResource(JsonResource):
         groups.register_servlets(hs, client_resource)
         room_upgrade_rest_servlet.register_servlets(hs, client_resource)
         capabilities.register_servlets(hs, client_resource)
+        account_validity.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/account_validity.py b/synapse/rest/client/v2_alpha/account_validity.py
new file mode 100644
index 0000000000..1ff6a6b638
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/account_validity.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.http.server import finish_request
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_patterns
+
+logger = logging.getLogger(__name__)
+
+
+class AccountValidityRenewServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/account_validity/renew$")
+    SUCCESS_HTML = b"<html><body>Your account has been successfully renewed.</body><html>"
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(AccountValidityRenewServlet, self).__init__()
+
+        self.hs = hs
+        self.account_activity_handler = hs.get_account_validity_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        if b"token" not in request.args:
+            raise SynapseError(400, "Missing renewal token")
+        renewal_token = request.args[b"token"][0]
+
+        yield self.account_activity_handler.renew_account(renewal_token.decode('utf8'))
+
+        request.setResponseCode(200)
+        request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+        request.setHeader(b"Content-Length", b"%d" % (
+            len(AccountValidityRenewServlet.SUCCESS_HTML),
+        ))
+        request.write(AccountValidityRenewServlet.SUCCESS_HTML)
+        finish_request(request)
+        defer.returnValue(None)
+
+
+def register_servlets(hs, http_server):
+    AccountValidityRenewServlet(hs).register(http_server)
diff --git a/synapse/server.py b/synapse/server.py
index dc8f1ccb8c..8c30ac2fa5 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -47,6 +47,7 @@ from synapse.federation.transport.client import TransportLayerClient
 from synapse.groups.attestations import GroupAttestationSigning, GroupAttestionRenewer
 from synapse.groups.groups_server import GroupsServerHandler
 from synapse.handlers import Handlers
+from synapse.handlers.account_validity import AccountValidityHandler
 from synapse.handlers.acme import AcmeHandler
 from synapse.handlers.appservice import ApplicationServicesHandler
 from synapse.handlers.auth import AuthHandler, MacaroonGenerator
@@ -183,6 +184,7 @@ class HomeServer(object):
         'room_context_handler',
         'sendmail',
         'registration_handler',
+        'account_validity_handler',
     ]
 
     REQUIRED_ON_MASTER_STARTUP = [
@@ -506,6 +508,9 @@ class HomeServer(object):
     def build_registration_handler(self):
         return RegistrationHandler(self)
 
+    def build_account_validity_handler(self):
+        return AccountValidityHandler(self)
+
     def remove_pusher(self, app_id, push_key, user_id):
         return self.get_pusherpool().remove_pusher(app_id, push_key, user_id)
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index eede8ae4d2..a78850259f 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -32,6 +32,7 @@ class RegistrationWorkerStore(SQLBaseStore):
         super(RegistrationWorkerStore, self).__init__(db_conn, hs)
 
         self.config = hs.config
+        self.clock = hs.get_clock()
 
     @cached()
     def get_user_by_id(self, user_id):
@@ -87,25 +88,156 @@ class RegistrationWorkerStore(SQLBaseStore):
         )
 
     @cachedInlineCallbacks()
-    def get_expiration_ts_for_user(self, user):
+    def get_expiration_ts_for_user(self, user_id):
         """Get the expiration timestamp for the account bearing a given user ID.
 
         Args:
-            user (str): The ID of the user.
+            user_id (str): The ID of the user.
         Returns:
             defer.Deferred: None, if the account has no expiration timestamp,
-            otherwise int representation of the timestamp (as a number of
-            milliseconds since epoch).
+                otherwise int representation of the timestamp (as a number of
+                milliseconds since epoch).
         """
         res = yield self._simple_select_one_onecol(
             table="account_validity",
-            keyvalues={"user_id": user.to_string()},
+            keyvalues={"user_id": user_id},
             retcol="expiration_ts_ms",
             allow_none=True,
-            desc="get_expiration_date_for_user",
+            desc="get_expiration_ts_for_user",
+        )
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def renew_account_for_user(self, user_id, new_expiration_ts):
+        """Updates the account validity table with a new timestamp for a given
+        user, removes the existing renewal token from this user, and unsets the
+        flag indicating that an email has been sent for renewing this account.
+
+        Args:
+            user_id (str): ID of the user whose account validity to renew.
+            new_expiration_ts: New expiration date, as a timestamp in milliseconds
+                since epoch.
+        """
+        def renew_account_for_user_txn(txn):
+            self._simple_update_txn(
+                txn=txn,
+                table="account_validity",
+                keyvalues={"user_id": user_id},
+                updatevalues={
+                    "expiration_ts_ms": new_expiration_ts,
+                    "email_sent": False,
+                    "renewal_token": None,
+                },
+            )
+            self._invalidate_cache_and_stream(
+                txn, self.get_expiration_ts_for_user, (user_id,),
+            )
+
+        yield self.runInteraction(
+            "renew_account_for_user",
+            renew_account_for_user_txn,
+        )
+
+    @defer.inlineCallbacks
+    def set_renewal_token_for_user(self, user_id, renewal_token):
+        """Defines a renewal token for a given user.
+
+        Args:
+            user_id (str): ID of the user to set the renewal token for.
+            renewal_token (str): Random unique string that will be used to renew the
+                user's account.
+
+        Raises:
+            StoreError: The provided token is already set for another user.
+        """
+        yield self._simple_update_one(
+            table="account_validity",
+            keyvalues={"user_id": user_id},
+            updatevalues={"renewal_token": renewal_token},
+            desc="set_renewal_token_for_user",
+        )
+
+    @defer.inlineCallbacks
+    def get_user_from_renewal_token(self, renewal_token):
+        """Get a user ID from a renewal token.
+
+        Args:
+            renewal_token (str): The renewal token to perform the lookup with.
+
+        Returns:
+            defer.Deferred[str]: The ID of the user to which the token belongs.
+        """
+        res = yield self._simple_select_one_onecol(
+            table="account_validity",
+            keyvalues={"renewal_token": renewal_token},
+            retcol="user_id",
+            desc="get_user_from_renewal_token",
+        )
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def get_renewal_token_for_user(self, user_id):
+        """Get the renewal token associated with a given user ID.
+
+        Args:
+            user_id (str): The user ID to lookup a token for.
+
+        Returns:
+            defer.Deferred[str]: The renewal token associated with this user ID.
+        """
+        res = yield self._simple_select_one_onecol(
+            table="account_validity",
+            keyvalues={"user_id": user_id},
+            retcol="renewal_token",
+            desc="get_renewal_token_for_user",
         )
+
         defer.returnValue(res)
 
+    @defer.inlineCallbacks
+    def get_users_expiring_soon(self):
+        """Selects users whose account will expire in the [now, now + renew_at] time
+        window (see configuration for account_validity for information on what renew_at
+        refers to).
+
+        Returns:
+            Deferred: Resolves to a list[dict[user_id (str), expiration_ts_ms (int)]]
+        """
+        def select_users_txn(txn, now_ms, renew_at):
+            sql = (
+                "SELECT user_id, expiration_ts_ms FROM account_validity"
+                " WHERE email_sent = ? AND (expiration_ts_ms - ?) <= ?"
+            )
+            values = [False, now_ms, renew_at]
+            txn.execute(sql, values)
+            return self.cursor_to_dict(txn)
+
+        res = yield self.runInteraction(
+            "get_users_expiring_soon",
+            select_users_txn,
+            self.clock.time_msec(), self.config.account_validity.renew_at,
+        )
+
+        defer.returnValue(res)
+
+    @defer.inlineCallbacks
+    def set_renewal_mail_status(self, user_id, email_sent):
+        """Sets or unsets the flag that indicates whether a renewal email has been sent
+        to the user (and the user hasn't renewed their account yet).
+
+        Args:
+            user_id (str): ID of the user to set/unset the flag for.
+            email_sent (bool): Flag which indicates whether a renewal email has been sent
+                to this user.
+        """
+        yield self._simple_update_one(
+            table="account_validity",
+            keyvalues={"user_id": user_id},
+            updatevalues={"email_sent": email_sent},
+            desc="set_renewal_mail_status",
+        )
+
     @defer.inlineCallbacks
     def is_server_admin(self, user):
         res = yield self._simple_select_one_onecol(
@@ -508,22 +640,24 @@ class RegistrationStore(RegistrationWorkerStore,
                     }
                 )
 
-            if self._account_validity.enabled:
-                now_ms = self.clock.time_msec()
-                expiration_ts = now_ms + self._account_validity.period
-                self._simple_insert_txn(
-                    txn,
-                    "account_validity",
-                    values={
-                        "user_id": user_id,
-                        "expiration_ts_ms": expiration_ts,
-                    }
-                )
         except self.database_engine.module.IntegrityError:
             raise StoreError(
                 400, "User ID already taken.", errcode=Codes.USER_IN_USE
             )
 
+        if self._account_validity.enabled:
+            now_ms = self.clock.time_msec()
+            expiration_ts = now_ms + self._account_validity.period
+            self._simple_insert_txn(
+                txn,
+                "account_validity",
+                values={
+                    "user_id": user_id,
+                    "expiration_ts_ms": expiration_ts,
+                    "email_sent": False,
+                }
+            )
+
         if token:
             # it's possible for this to get a conflict, but only for a single user
             # since tokens are namespaced based on their user ID
diff --git a/synapse/storage/schema/delta/54/account_validity.sql b/synapse/storage/schema/delta/54/account_validity.sql
index 57249262d7..2357626000 100644
--- a/synapse/storage/schema/delta/54/account_validity.sql
+++ b/synapse/storage/schema/delta/54/account_validity.sql
@@ -13,8 +13,15 @@
  * limitations under the License.
  */
 
+DROP TABLE IF EXISTS account_validity;
+
 -- Track what users are in public rooms.
 CREATE TABLE IF NOT EXISTS account_validity (
     user_id TEXT PRIMARY KEY,
-    expiration_ts_ms BIGINT NOT NULL
+    expiration_ts_ms BIGINT NOT NULL,
+    email_sent BOOLEAN NOT NULL,
+    renewal_token TEXT
 );
+
+CREATE INDEX account_validity_email_sent_idx ON account_validity(email_sent, expiration_ts_ms)
+CREATE UNIQUE INDEX account_validity_renewal_string_idx ON account_validity(renewal_token)
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index d3611ed21f..8fb5140a05 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -1,14 +1,22 @@
 import datetime
 import json
+import os
+
+import pkg_resources
 
 from synapse.api.constants import LoginType
 from synapse.api.errors import Codes
 from synapse.appservice import ApplicationService
 from synapse.rest.client.v1 import admin, login
-from synapse.rest.client.v2_alpha import register, sync
+from synapse.rest.client.v2_alpha import account_validity, register, sync
 
 from tests import unittest
 
+try:
+    from synapse.push.mailer import load_jinja2_templates
+except ImportError:
+    load_jinja2_templates = None
+
 
 class RegisterRestServletTestCase(unittest.HomeserverTestCase):
 
@@ -197,6 +205,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
         config = self.default_config()
+        # Test for account expiring after a week.
         config.enable_registration = True
         config.account_validity.enabled = True
         config.account_validity.period = 604800000  # Time in ms for 1 week
@@ -228,3 +237,92 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
         self.assertEquals(
             channel.json_body["errcode"], Codes.EXPIRED_ACCOUNT, channel.result,
         )
+
+
+class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
+
+    skip = "No Jinja installed" if not load_jinja2_templates else None
+    servlets = [
+        register.register_servlets,
+        admin.register_servlets,
+        login.register_servlets,
+        sync.register_servlets,
+        account_validity.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+        config = self.default_config()
+        # Test for account expiring after a week and renewal emails being sent 2
+        # days before expiry.
+        config.enable_registration = True
+        config.account_validity.enabled = True
+        config.account_validity.renew_by_email_enabled = True
+        config.account_validity.period = 604800000  # Time in ms for 1 week
+        config.account_validity.renew_at = 172800000  # Time in ms for 2 days
+        config.account_validity.renew_email_subject = "Renew your account"
+
+        # Email config.
+        self.email_attempts = []
+
+        def sendmail(*args, **kwargs):
+            self.email_attempts.append((args, kwargs))
+            return
+
+        config.email_template_dir = os.path.abspath(
+            pkg_resources.resource_filename('synapse', 'res/templates')
+        )
+        config.email_expiry_template_html = "notice_expiry.html"
+        config.email_expiry_template_text = "notice_expiry.txt"
+        config.email_smtp_host = "127.0.0.1"
+        config.email_smtp_port = 20
+        config.require_transport_security = False
+        config.email_smtp_user = None
+        config.email_smtp_pass = None
+        config.email_notif_from = "test@example.com"
+
+        self.hs = self.setup_test_homeserver(config=config, sendmail=sendmail)
+
+        self.store = self.hs.get_datastore()
+
+        return self.hs
+
+    def test_renewal_email(self):
+        user_id = self.register_user("kermit", "monkey")
+        tok = self.login("kermit", "monkey")
+        # We need to manually add an email address otherwise the handler will do
+        # nothing.
+        now = self.hs.clock.time_msec()
+        self.get_success(self.store.user_add_threepid(
+            user_id=user_id, medium="email", address="kermit@example.com",
+            validated_at=now, added_at=now,
+        ))
+
+        # The specific endpoint doesn't matter, all we need is an authenticated
+        # endpoint.
+        request, channel = self.make_request(
+            b"GET", "/sync", access_token=tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+        # Move 6 days forward. This should trigger a renewal email to be sent.
+        self.reactor.advance(datetime.timedelta(days=6).total_seconds())
+        self.assertEqual(len(self.email_attempts), 1)
+
+        # Retrieving the URL from the email is too much pain for now, so we
+        # retrieve the token from the DB.
+        renewal_token = self.get_success(self.store.get_renewal_token_for_user(user_id))
+        url = "/_matrix/client/unstable/account_validity/renew?token=%s" % renewal_token
+        request, channel = self.make_request(b"GET", url)
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+        # Move 3 days forward. If the renewal failed, every authed request with
+        # our access token should be denied from now, otherwise they should
+        # succeed.
+        self.reactor.advance(datetime.timedelta(days=3).total_seconds())
+        request, channel = self.make_request(
+            b"GET", "/sync", access_token=tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
-- 
cgit 1.5.1


From eaf41a943b2cd3f7f32d142c9552d558eb37a074 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Tue, 16 Apr 2019 20:13:59 +0100
Subject: Add management endpoints for account validity

---
 changelog.d/5073.feature                         |  1 +
 docs/admin_api/account_validity.rst              | 42 +++++++++++
 synapse/api/auth.py                              |  2 +-
 synapse/handlers/account_validity.py             | 33 +++++++-
 synapse/rest/client/v1/admin.py                  | 39 ++++++++++
 synapse/rest/client/v2_alpha/account_validity.py | 31 +++++++-
 synapse/storage/registration.py                  | 29 +++++---
 tests/rest/client/v2_alpha/test_register.py      | 95 ++++++++++++++++++++++--
 8 files changed, 246 insertions(+), 26 deletions(-)
 create mode 100644 changelog.d/5073.feature
 create mode 100644 docs/admin_api/account_validity.rst

(limited to 'synapse/rest')

diff --git a/changelog.d/5073.feature b/changelog.d/5073.feature
new file mode 100644
index 0000000000..12766a82a7
--- /dev/null
+++ b/changelog.d/5073.feature
@@ -0,0 +1 @@
+Add time-based account expiration.
diff --git a/docs/admin_api/account_validity.rst b/docs/admin_api/account_validity.rst
new file mode 100644
index 0000000000..980ea23605
--- /dev/null
+++ b/docs/admin_api/account_validity.rst
@@ -0,0 +1,42 @@
+Account validity API
+====================
+
+This API allows a server administrator to manage the validity of an account. To
+use it, you must enable the account validity feature (under
+``account_validity``) in Synapse's configuration.
+
+Renew account
+-------------
+
+This API extends the validity of an account by as much time as configured in the
+``period`` parameter from the ``account_validity`` configuration.
+
+The API is::
+
+    POST /_matrix/client/unstable/account_validity/send_mail
+
+with the following body:
+
+.. code:: json
+
+    {
+        "user_id": "<user ID for the account to renew>",
+        "expiration_ts": 0,
+        "enable_renewal_emails": true
+    }
+
+
+``expiration_ts`` is an optional parameter and overrides the expiration date,
+which otherwise defaults to now + validity period.
+
+``enable_renewal_emails`` is also an optional parameter and enables/disables
+sending renewal emails to the user. Defaults to true.
+
+The API returns with the new expiration date for this account, as a timestamp in
+milliseconds since epoch:
+
+.. code:: json
+
+    {
+        "expiration_ts": 0
+    }
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 4482962510..960e66dbdc 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -232,7 +232,7 @@ class Auth(object):
             if self._account_validity.enabled:
                 user_id = user.to_string()
                 expiration_ts = yield self.store.get_expiration_ts_for_user(user_id)
-                if expiration_ts and self.clock.time_msec() >= expiration_ts:
+                if expiration_ts is not None and self.clock.time_msec() >= expiration_ts:
                     raise AuthError(
                         403,
                         "User account has expired",
diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py
index e82049e42d..261446517d 100644
--- a/synapse/handlers/account_validity.py
+++ b/synapse/handlers/account_validity.py
@@ -90,6 +90,11 @@ class AccountValidityHandler(object):
                     expiration_ts=user["expiration_ts_ms"],
                 )
 
+    @defer.inlineCallbacks
+    def send_renewal_email_to_user(self, user_id):
+        expiration_ts = yield self.store.get_expiration_ts_for_user(user_id)
+        yield self._send_renewal_email(user_id, expiration_ts)
+
     @defer.inlineCallbacks
     def _send_renewal_email(self, user_id, expiration_ts):
         """Sends out a renewal email to every email address attached to the given user
@@ -217,12 +222,32 @@ class AccountValidityHandler(object):
             renewal_token (str): Token sent with the renewal request.
         """
         user_id = yield self.store.get_user_from_renewal_token(renewal_token)
-
         logger.debug("Renewing an account for user %s", user_id)
+        yield self.renew_account_for_user(user_id)
 
-        new_expiration_date = self.clock.time_msec() + self._account_validity.period
+    @defer.inlineCallbacks
+    def renew_account_for_user(self, user_id, expiration_ts=None, email_sent=False):
+        """Renews the account attached to a given user by pushing back the
+        expiration date by the current validity period in the server's
+        configuration.
 
-        yield self.store.renew_account_for_user(
+        Args:
+            renewal_token (str): Token sent with the renewal request.
+            expiration_ts (int): New expiration date. Defaults to now + validity period.
+            email_sent (bool): Whether an email has been sent for this validity period.
+                Defaults to False.
+
+        Returns:
+            defer.Deferred[int]: New expiration date for this account, as a timestamp
+                in milliseconds since epoch.
+        """
+        if expiration_ts is None:
+            expiration_ts = self.clock.time_msec() + self._account_validity.period
+
+        yield self.store.set_account_validity_for_user(
             user_id=user_id,
-            new_expiration_ts=new_expiration_date,
+            expiration_ts=expiration_ts,
+            email_sent=email_sent,
         )
+
+        defer.returnValue(expiration_ts)
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index e788769639..d27472c538 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -786,6 +786,44 @@ class SearchUsersRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
+class AccountValidityRenewServlet(ClientV1RestServlet):
+    PATTERNS = client_path_patterns("/admin/account_validity/validity$")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(AccountValidityRenewServlet, self).__init__(hs)
+
+        self.hs = hs
+        self.account_activity_handler = hs.get_account_validity_handler()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        body = parse_json_object_from_request(request)
+
+        if "user_id" not in body:
+            raise SynapseError(400, "Missing property 'user_id' in the request body")
+
+        expiration_ts = yield self.account_activity_handler.renew_account_for_user(
+            body["user_id"], body.get("expiration_ts"),
+            not body.get("enable_renewal_emails", True),
+        )
+
+        res = {
+            "expiration_ts": expiration_ts,
+        }
+        defer.returnValue((200, res))
+
+
 def register_servlets(hs, http_server):
     WhoisRestServlet(hs).register(http_server)
     PurgeMediaCacheRestServlet(hs).register(http_server)
@@ -801,3 +839,4 @@ def register_servlets(hs, http_server):
     ListMediaInRoom(hs).register(http_server)
     UserRegisterServlet(hs).register(http_server)
     VersionServlet(hs).register(http_server)
+    AccountValidityRenewServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/account_validity.py b/synapse/rest/client/v2_alpha/account_validity.py
index 1ff6a6b638..fc8dbeb617 100644
--- a/synapse/rest/client/v2_alpha/account_validity.py
+++ b/synapse/rest/client/v2_alpha/account_validity.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
+from synapse.api.errors import AuthError, SynapseError
 from synapse.http.server import finish_request
 from synapse.http.servlet import RestServlet
 
@@ -39,6 +39,7 @@ class AccountValidityRenewServlet(RestServlet):
 
         self.hs = hs
         self.account_activity_handler = hs.get_account_validity_handler()
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_GET(self, request):
@@ -58,5 +59,33 @@ class AccountValidityRenewServlet(RestServlet):
         defer.returnValue(None)
 
 
+class AccountValiditySendMailServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/account_validity/send_mail$")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(AccountValiditySendMailServlet, self).__init__()
+
+        self.hs = hs
+        self.account_activity_handler = hs.get_account_validity_handler()
+        self.auth = hs.get_auth()
+        self.account_validity = self.hs.config.account_validity
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        if not self.account_validity.renew_by_email_enabled:
+            raise AuthError(403, "Account renewal via email is disabled on this server.")
+
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+        yield self.account_activity_handler.send_renewal_email_to_user(user_id)
+
+        defer.returnValue((200, {}))
+
+
 def register_servlets(hs, http_server):
     AccountValidityRenewServlet(hs).register(http_server)
+    AccountValiditySendMailServlet(hs).register(http_server)
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index a78850259f..dfdb4e7e34 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -108,25 +108,30 @@ class RegistrationWorkerStore(SQLBaseStore):
         defer.returnValue(res)
 
     @defer.inlineCallbacks
-    def renew_account_for_user(self, user_id, new_expiration_ts):
-        """Updates the account validity table with a new timestamp for a given
-        user, removes the existing renewal token from this user, and unsets the
-        flag indicating that an email has been sent for renewing this account.
+    def set_account_validity_for_user(self, user_id, expiration_ts, email_sent,
+                                      renewal_token=None):
+        """Updates the account validity properties of the given account, with the
+        given values.
 
         Args:
-            user_id (str): ID of the user whose account validity to renew.
-            new_expiration_ts: New expiration date, as a timestamp in milliseconds
+            user_id (str): ID of the account to update properties for.
+            expiration_ts (int): New expiration date, as a timestamp in milliseconds
                 since epoch.
+            email_sent (bool): True means a renewal email has been sent for this
+                account and there's no need to send another one for the current validity
+                period.
+            renewal_token (str): Renewal token the user can use to extend the validity
+                of their account. Defaults to no token.
         """
-        def renew_account_for_user_txn(txn):
+        def set_account_validity_for_user_txn(txn):
             self._simple_update_txn(
                 txn=txn,
                 table="account_validity",
                 keyvalues={"user_id": user_id},
                 updatevalues={
-                    "expiration_ts_ms": new_expiration_ts,
-                    "email_sent": False,
-                    "renewal_token": None,
+                    "expiration_ts_ms": expiration_ts,
+                    "email_sent": email_sent,
+                    "renewal_token": renewal_token,
                 },
             )
             self._invalidate_cache_and_stream(
@@ -134,8 +139,8 @@ class RegistrationWorkerStore(SQLBaseStore):
             )
 
         yield self.runInteraction(
-            "renew_account_for_user",
-            renew_account_for_user_txn,
+            "set_account_validity_for_user",
+            set_account_validity_for_user_txn,
         )
 
     @defer.inlineCallbacks
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 8fb5140a05..3d44667489 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -201,6 +201,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
         admin.register_servlets,
         login.register_servlets,
         sync.register_servlets,
+        account_validity.register_servlets,
     ]
 
     def make_homeserver(self, reactor, clock):
@@ -238,6 +239,68 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
             channel.json_body["errcode"], Codes.EXPIRED_ACCOUNT, channel.result,
         )
 
+    def test_manual_renewal(self):
+        user_id = self.register_user("kermit", "monkey")
+        tok = self.login("kermit", "monkey")
+
+        self.reactor.advance(datetime.timedelta(weeks=1).total_seconds())
+
+        # If we register the admin user at the beginning of the test, it will
+        # expire at the same time as the normal user and the renewal request
+        # will be denied.
+        self.register_user("admin", "adminpassword", admin=True)
+        admin_tok = self.login("admin", "adminpassword")
+
+        url = "/_matrix/client/unstable/admin/account_validity/validity"
+        params = {
+            "user_id": user_id,
+        }
+        request_data = json.dumps(params)
+        request, channel = self.make_request(
+            b"POST", url, request_data, access_token=admin_tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+        # The specific endpoint doesn't matter, all we need is an authenticated
+        # endpoint.
+        request, channel = self.make_request(
+            b"GET", "/sync", access_token=tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+    def test_manual_expire(self):
+        user_id = self.register_user("kermit", "monkey")
+        tok = self.login("kermit", "monkey")
+
+        self.register_user("admin", "adminpassword", admin=True)
+        admin_tok = self.login("admin", "adminpassword")
+
+        url = "/_matrix/client/unstable/admin/account_validity/validity"
+        params = {
+            "user_id": user_id,
+            "expiration_ts": 0,
+            "enable_renewal_emails": False,
+        }
+        request_data = json.dumps(params)
+        request, channel = self.make_request(
+            b"POST", url, request_data, access_token=admin_tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+        # The specific endpoint doesn't matter, all we need is an authenticated
+        # endpoint.
+        request, channel = self.make_request(
+            b"GET", "/sync", access_token=tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"403", channel.result)
+        self.assertEquals(
+            channel.json_body["errcode"], Codes.EXPIRED_ACCOUNT, channel.result,
+        )
+
 
 class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
 
@@ -287,6 +350,8 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
         return self.hs
 
     def test_renewal_email(self):
+        self.email_attempts = []
+
         user_id = self.register_user("kermit", "monkey")
         tok = self.login("kermit", "monkey")
         # We need to manually add an email address otherwise the handler will do
@@ -297,14 +362,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
             validated_at=now, added_at=now,
         ))
 
-        # The specific endpoint doesn't matter, all we need is an authenticated
-        # endpoint.
-        request, channel = self.make_request(
-            b"GET", "/sync", access_token=tok,
-        )
-        self.render(request)
-        self.assertEquals(channel.result["code"], b"200", channel.result)
-
         # Move 6 days forward. This should trigger a renewal email to be sent.
         self.reactor.advance(datetime.timedelta(days=6).total_seconds())
         self.assertEqual(len(self.email_attempts), 1)
@@ -326,3 +383,25 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
         )
         self.render(request)
         self.assertEquals(channel.result["code"], b"200", channel.result)
+
+    def test_manual_email_send(self):
+        self.email_attempts = []
+
+        user_id = self.register_user("kermit", "monkey")
+        tok = self.login("kermit", "monkey")
+        # We need to manually add an email address otherwise the handler will do
+        # nothing.
+        now = self.hs.clock.time_msec()
+        self.get_success(self.store.user_add_threepid(
+            user_id=user_id, medium="email", address="kermit@example.com",
+            validated_at=now, added_at=now,
+        ))
+
+        request, channel = self.make_request(
+            b"POST", "/_matrix/client/unstable/account_validity/send_mail",
+            access_token=tok,
+        )
+        self.render(request)
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+        self.assertEqual(len(self.email_attempts), 1)
-- 
cgit 1.5.1


From 6b2b9a58c44c9c2f7efc45e8b588cde17fcdb0d4 Mon Sep 17 00:00:00 2001
From: Amber Brown <hawkowl@atleastfornow.net>
Date: Thu, 25 Apr 2019 02:37:33 +1000
Subject: Prevent "producer not unregistered" message (#5009)

---
 changelog.d/5009.bugfix        | 1 +
 synapse/rest/media/v1/_base.py | 4 ++++
 2 files changed, 5 insertions(+)
 create mode 100644 changelog.d/5009.bugfix

(limited to 'synapse/rest')

diff --git a/changelog.d/5009.bugfix b/changelog.d/5009.bugfix
new file mode 100644
index 0000000000..e66d3dd1aa
--- /dev/null
+++ b/changelog.d/5009.bugfix
@@ -0,0 +1 @@
+Clients timing out/disappearing while downloading from the media repository will now no longer log a spurious "Producer was not unregistered" message.
\ No newline at end of file
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index e2b5df701d..2dcc8f74d6 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -191,6 +191,10 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam
         # in that case.
         logger.warning("Failed to write to consumer: %s %s", type(e), e)
 
+        # Unregister the producer, if it has one, so Twisted doesn't complain
+        if request.producer:
+            request.unregisterProducer()
+
     finish_request(request)
 
 
-- 
cgit 1.5.1


From 788163e2048b560e62005ad3da5ba274ba2726e9 Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Wed, 24 Apr 2019 17:44:06 +0100
Subject: Remove log error for .well-known/matrix/client (#4972)

---
 changelog.d/4972.misc      | 1 +
 synapse/rest/well_known.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/4972.misc

(limited to 'synapse/rest')

diff --git a/changelog.d/4972.misc b/changelog.d/4972.misc
new file mode 100644
index 0000000000..e104a9bb34
--- /dev/null
+++ b/changelog.d/4972.misc
@@ -0,0 +1 @@
+Reduce log level of .well-known/matrix/client responses.
diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py
index ab901e63f2..a7fa4f39af 100644
--- a/synapse/rest/well_known.py
+++ b/synapse/rest/well_known.py
@@ -68,6 +68,6 @@ class WellKnownResource(Resource):
             request.setHeader(b"Content-Type", b"text/plain")
             return b'.well-known not available'
 
-        logger.error("returning: %s", r)
+        logger.debug("returning: %s", r)
         request.setHeader(b"Content-Type", b"application/json")
         return json.dumps(r).encode("utf-8")
-- 
cgit 1.5.1


From 579b637b6c52991f59bbcf7d836587218e42e744 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 May 2019 10:40:33 +0100
Subject: Move admin API away from ClientV1RestServlet

---
 synapse/rest/client/v1/admin.py | 66 ++++++++++++++++++++---------------------
 1 file changed, 33 insertions(+), 33 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 0a1e233b23..8efa457d15 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -28,6 +28,7 @@ import synapse
 from synapse.api.constants import Membership, UserTypes
 from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
 from synapse.http.servlet import (
+    RestServlet,
     assert_params_in_dict,
     parse_integer,
     parse_json_object_from_request,
@@ -36,16 +37,17 @@ from synapse.http.servlet import (
 from synapse.types import UserID, create_requester
 from synapse.util.versionstring import get_version_string
 
-from .base import ClientV1RestServlet, client_path_patterns
+from .base import client_path_patterns
 
 logger = logging.getLogger(__name__)
 
 
-class UsersRestServlet(ClientV1RestServlet):
+class UsersRestServlet(RestServlet):
     PATTERNS = client_path_patterns("/admin/users/(?P<user_id>[^/]*)")
 
     def __init__(self, hs):
-        super(UsersRestServlet, self).__init__(hs)
+        self.hs = hs
+        self.auth = hs.get_auth()
         self.handlers = hs.get_handlers()
 
     @defer.inlineCallbacks
@@ -69,9 +71,12 @@ class UsersRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
-class VersionServlet(ClientV1RestServlet):
+class VersionServlet(RestServlet):
     PATTERNS = client_path_patterns("/admin/server_version")
 
+    def __init__(self, hs):
+        self.auth = hs.get_auth()
+
     @defer.inlineCallbacks
     def on_GET(self, request):
         requester = yield self.auth.get_user_by_req(request)
@@ -88,7 +93,7 @@ class VersionServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
-class UserRegisterServlet(ClientV1RestServlet):
+class UserRegisterServlet(RestServlet):
     """
     Attributes:
          NONCE_TIMEOUT (int): Seconds until a generated nonce won't be accepted
@@ -99,7 +104,6 @@ class UserRegisterServlet(ClientV1RestServlet):
     NONCE_TIMEOUT = 60
 
     def __init__(self, hs):
-        super(UserRegisterServlet, self).__init__(hs)
         self.handlers = hs.get_handlers()
         self.reactor = hs.get_reactor()
         self.nonces = {}
@@ -226,11 +230,12 @@ class UserRegisterServlet(ClientV1RestServlet):
         defer.returnValue((200, result))
 
 
-class WhoisRestServlet(ClientV1RestServlet):
+class WhoisRestServlet(RestServlet):
     PATTERNS = client_path_patterns("/admin/whois/(?P<user_id>[^/]*)")
 
     def __init__(self, hs):
-        super(WhoisRestServlet, self).__init__(hs)
+        self.hs = hs
+        self.auth = hs.get_auth()
         self.handlers = hs.get_handlers()
 
     @defer.inlineCallbacks
@@ -251,12 +256,12 @@ class WhoisRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
-class PurgeMediaCacheRestServlet(ClientV1RestServlet):
+class PurgeMediaCacheRestServlet(RestServlet):
     PATTERNS = client_path_patterns("/admin/purge_media_cache")
 
     def __init__(self, hs):
         self.media_repository = hs.get_media_repository()
-        super(PurgeMediaCacheRestServlet, self).__init__(hs)
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_POST(self, request):
@@ -274,7 +279,7 @@ class PurgeMediaCacheRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
-class PurgeHistoryRestServlet(ClientV1RestServlet):
+class PurgeHistoryRestServlet(RestServlet):
     PATTERNS = client_path_patterns(
         "/admin/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
     )
@@ -285,9 +290,9 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
         Args:
             hs (synapse.server.HomeServer)
         """
-        super(PurgeHistoryRestServlet, self).__init__(hs)
         self.pagination_handler = hs.get_pagination_handler()
         self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, event_id):
@@ -371,7 +376,7 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
         }))
 
 
-class PurgeHistoryStatusRestServlet(ClientV1RestServlet):
+class PurgeHistoryStatusRestServlet(RestServlet):
     PATTERNS = client_path_patterns(
         "/admin/purge_history_status/(?P<purge_id>[^/]+)"
     )
@@ -382,8 +387,8 @@ class PurgeHistoryStatusRestServlet(ClientV1RestServlet):
         Args:
             hs (synapse.server.HomeServer)
         """
-        super(PurgeHistoryStatusRestServlet, self).__init__(hs)
         self.pagination_handler = hs.get_pagination_handler()
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_GET(self, request, purge_id):
@@ -400,12 +405,12 @@ class PurgeHistoryStatusRestServlet(ClientV1RestServlet):
         defer.returnValue((200, purge_status.asdict()))
 
 
-class DeactivateAccountRestServlet(ClientV1RestServlet):
+class DeactivateAccountRestServlet(RestServlet):
     PATTERNS = client_path_patterns("/admin/deactivate/(?P<target_user_id>[^/]*)")
 
     def __init__(self, hs):
-        super(DeactivateAccountRestServlet, self).__init__(hs)
         self._deactivate_account_handler = hs.get_deactivate_account_handler()
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_POST(self, request, target_user_id):
@@ -438,7 +443,7 @@ class DeactivateAccountRestServlet(ClientV1RestServlet):
         }))
 
 
-class ShutdownRoomRestServlet(ClientV1RestServlet):
+class ShutdownRoomRestServlet(RestServlet):
     """Shuts down a room by removing all local users from the room and blocking
     all future invites and joins to the room. Any local aliases will be repointed
     to a new room created by `new_room_user_id` and kicked users will be auto
@@ -452,12 +457,13 @@ class ShutdownRoomRestServlet(ClientV1RestServlet):
     )
 
     def __init__(self, hs):
-        super(ShutdownRoomRestServlet, self).__init__(hs)
+        self.hs = hs
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
         self._room_creation_handler = hs.get_room_creation_handler()
         self.event_creation_handler = hs.get_event_creation_handler()
         self.room_member_handler = hs.get_room_member_handler()
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id):
@@ -564,15 +570,15 @@ class ShutdownRoomRestServlet(ClientV1RestServlet):
         }))
 
 
-class QuarantineMediaInRoom(ClientV1RestServlet):
+class QuarantineMediaInRoom(RestServlet):
     """Quarantines all media in a room so that no one can download it via
     this server.
     """
     PATTERNS = client_path_patterns("/admin/quarantine_media/(?P<room_id>[^/]+)")
 
     def __init__(self, hs):
-        super(QuarantineMediaInRoom, self).__init__(hs)
         self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id):
@@ -588,13 +594,12 @@ class QuarantineMediaInRoom(ClientV1RestServlet):
         defer.returnValue((200, {"num_quarantined": num_quarantined}))
 
 
-class ListMediaInRoom(ClientV1RestServlet):
+class ListMediaInRoom(RestServlet):
     """Lists all of the media in a given room.
     """
     PATTERNS = client_path_patterns("/admin/room/(?P<room_id>[^/]+)/media")
 
     def __init__(self, hs):
-        super(ListMediaInRoom, self).__init__(hs)
         self.store = hs.get_datastore()
 
     @defer.inlineCallbacks
@@ -609,7 +614,7 @@ class ListMediaInRoom(ClientV1RestServlet):
         defer.returnValue((200, {"local": local_mxcs, "remote": remote_mxcs}))
 
 
-class ResetPasswordRestServlet(ClientV1RestServlet):
+class ResetPasswordRestServlet(RestServlet):
     """Post request to allow an administrator reset password for a user.
     This needs user to have administrator access in Synapse.
         Example:
@@ -626,7 +631,6 @@ class ResetPasswordRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
-        super(ResetPasswordRestServlet, self).__init__(hs)
         self.hs = hs
         self.auth = hs.get_auth()
         self._set_password_handler = hs.get_set_password_handler()
@@ -653,7 +657,7 @@ class ResetPasswordRestServlet(ClientV1RestServlet):
         defer.returnValue((200, {}))
 
 
-class GetUsersPaginatedRestServlet(ClientV1RestServlet):
+class GetUsersPaginatedRestServlet(RestServlet):
     """Get request to get specific number of users from Synapse.
     This needs user to have administrator access in Synapse.
         Example:
@@ -666,7 +670,6 @@ class GetUsersPaginatedRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
-        super(GetUsersPaginatedRestServlet, self).__init__(hs)
         self.hs = hs
         self.auth = hs.get_auth()
         self.handlers = hs.get_handlers()
@@ -736,7 +739,7 @@ class GetUsersPaginatedRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
-class SearchUsersRestServlet(ClientV1RestServlet):
+class SearchUsersRestServlet(RestServlet):
     """Get request to search user table for specific users according to
     search term.
     This needs user to have administrator access in Synapse.
@@ -750,7 +753,6 @@ class SearchUsersRestServlet(ClientV1RestServlet):
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
-        super(SearchUsersRestServlet, self).__init__(hs)
         self.hs = hs
         self.auth = hs.get_auth()
         self.handlers = hs.get_handlers()
@@ -784,15 +786,15 @@ class SearchUsersRestServlet(ClientV1RestServlet):
         defer.returnValue((200, ret))
 
 
-class DeleteGroupAdminRestServlet(ClientV1RestServlet):
+class DeleteGroupAdminRestServlet(RestServlet):
     """Allows deleting of local groups
     """
     PATTERNS = client_path_patterns("/admin/delete_group/(?P<group_id>[^/]*)")
 
     def __init__(self, hs):
-        super(DeleteGroupAdminRestServlet, self).__init__(hs)
         self.group_server = hs.get_groups_server_handler()
         self.is_mine_id = hs.is_mine_id
+        self.auth = hs.get_auth()
 
     @defer.inlineCallbacks
     def on_POST(self, request, group_id):
@@ -809,7 +811,7 @@ class DeleteGroupAdminRestServlet(ClientV1RestServlet):
         defer.returnValue((200, {}))
 
 
-class AccountValidityRenewServlet(ClientV1RestServlet):
+class AccountValidityRenewServlet(RestServlet):
     PATTERNS = client_path_patterns("/admin/account_validity/validity$")
 
     def __init__(self, hs):
@@ -817,8 +819,6 @@ class AccountValidityRenewServlet(ClientV1RestServlet):
         Args:
             hs (synapse.server.HomeServer): server
         """
-        super(AccountValidityRenewServlet, self).__init__(hs)
-
         self.hs = hs
         self.account_activity_handler = hs.get_account_validity_handler()
         self.auth = hs.get_auth()
-- 
cgit 1.5.1


From 8e9ca8353730949290460f9183c5ee48941d1f75 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 May 2019 15:18:58 +0100
Subject: Move admin API to a new prefix

---
 docs/admin_api/account_validity.rst   |  2 +-
 docs/admin_api/delete_group.md        |  2 +-
 docs/admin_api/media_admin_api.md     |  2 +-
 docs/admin_api/purge_history_api.rst  |  4 +-
 docs/admin_api/purge_remote_media.rst |  2 +-
 docs/admin_api/register_api.rst       |  4 +-
 docs/admin_api/user_admin_api.rst     |  6 +--
 docs/admin_api/version_api.rst        |  2 +-
 synapse/app/homeserver.py             |  2 +
 synapse/config/server.py              |  4 +-
 synapse/rest/__init__.py              | 12 +++++-
 synapse/rest/client/v1/admin.py       | 70 +++++++++++++++++++++++++----------
 12 files changed, 76 insertions(+), 36 deletions(-)

(limited to 'synapse/rest')

diff --git a/docs/admin_api/account_validity.rst b/docs/admin_api/account_validity.rst
index c26353752f..7559de4c57 100644
--- a/docs/admin_api/account_validity.rst
+++ b/docs/admin_api/account_validity.rst
@@ -13,7 +13,7 @@ This API extends the validity of an account by as much time as configured in the
 
 The API is::
 
-    POST /_matrix/client/unstable/admin/account_validity/validity
+    POST /_synapse/admin/v1/account_validity/validity
 
 with the following body:
 
diff --git a/docs/admin_api/delete_group.md b/docs/admin_api/delete_group.md
index d703d108b0..1710488ea8 100644
--- a/docs/admin_api/delete_group.md
+++ b/docs/admin_api/delete_group.md
@@ -8,7 +8,7 @@ being deleted.
 The API is:
 
 ```
-POST /_matrix/client/r0/admin/delete_group/<group_id>
+POST /_synapse/admin/v1/delete_group/<group_id>
 ```
 
 including an `access_token` of a server admin.
diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md
index abdbc1ea86..5e9f8e5d84 100644
--- a/docs/admin_api/media_admin_api.md
+++ b/docs/admin_api/media_admin_api.md
@@ -4,7 +4,7 @@ This API gets a list of known media in a room.
 
 The API is:
 ```
-GET /_matrix/client/r0/admin/room/<room_id>/media
+GET /_synapse/admin/v1/room/<room_id>/media
 ```
 including an `access_token` of a server admin.
 
diff --git a/docs/admin_api/purge_history_api.rst b/docs/admin_api/purge_history_api.rst
index a5c3dc8149..f7be226fd9 100644
--- a/docs/admin_api/purge_history_api.rst
+++ b/docs/admin_api/purge_history_api.rst
@@ -10,7 +10,7 @@ paginate further back in the room from the point being purged from.
 
 The API is:
 
-``POST /_matrix/client/r0/admin/purge_history/<room_id>[/<event_id>]``
+``POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]``
 
 including an ``access_token`` of a server admin.
 
@@ -49,7 +49,7 @@ Purge status query
 
 It is possible to poll for updates on recent purges with a second API;
 
-``GET /_matrix/client/r0/admin/purge_history_status/<purge_id>``
+``GET /_synapse/admin/v1/purge_history_status/<purge_id>``
 
 (again, with a suitable ``access_token``). This API returns a JSON body like
 the following:
diff --git a/docs/admin_api/purge_remote_media.rst b/docs/admin_api/purge_remote_media.rst
index 5deb02a3df..dacd5bc8fb 100644
--- a/docs/admin_api/purge_remote_media.rst
+++ b/docs/admin_api/purge_remote_media.rst
@@ -6,7 +6,7 @@ media.
 
 The API is::
 
-    POST /_matrix/client/r0/admin/purge_media_cache?before_ts=<unix_timestamp_in_ms>&access_token=<access_token>
+    POST /_synapse/admin/v1/purge_media_cache?before_ts=<unix_timestamp_in_ms>&access_token=<access_token>
 
     {}
 
diff --git a/docs/admin_api/register_api.rst b/docs/admin_api/register_api.rst
index 084e74ebf5..3a63109aa0 100644
--- a/docs/admin_api/register_api.rst
+++ b/docs/admin_api/register_api.rst
@@ -12,7 +12,7 @@ is not enabled.
 
 To fetch the nonce, you need to request one from the API::
 
-  > GET /_matrix/client/r0/admin/register
+  > GET /_synapse/admin/v1/register
 
   < {"nonce": "thisisanonce"}
 
@@ -22,7 +22,7 @@ body containing the nonce, username, password, whether they are an admin
 
 As an example::
 
-  > POST /_matrix/client/r0/admin/register
+  > POST /_synapse/admin/v1/register
   > {
      "nonce": "thisisanonce",
      "username": "pepper_roni",
diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst
index d17121a188..8aca4f158d 100644
--- a/docs/admin_api/user_admin_api.rst
+++ b/docs/admin_api/user_admin_api.rst
@@ -5,7 +5,7 @@ This API returns information about a specific user account.
 
 The api is::
 
-    GET /_matrix/client/r0/admin/whois/<user_id>
+    GET /_synapse/admin/v1/whois/<user_id>
 
 including an ``access_token`` of a server admin.
 
@@ -50,7 +50,7 @@ references to it).
 
 The api is::
 
-    POST /_matrix/client/r0/admin/deactivate/<user_id>
+    POST /_synapse/admin/v1/deactivate/<user_id>
 
 with a body of:
 
@@ -73,7 +73,7 @@ Changes the password of another user.
 
 The api is::
 
-    POST /_matrix/client/r0/admin/reset_password/<user_id>
+    POST /_synapse/admin/v1/reset_password/<user_id>
 
 with a body of:
 
diff --git a/docs/admin_api/version_api.rst b/docs/admin_api/version_api.rst
index 30a91b5f43..6d66543b96 100644
--- a/docs/admin_api/version_api.rst
+++ b/docs/admin_api/version_api.rst
@@ -8,7 +8,7 @@ contains Synapse version information).
 
 The api is::
 
-    GET /_matrix/client/r0/admin/server_version
+    GET /_synapse/admin/v1/server_version
 
 including an ``access_token`` of a server admin.
 
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 79be977ea6..3d7db61d14 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -62,6 +62,7 @@ from synapse.python_dependencies import check_requirements
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
+from synapse.rest.client.v1.admin import AdminRestResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.rest.well_known import WellKnownResource
@@ -180,6 +181,7 @@ class SynapseHomeServer(HomeServer):
                 "/_matrix/client/v2_alpha": client_resource,
                 "/_matrix/client/versions": client_resource,
                 "/.well-known/matrix/client": WellKnownResource(self),
+                "/_synapse/admin": AdminRestResource(self),
             })
 
             if self.get_config().saml2_enabled:
diff --git a/synapse/config/server.py b/synapse/config/server.py
index cdf1e4d286..cc232422b9 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -388,8 +388,8 @@ class ServerConfig(Config):
         #
         # Valid resource names are:
         #
-        #   client: the client-server API (/_matrix/client). Also implies 'media' and
-        #       'static'.
+        #   client: the client-server API (/_matrix/client), and the synapse admin
+        #       API (/_synapse/admin). Also implies 'media' and 'static'.
         #
         #   consent: user consent forms (/_matrix/consent). See
         #       docs/consent_tracking.md.
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index a66885d349..6bc50f78e1 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -58,8 +58,14 @@ from synapse.rest.client.v2_alpha import (
 
 
 class ClientRestResource(JsonResource):
-    """A resource for version 1 of the matrix client API."""
+    """Matrix Client API REST resource.
 
+    This gets mounted at various points under /_matrix/client, including:
+       * /_matrix/client/r0
+       * /_matrix/client/api/v1
+       * /_matrix/client/unstable
+       * etc
+    """
     def __init__(self, hs):
         JsonResource.__init__(self, hs, canonical_json=False)
         self.register_servlets(self, hs)
@@ -82,7 +88,6 @@ class ClientRestResource(JsonResource):
         presence.register_servlets(hs, client_resource)
         directory.register_servlets(hs, client_resource)
         voip.register_servlets(hs, client_resource)
-        admin.register_servlets(hs, client_resource)
         pusher.register_servlets(hs, client_resource)
         push_rule.register_servlets(hs, client_resource)
         logout.register_servlets(hs, client_resource)
@@ -111,3 +116,6 @@ class ClientRestResource(JsonResource):
         room_upgrade_rest_servlet.register_servlets(hs, client_resource)
         capabilities.register_servlets(hs, client_resource)
         account_validity.register_servlets(hs, client_resource)
+
+        # moving to /_synapse/admin
+        admin.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 8efa457d15..d4f83e4ae8 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -18,6 +18,7 @@ import hashlib
 import hmac
 import logging
 import platform
+import re
 
 from six import text_type
 from six.moves import http_client
@@ -27,6 +28,7 @@ from twisted.internet import defer
 import synapse
 from synapse.api.constants import Membership, UserTypes
 from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
+from synapse.http.server import JsonResource
 from synapse.http.servlet import (
     RestServlet,
     assert_params_in_dict,
@@ -37,13 +39,33 @@ from synapse.http.servlet import (
 from synapse.types import UserID, create_requester
 from synapse.util.versionstring import get_version_string
 
-from .base import client_path_patterns
-
 logger = logging.getLogger(__name__)
 
 
+def historical_admin_path_patterns(path_regex):
+    """Returns the list of patterns for an admin endpoint, including historical ones
+
+    This is a backwards-compatibility hack. Previously, the Admin API was exposed at
+    various paths under /_matrix/client. This function returns a list of patterns
+    matching those paths (as well as the new one), so that existing scripts which rely
+    on the endpoints being available there are not broken.
+
+    Note that this should only be used for existing endpoints: new ones should just
+    register for the /_synapse/admin path.
+    """
+    return list(
+        re.compile(prefix + path_regex)
+        for prefix in (
+            "^/_synapse/admin/v1",
+            "^/_matrix/client/api/v1/admin",
+            "^/_matrix/client/unstable/admin",
+            "^/_matrix/client/r0/admin"
+        )
+    )
+
+
 class UsersRestServlet(RestServlet):
-    PATTERNS = client_path_patterns("/admin/users/(?P<user_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/users/(?P<user_id>[^/]*)")
 
     def __init__(self, hs):
         self.hs = hs
@@ -72,7 +94,7 @@ class UsersRestServlet(RestServlet):
 
 
 class VersionServlet(RestServlet):
-    PATTERNS = client_path_patterns("/admin/server_version")
+    PATTERNS = historical_admin_path_patterns("/server_version")
 
     def __init__(self, hs):
         self.auth = hs.get_auth()
@@ -100,7 +122,7 @@ class UserRegisterServlet(RestServlet):
          nonces (dict[str, int]): The nonces that we will accept. A dict of
              nonce to the time it was generated, in int seconds.
     """
-    PATTERNS = client_path_patterns("/admin/register")
+    PATTERNS = historical_admin_path_patterns("/register")
     NONCE_TIMEOUT = 60
 
     def __init__(self, hs):
@@ -231,7 +253,7 @@ class UserRegisterServlet(RestServlet):
 
 
 class WhoisRestServlet(RestServlet):
-    PATTERNS = client_path_patterns("/admin/whois/(?P<user_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/whois/(?P<user_id>[^/]*)")
 
     def __init__(self, hs):
         self.hs = hs
@@ -257,7 +279,7 @@ class WhoisRestServlet(RestServlet):
 
 
 class PurgeMediaCacheRestServlet(RestServlet):
-    PATTERNS = client_path_patterns("/admin/purge_media_cache")
+    PATTERNS = historical_admin_path_patterns("/purge_media_cache")
 
     def __init__(self, hs):
         self.media_repository = hs.get_media_repository()
@@ -280,8 +302,8 @@ class PurgeMediaCacheRestServlet(RestServlet):
 
 
 class PurgeHistoryRestServlet(RestServlet):
-    PATTERNS = client_path_patterns(
-        "/admin/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
+    PATTERNS = historical_admin_path_patterns(
+        "/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
     )
 
     def __init__(self, hs):
@@ -377,8 +399,8 @@ class PurgeHistoryRestServlet(RestServlet):
 
 
 class PurgeHistoryStatusRestServlet(RestServlet):
-    PATTERNS = client_path_patterns(
-        "/admin/purge_history_status/(?P<purge_id>[^/]+)"
+    PATTERNS = historical_admin_path_patterns(
+        "/purge_history_status/(?P<purge_id>[^/]+)"
     )
 
     def __init__(self, hs):
@@ -406,7 +428,7 @@ class PurgeHistoryStatusRestServlet(RestServlet):
 
 
 class DeactivateAccountRestServlet(RestServlet):
-    PATTERNS = client_path_patterns("/admin/deactivate/(?P<target_user_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/deactivate/(?P<target_user_id>[^/]*)")
 
     def __init__(self, hs):
         self._deactivate_account_handler = hs.get_deactivate_account_handler()
@@ -449,7 +471,7 @@ class ShutdownRoomRestServlet(RestServlet):
     to a new room created by `new_room_user_id` and kicked users will be auto
     joined to the new room.
     """
-    PATTERNS = client_path_patterns("/admin/shutdown_room/(?P<room_id>[^/]+)")
+    PATTERNS = historical_admin_path_patterns("/shutdown_room/(?P<room_id>[^/]+)")
 
     DEFAULT_MESSAGE = (
         "Sharing illegal content on this server is not permitted and rooms in"
@@ -574,7 +596,7 @@ class QuarantineMediaInRoom(RestServlet):
     """Quarantines all media in a room so that no one can download it via
     this server.
     """
-    PATTERNS = client_path_patterns("/admin/quarantine_media/(?P<room_id>[^/]+)")
+    PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P<room_id>[^/]+)")
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
@@ -597,7 +619,7 @@ class QuarantineMediaInRoom(RestServlet):
 class ListMediaInRoom(RestServlet):
     """Lists all of the media in a given room.
     """
-    PATTERNS = client_path_patterns("/admin/room/(?P<room_id>[^/]+)/media")
+    PATTERNS = historical_admin_path_patterns("/room/(?P<room_id>[^/]+)/media")
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
@@ -627,7 +649,7 @@ class ResetPasswordRestServlet(RestServlet):
         Returns:
             200 OK with empty object if success otherwise an error.
         """
-    PATTERNS = client_path_patterns("/admin/reset_password/(?P<target_user_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/reset_password/(?P<target_user_id>[^/]*)")
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
@@ -666,7 +688,7 @@ class GetUsersPaginatedRestServlet(RestServlet):
         Returns:
             200 OK with json object {list[dict[str, Any]], count} or empty object.
         """
-    PATTERNS = client_path_patterns("/admin/users_paginate/(?P<target_user_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/users_paginate/(?P<target_user_id>[^/]*)")
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
@@ -749,7 +771,7 @@ class SearchUsersRestServlet(RestServlet):
         Returns:
             200 OK with json object {list[dict[str, Any]], count} or empty object.
     """
-    PATTERNS = client_path_patterns("/admin/search_users/(?P<target_user_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/search_users/(?P<target_user_id>[^/]*)")
 
     def __init__(self, hs):
         self.store = hs.get_datastore()
@@ -789,7 +811,7 @@ class SearchUsersRestServlet(RestServlet):
 class DeleteGroupAdminRestServlet(RestServlet):
     """Allows deleting of local groups
     """
-    PATTERNS = client_path_patterns("/admin/delete_group/(?P<group_id>[^/]*)")
+    PATTERNS = historical_admin_path_patterns("/delete_group/(?P<group_id>[^/]*)")
 
     def __init__(self, hs):
         self.group_server = hs.get_groups_server_handler()
@@ -812,7 +834,7 @@ class DeleteGroupAdminRestServlet(RestServlet):
 
 
 class AccountValidityRenewServlet(RestServlet):
-    PATTERNS = client_path_patterns("/admin/account_validity/validity$")
+    PATTERNS = historical_admin_path_patterns("/account_validity/validity$")
 
     def __init__(self, hs):
         """
@@ -847,6 +869,14 @@ class AccountValidityRenewServlet(RestServlet):
         defer.returnValue((200, res))
 
 
+class AdminRestResource(JsonResource):
+    """The REST resource which gets mounted at /_synapse/admin"""
+
+    def __init__(self, hs):
+        JsonResource.__init__(self, hs, canonical_json=False)
+        register_servlets(hs, self)
+
+
 def register_servlets(hs, http_server):
     WhoisRestServlet(hs).register(http_server)
     PurgeMediaCacheRestServlet(hs).register(http_server)
-- 
cgit 1.5.1


From 40e576e29cf6f06d6b5244c5d1df34cf33b1f556 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 May 2019 15:32:38 +0100
Subject: Move admin api impl to its own package

It doesn't really belong under rest/client/v1 any more.
---
 synapse/app/homeserver.py                       |   2 +-
 synapse/rest/__init__.py                        |   5 +-
 synapse/rest/admin/__init__.py                  | 896 ++++++++++++++++++++++++
 synapse/rest/client/v1/admin.py                 | 896 ------------------------
 tests/handlers/test_user_directory.py           |   7 +-
 tests/push/test_email.py                        |   5 +-
 tests/push/test_http.py                         |   5 +-
 tests/rest/admin/__init__.py                    |  14 +
 tests/rest/admin/test_admin.py                  | 617 ++++++++++++++++
 tests/rest/client/test_consent.py               |   5 +-
 tests/rest/client/test_identity.py              |   5 +-
 tests/rest/client/v1/test_admin.py              | 616 ----------------
 tests/rest/client/v1/test_events.py             |   5 +-
 tests/rest/client/v1/test_login.py              |   5 +-
 tests/rest/client/v1/test_rooms.py              |   5 +-
 tests/rest/client/v2_alpha/test_auth.py         |   4 +-
 tests/rest/client/v2_alpha/test_capabilities.py |   6 +-
 tests/rest/client/v2_alpha/test_register.py     |   7 +-
 tests/rest/client/v2_alpha/test_sync.py         |   5 +-
 tests/server_notices/test_consent.py            |   6 +-
 tests/storage/test_client_ips.py                |   5 +-
 21 files changed, 1573 insertions(+), 1548 deletions(-)
 create mode 100644 synapse/rest/admin/__init__.py
 delete mode 100644 synapse/rest/client/v1/admin.py
 create mode 100644 tests/rest/admin/__init__.py
 create mode 100644 tests/rest/admin/test_admin.py
 delete mode 100644 tests/rest/client/v1/test_admin.py

(limited to 'synapse/rest')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 3d7db61d14..1045d28949 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -62,7 +62,7 @@ from synapse.python_dependencies import check_requirements
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
 from synapse.rest import ClientRestResource
-from synapse.rest.client.v1.admin import AdminRestResource
+from synapse.rest.admin import AdminRestResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.rest.well_known import WellKnownResource
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 6bc50f78e1..e8e1bcddea 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -13,11 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import synapse.rest.admin
 from synapse.http.server import JsonResource
 from synapse.rest.client import versions
 from synapse.rest.client.v1 import (
-    admin,
     directory,
     events,
     initial_sync,
@@ -118,4 +117,4 @@ class ClientRestResource(JsonResource):
         account_validity.register_servlets(hs, client_resource)
 
         # moving to /_synapse/admin
-        admin.register_servlets(hs, client_resource)
+        synapse.rest.admin.register_servlets(hs, client_resource)
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
new file mode 100644
index 0000000000..ddf9ced1a3
--- /dev/null
+++ b/synapse/rest/admin/__init__.py
@@ -0,0 +1,896 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2018-2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import hmac
+import logging
+import platform
+import re
+
+from six import text_type
+from six.moves import http_client
+
+from twisted.internet import defer
+
+import synapse
+from synapse.api.constants import Membership, UserTypes
+from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
+from synapse.http.server import JsonResource
+from synapse.http.servlet import (
+    RestServlet,
+    assert_params_in_dict,
+    parse_integer,
+    parse_json_object_from_request,
+    parse_string,
+)
+from synapse.types import UserID, create_requester
+from synapse.util.versionstring import get_version_string
+
+logger = logging.getLogger(__name__)
+
+
+def historical_admin_path_patterns(path_regex):
+    """Returns the list of patterns for an admin endpoint, including historical ones
+
+    This is a backwards-compatibility hack. Previously, the Admin API was exposed at
+    various paths under /_matrix/client. This function returns a list of patterns
+    matching those paths (as well as the new one), so that existing scripts which rely
+    on the endpoints being available there are not broken.
+
+    Note that this should only be used for existing endpoints: new ones should just
+    register for the /_synapse/admin path.
+    """
+    return list(
+        re.compile(prefix + path_regex)
+        for prefix in (
+            "^/_synapse/admin/v1",
+            "^/_matrix/client/api/v1/admin",
+            "^/_matrix/client/unstable/admin",
+            "^/_matrix/client/r0/admin"
+        )
+    )
+
+
+class UsersRestServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns("/users/(?P<user_id>[^/]*)")
+
+    def __init__(self, hs):
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.handlers = hs.get_handlers()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, user_id):
+        target_user = UserID.from_string(user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        # To allow all users to get the users list
+        # if not is_admin and target_user != auth_user:
+        #     raise AuthError(403, "You are not a server admin")
+
+        if not self.hs.is_mine(target_user):
+            raise SynapseError(400, "Can only users a local user")
+
+        ret = yield self.handlers.admin_handler.get_users()
+
+        defer.returnValue((200, ret))
+
+
+class VersionServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns("/server_version")
+
+    def __init__(self, hs):
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        ret = {
+            'server_version': get_version_string(synapse),
+            'python_version': platform.python_version(),
+        }
+
+        defer.returnValue((200, ret))
+
+
+class UserRegisterServlet(RestServlet):
+    """
+    Attributes:
+         NONCE_TIMEOUT (int): Seconds until a generated nonce won't be accepted
+         nonces (dict[str, int]): The nonces that we will accept. A dict of
+             nonce to the time it was generated, in int seconds.
+    """
+    PATTERNS = historical_admin_path_patterns("/register")
+    NONCE_TIMEOUT = 60
+
+    def __init__(self, hs):
+        self.handlers = hs.get_handlers()
+        self.reactor = hs.get_reactor()
+        self.nonces = {}
+        self.hs = hs
+
+    def _clear_old_nonces(self):
+        """
+        Clear out old nonces that are older than NONCE_TIMEOUT.
+        """
+        now = int(self.reactor.seconds())
+
+        for k, v in list(self.nonces.items()):
+            if now - v > self.NONCE_TIMEOUT:
+                del self.nonces[k]
+
+    def on_GET(self, request):
+        """
+        Generate a new nonce.
+        """
+        self._clear_old_nonces()
+
+        nonce = self.hs.get_secrets().token_hex(64)
+        self.nonces[nonce] = int(self.reactor.seconds())
+        return (200, {"nonce": nonce})
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        self._clear_old_nonces()
+
+        if not self.hs.config.registration_shared_secret:
+            raise SynapseError(400, "Shared secret registration is not enabled")
+
+        body = parse_json_object_from_request(request)
+
+        if "nonce" not in body:
+            raise SynapseError(
+                400, "nonce must be specified", errcode=Codes.BAD_JSON,
+            )
+
+        nonce = body["nonce"]
+
+        if nonce not in self.nonces:
+            raise SynapseError(
+                400, "unrecognised nonce",
+            )
+
+        # Delete the nonce, so it can't be reused, even if it's invalid
+        del self.nonces[nonce]
+
+        if "username" not in body:
+            raise SynapseError(
+                400, "username must be specified", errcode=Codes.BAD_JSON,
+            )
+        else:
+            if (
+                not isinstance(body['username'], text_type)
+                or len(body['username']) > 512
+            ):
+                raise SynapseError(400, "Invalid username")
+
+            username = body["username"].encode("utf-8")
+            if b"\x00" in username:
+                raise SynapseError(400, "Invalid username")
+
+        if "password" not in body:
+            raise SynapseError(
+                400, "password must be specified", errcode=Codes.BAD_JSON,
+            )
+        else:
+            if (
+                not isinstance(body['password'], text_type)
+                or len(body['password']) > 512
+            ):
+                raise SynapseError(400, "Invalid password")
+
+            password = body["password"].encode("utf-8")
+            if b"\x00" in password:
+                raise SynapseError(400, "Invalid password")
+
+        admin = body.get("admin", None)
+        user_type = body.get("user_type", None)
+
+        if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES:
+            raise SynapseError(400, "Invalid user type")
+
+        got_mac = body["mac"]
+
+        want_mac = hmac.new(
+            key=self.hs.config.registration_shared_secret.encode(),
+            digestmod=hashlib.sha1,
+        )
+        want_mac.update(nonce.encode('utf8'))
+        want_mac.update(b"\x00")
+        want_mac.update(username)
+        want_mac.update(b"\x00")
+        want_mac.update(password)
+        want_mac.update(b"\x00")
+        want_mac.update(b"admin" if admin else b"notadmin")
+        if user_type:
+            want_mac.update(b"\x00")
+            want_mac.update(user_type.encode('utf8'))
+        want_mac = want_mac.hexdigest()
+
+        if not hmac.compare_digest(
+                want_mac.encode('ascii'),
+                got_mac.encode('ascii')
+        ):
+            raise SynapseError(403, "HMAC incorrect")
+
+        # Reuse the parts of RegisterRestServlet to reduce code duplication
+        from synapse.rest.client.v2_alpha.register import RegisterRestServlet
+
+        register = RegisterRestServlet(self.hs)
+
+        (user_id, _) = yield register.registration_handler.register(
+            localpart=body['username'].lower(),
+            password=body["password"],
+            admin=bool(admin),
+            generate_token=False,
+            user_type=user_type,
+        )
+
+        result = yield register._create_registration_details(user_id, body)
+        defer.returnValue((200, result))
+
+
+class WhoisRestServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns("/whois/(?P<user_id>[^/]*)")
+
+    def __init__(self, hs):
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.handlers = hs.get_handlers()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, user_id):
+        target_user = UserID.from_string(user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        auth_user = requester.user
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin and target_user != auth_user:
+            raise AuthError(403, "You are not a server admin")
+
+        if not self.hs.is_mine(target_user):
+            raise SynapseError(400, "Can only whois a local user")
+
+        ret = yield self.handlers.admin_handler.get_whois(target_user)
+
+        defer.returnValue((200, ret))
+
+
+class PurgeMediaCacheRestServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns("/purge_media_cache")
+
+    def __init__(self, hs):
+        self.media_repository = hs.get_media_repository()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        before_ts = parse_integer(request, "before_ts", required=True)
+        logger.info("before_ts: %r", before_ts)
+
+        ret = yield self.media_repository.delete_old_remote_media(before_ts)
+
+        defer.returnValue((200, ret))
+
+
+class PurgeHistoryRestServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns(
+        "/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
+    )
+
+    def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer)
+        """
+        self.pagination_handler = hs.get_pagination_handler()
+        self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, room_id, event_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        body = parse_json_object_from_request(request, allow_empty_body=True)
+
+        delete_local_events = bool(body.get("delete_local_events", False))
+
+        # establish the topological ordering we should keep events from. The
+        # user can provide an event_id in the URL or the request body, or can
+        # provide a timestamp in the request body.
+        if event_id is None:
+            event_id = body.get('purge_up_to_event_id')
+
+        if event_id is not None:
+            event = yield self.store.get_event(event_id)
+
+            if event.room_id != room_id:
+                raise SynapseError(400, "Event is for wrong room.")
+
+            token = yield self.store.get_topological_token_for_event(event_id)
+
+            logger.info(
+                "[purge] purging up to token %s (event_id %s)",
+                token, event_id,
+            )
+        elif 'purge_up_to_ts' in body:
+            ts = body['purge_up_to_ts']
+            if not isinstance(ts, int):
+                raise SynapseError(
+                    400, "purge_up_to_ts must be an int",
+                    errcode=Codes.BAD_JSON,
+                )
+
+            stream_ordering = (
+                yield self.store.find_first_stream_ordering_after_ts(ts)
+            )
+
+            r = (
+                yield self.store.get_room_event_after_stream_ordering(
+                    room_id, stream_ordering,
+                )
+            )
+            if not r:
+                logger.warn(
+                    "[purge] purging events not possible: No event found "
+                    "(received_ts %i => stream_ordering %i)",
+                    ts, stream_ordering,
+                )
+                raise SynapseError(
+                    404,
+                    "there is no event to be purged",
+                    errcode=Codes.NOT_FOUND,
+                )
+            (stream, topo, _event_id) = r
+            token = "t%d-%d" % (topo, stream)
+            logger.info(
+                "[purge] purging up to token %s (received_ts %i => "
+                "stream_ordering %i)",
+                token, ts, stream_ordering,
+            )
+        else:
+            raise SynapseError(
+                400,
+                "must specify purge_up_to_event_id or purge_up_to_ts",
+                errcode=Codes.BAD_JSON,
+            )
+
+        purge_id = yield self.pagination_handler.start_purge_history(
+            room_id, token,
+            delete_local_events=delete_local_events,
+        )
+
+        defer.returnValue((200, {
+            "purge_id": purge_id,
+        }))
+
+
+class PurgeHistoryStatusRestServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns(
+        "/purge_history_status/(?P<purge_id>[^/]+)"
+    )
+
+    def __init__(self, hs):
+        """
+
+        Args:
+            hs (synapse.server.HomeServer)
+        """
+        self.pagination_handler = hs.get_pagination_handler()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, purge_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        purge_status = self.pagination_handler.get_purge_status(purge_id)
+        if purge_status is None:
+            raise NotFoundError("purge id '%s' not found" % purge_id)
+
+        defer.returnValue((200, purge_status.asdict()))
+
+
+class DeactivateAccountRestServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns("/deactivate/(?P<target_user_id>[^/]*)")
+
+    def __init__(self, hs):
+        self._deactivate_account_handler = hs.get_deactivate_account_handler()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, target_user_id):
+        body = parse_json_object_from_request(request, allow_empty_body=True)
+        erase = body.get("erase", False)
+        if not isinstance(erase, bool):
+            raise SynapseError(
+                http_client.BAD_REQUEST,
+                "Param 'erase' must be a boolean, if given",
+                Codes.BAD_JSON,
+            )
+
+        UserID.from_string(target_user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        result = yield self._deactivate_account_handler.deactivate_account(
+            target_user_id, erase,
+        )
+        if result:
+            id_server_unbind_result = "success"
+        else:
+            id_server_unbind_result = "no-support"
+
+        defer.returnValue((200, {
+            "id_server_unbind_result": id_server_unbind_result,
+        }))
+
+
+class ShutdownRoomRestServlet(RestServlet):
+    """Shuts down a room by removing all local users from the room and blocking
+    all future invites and joins to the room. Any local aliases will be repointed
+    to a new room created by `new_room_user_id` and kicked users will be auto
+    joined to the new room.
+    """
+    PATTERNS = historical_admin_path_patterns("/shutdown_room/(?P<room_id>[^/]+)")
+
+    DEFAULT_MESSAGE = (
+        "Sharing illegal content on this server is not permitted and rooms in"
+        " violation will be blocked."
+    )
+
+    def __init__(self, hs):
+        self.hs = hs
+        self.store = hs.get_datastore()
+        self.state = hs.get_state_handler()
+        self._room_creation_handler = hs.get_room_creation_handler()
+        self.event_creation_handler = hs.get_event_creation_handler()
+        self.room_member_handler = hs.get_room_member_handler()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        content = parse_json_object_from_request(request)
+        assert_params_in_dict(content, ["new_room_user_id"])
+        new_room_user_id = content["new_room_user_id"]
+
+        room_creator_requester = create_requester(new_room_user_id)
+
+        message = content.get("message", self.DEFAULT_MESSAGE)
+        room_name = content.get("room_name", "Content Violation Notification")
+
+        info = yield self._room_creation_handler.create_room(
+            room_creator_requester,
+            config={
+                "preset": "public_chat",
+                "name": room_name,
+                "power_level_content_override": {
+                    "users_default": -10,
+                },
+            },
+            ratelimit=False,
+        )
+        new_room_id = info["room_id"]
+
+        requester_user_id = requester.user.to_string()
+
+        logger.info(
+            "Shutting down room %r, joining to new room: %r",
+            room_id, new_room_id,
+        )
+
+        # This will work even if the room is already blocked, but that is
+        # desirable in case the first attempt at blocking the room failed below.
+        yield self.store.block_room(room_id, requester_user_id)
+
+        users = yield self.state.get_current_users_in_room(room_id)
+        kicked_users = []
+        failed_to_kick_users = []
+        for user_id in users:
+            if not self.hs.is_mine_id(user_id):
+                continue
+
+            logger.info("Kicking %r from %r...", user_id, room_id)
+
+            try:
+                target_requester = create_requester(user_id)
+                yield self.room_member_handler.update_membership(
+                    requester=target_requester,
+                    target=target_requester.user,
+                    room_id=room_id,
+                    action=Membership.LEAVE,
+                    content={},
+                    ratelimit=False,
+                    require_consent=False,
+                )
+
+                yield self.room_member_handler.forget(target_requester.user, room_id)
+
+                yield self.room_member_handler.update_membership(
+                    requester=target_requester,
+                    target=target_requester.user,
+                    room_id=new_room_id,
+                    action=Membership.JOIN,
+                    content={},
+                    ratelimit=False,
+                    require_consent=False,
+                )
+
+                kicked_users.append(user_id)
+            except Exception:
+                logger.exception(
+                    "Failed to leave old room and join new room for %r", user_id,
+                )
+                failed_to_kick_users.append(user_id)
+
+        yield self.event_creation_handler.create_and_send_nonmember_event(
+            room_creator_requester,
+            {
+                "type": "m.room.message",
+                "content": {"body": message, "msgtype": "m.text"},
+                "room_id": new_room_id,
+                "sender": new_room_user_id,
+            },
+            ratelimit=False,
+        )
+
+        aliases_for_room = yield self.store.get_aliases_for_room(room_id)
+
+        yield self.store.update_aliases_for_room(
+            room_id, new_room_id, requester_user_id
+        )
+
+        defer.returnValue((200, {
+            "kicked_users": kicked_users,
+            "failed_to_kick_users": failed_to_kick_users,
+            "local_aliases": aliases_for_room,
+            "new_room_id": new_room_id,
+        }))
+
+
+class QuarantineMediaInRoom(RestServlet):
+    """Quarantines all media in a room so that no one can download it via
+    this server.
+    """
+    PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P<room_id>[^/]+)")
+
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        num_quarantined = yield self.store.quarantine_media_ids_in_room(
+            room_id, requester.user.to_string(),
+        )
+
+        defer.returnValue((200, {"num_quarantined": num_quarantined}))
+
+
+class ListMediaInRoom(RestServlet):
+    """Lists all of the media in a given room.
+    """
+    PATTERNS = historical_admin_path_patterns("/room/(?P<room_id>[^/]+)/media")
+
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, room_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id)
+
+        defer.returnValue((200, {"local": local_mxcs, "remote": remote_mxcs}))
+
+
+class ResetPasswordRestServlet(RestServlet):
+    """Post request to allow an administrator reset password for a user.
+    This needs user to have administrator access in Synapse.
+        Example:
+            http://localhost:8008/_matrix/client/api/v1/admin/reset_password/
+            @user:to_reset_password?access_token=admin_access_token
+        JsonBodyToSend:
+            {
+                "new_password": "secret"
+            }
+        Returns:
+            200 OK with empty object if success otherwise an error.
+        """
+    PATTERNS = historical_admin_path_patterns("/reset_password/(?P<target_user_id>[^/]*)")
+
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self._set_password_handler = hs.get_set_password_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, target_user_id):
+        """Post request to allow an administrator reset password for a user.
+        This needs user to have administrator access in Synapse.
+        """
+        UserID.from_string(target_user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        params = parse_json_object_from_request(request)
+        assert_params_in_dict(params, ["new_password"])
+        new_password = params['new_password']
+
+        yield self._set_password_handler.set_password(
+            target_user_id, new_password, requester
+        )
+        defer.returnValue((200, {}))
+
+
+class GetUsersPaginatedRestServlet(RestServlet):
+    """Get request to get specific number of users from Synapse.
+    This needs user to have administrator access in Synapse.
+        Example:
+            http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/
+            @admin:user?access_token=admin_access_token&start=0&limit=10
+        Returns:
+            200 OK with json object {list[dict[str, Any]], count} or empty object.
+        """
+    PATTERNS = historical_admin_path_patterns("/users_paginate/(?P<target_user_id>[^/]*)")
+
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.handlers = hs.get_handlers()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, target_user_id):
+        """Get request to get specific number of users from Synapse.
+        This needs user to have administrator access in Synapse.
+        """
+        target_user = UserID.from_string(target_user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        # To allow all users to get the users list
+        # if not is_admin and target_user != auth_user:
+        #     raise AuthError(403, "You are not a server admin")
+
+        if not self.hs.is_mine(target_user):
+            raise SynapseError(400, "Can only users a local user")
+
+        order = "name"  # order by name in user table
+        start = parse_integer(request, "start", required=True)
+        limit = parse_integer(request, "limit", required=True)
+
+        logger.info("limit: %s, start: %s", limit, start)
+
+        ret = yield self.handlers.admin_handler.get_users_paginate(
+            order, start, limit
+        )
+        defer.returnValue((200, ret))
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, target_user_id):
+        """Post request to get specific number of users from Synapse..
+        This needs user to have administrator access in Synapse.
+        Example:
+            http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/
+            @admin:user?access_token=admin_access_token
+        JsonBodyToSend:
+            {
+                "start": "0",
+                "limit": "10
+            }
+        Returns:
+            200 OK with json object {list[dict[str, Any]], count} or empty object.
+        """
+        UserID.from_string(target_user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        order = "name"  # order by name in user table
+        params = parse_json_object_from_request(request)
+        assert_params_in_dict(params, ["limit", "start"])
+        limit = params['limit']
+        start = params['start']
+        logger.info("limit: %s, start: %s", limit, start)
+
+        ret = yield self.handlers.admin_handler.get_users_paginate(
+            order, start, limit
+        )
+        defer.returnValue((200, ret))
+
+
+class SearchUsersRestServlet(RestServlet):
+    """Get request to search user table for specific users according to
+    search term.
+    This needs user to have administrator access in Synapse.
+        Example:
+            http://localhost:8008/_matrix/client/api/v1/admin/search_users/
+            @admin:user?access_token=admin_access_token&term=alice
+        Returns:
+            200 OK with json object {list[dict[str, Any]], count} or empty object.
+    """
+    PATTERNS = historical_admin_path_patterns("/search_users/(?P<target_user_id>[^/]*)")
+
+    def __init__(self, hs):
+        self.store = hs.get_datastore()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.handlers = hs.get_handlers()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, target_user_id):
+        """Get request to search user table for specific users according to
+        search term.
+        This needs user to have a administrator access in Synapse.
+        """
+        target_user = UserID.from_string(target_user_id)
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        # To allow all users to get the users list
+        # if not is_admin and target_user != auth_user:
+        #     raise AuthError(403, "You are not a server admin")
+
+        if not self.hs.is_mine(target_user):
+            raise SynapseError(400, "Can only users a local user")
+
+        term = parse_string(request, "term", required=True)
+        logger.info("term: %s ", term)
+
+        ret = yield self.handlers.admin_handler.search_users(
+            term
+        )
+        defer.returnValue((200, ret))
+
+
+class DeleteGroupAdminRestServlet(RestServlet):
+    """Allows deleting of local groups
+    """
+    PATTERNS = historical_admin_path_patterns("/delete_group/(?P<group_id>[^/]*)")
+
+    def __init__(self, hs):
+        self.group_server = hs.get_groups_server_handler()
+        self.is_mine_id = hs.is_mine_id
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, group_id):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        if not self.is_mine_id(group_id):
+            raise SynapseError(400, "Can only delete local groups")
+
+        yield self.group_server.delete_group(group_id, requester.user.to_string())
+        defer.returnValue((200, {}))
+
+
+class AccountValidityRenewServlet(RestServlet):
+    PATTERNS = historical_admin_path_patterns("/account_validity/validity$")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        self.hs = hs
+        self.account_activity_handler = hs.get_account_validity_handler()
+        self.auth = hs.get_auth()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        is_admin = yield self.auth.is_server_admin(requester.user)
+
+        if not is_admin:
+            raise AuthError(403, "You are not a server admin")
+
+        body = parse_json_object_from_request(request)
+
+        if "user_id" not in body:
+            raise SynapseError(400, "Missing property 'user_id' in the request body")
+
+        expiration_ts = yield self.account_activity_handler.renew_account_for_user(
+            body["user_id"], body.get("expiration_ts"),
+            not body.get("enable_renewal_emails", True),
+        )
+
+        res = {
+            "expiration_ts": expiration_ts,
+        }
+        defer.returnValue((200, res))
+
+
+class AdminRestResource(JsonResource):
+    """The REST resource which gets mounted at /_synapse/admin"""
+
+    def __init__(self, hs):
+        JsonResource.__init__(self, hs, canonical_json=False)
+        register_servlets(hs, self)
+
+
+def register_servlets(hs, http_server):
+    WhoisRestServlet(hs).register(http_server)
+    PurgeMediaCacheRestServlet(hs).register(http_server)
+    PurgeHistoryStatusRestServlet(hs).register(http_server)
+    DeactivateAccountRestServlet(hs).register(http_server)
+    PurgeHistoryRestServlet(hs).register(http_server)
+    UsersRestServlet(hs).register(http_server)
+    ResetPasswordRestServlet(hs).register(http_server)
+    GetUsersPaginatedRestServlet(hs).register(http_server)
+    SearchUsersRestServlet(hs).register(http_server)
+    ShutdownRoomRestServlet(hs).register(http_server)
+    QuarantineMediaInRoom(hs).register(http_server)
+    ListMediaInRoom(hs).register(http_server)
+    UserRegisterServlet(hs).register(http_server)
+    VersionServlet(hs).register(http_server)
+    DeleteGroupAdminRestServlet(hs).register(http_server)
+    AccountValidityRenewServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
deleted file mode 100644
index d4f83e4ae8..0000000000
--- a/synapse/rest/client/v1/admin.py
+++ /dev/null
@@ -1,896 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import hashlib
-import hmac
-import logging
-import platform
-import re
-
-from six import text_type
-from six.moves import http_client
-
-from twisted.internet import defer
-
-import synapse
-from synapse.api.constants import Membership, UserTypes
-from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
-from synapse.http.server import JsonResource
-from synapse.http.servlet import (
-    RestServlet,
-    assert_params_in_dict,
-    parse_integer,
-    parse_json_object_from_request,
-    parse_string,
-)
-from synapse.types import UserID, create_requester
-from synapse.util.versionstring import get_version_string
-
-logger = logging.getLogger(__name__)
-
-
-def historical_admin_path_patterns(path_regex):
-    """Returns the list of patterns for an admin endpoint, including historical ones
-
-    This is a backwards-compatibility hack. Previously, the Admin API was exposed at
-    various paths under /_matrix/client. This function returns a list of patterns
-    matching those paths (as well as the new one), so that existing scripts which rely
-    on the endpoints being available there are not broken.
-
-    Note that this should only be used for existing endpoints: new ones should just
-    register for the /_synapse/admin path.
-    """
-    return list(
-        re.compile(prefix + path_regex)
-        for prefix in (
-            "^/_synapse/admin/v1",
-            "^/_matrix/client/api/v1/admin",
-            "^/_matrix/client/unstable/admin",
-            "^/_matrix/client/r0/admin"
-        )
-    )
-
-
-class UsersRestServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/users/(?P<user_id>[^/]*)")
-
-    def __init__(self, hs):
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self.handlers = hs.get_handlers()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request, user_id):
-        target_user = UserID.from_string(user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        # To allow all users to get the users list
-        # if not is_admin and target_user != auth_user:
-        #     raise AuthError(403, "You are not a server admin")
-
-        if not self.hs.is_mine(target_user):
-            raise SynapseError(400, "Can only users a local user")
-
-        ret = yield self.handlers.admin_handler.get_users()
-
-        defer.returnValue((200, ret))
-
-
-class VersionServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/server_version")
-
-    def __init__(self, hs):
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        ret = {
-            'server_version': get_version_string(synapse),
-            'python_version': platform.python_version(),
-        }
-
-        defer.returnValue((200, ret))
-
-
-class UserRegisterServlet(RestServlet):
-    """
-    Attributes:
-         NONCE_TIMEOUT (int): Seconds until a generated nonce won't be accepted
-         nonces (dict[str, int]): The nonces that we will accept. A dict of
-             nonce to the time it was generated, in int seconds.
-    """
-    PATTERNS = historical_admin_path_patterns("/register")
-    NONCE_TIMEOUT = 60
-
-    def __init__(self, hs):
-        self.handlers = hs.get_handlers()
-        self.reactor = hs.get_reactor()
-        self.nonces = {}
-        self.hs = hs
-
-    def _clear_old_nonces(self):
-        """
-        Clear out old nonces that are older than NONCE_TIMEOUT.
-        """
-        now = int(self.reactor.seconds())
-
-        for k, v in list(self.nonces.items()):
-            if now - v > self.NONCE_TIMEOUT:
-                del self.nonces[k]
-
-    def on_GET(self, request):
-        """
-        Generate a new nonce.
-        """
-        self._clear_old_nonces()
-
-        nonce = self.hs.get_secrets().token_hex(64)
-        self.nonces[nonce] = int(self.reactor.seconds())
-        return (200, {"nonce": nonce})
-
-    @defer.inlineCallbacks
-    def on_POST(self, request):
-        self._clear_old_nonces()
-
-        if not self.hs.config.registration_shared_secret:
-            raise SynapseError(400, "Shared secret registration is not enabled")
-
-        body = parse_json_object_from_request(request)
-
-        if "nonce" not in body:
-            raise SynapseError(
-                400, "nonce must be specified", errcode=Codes.BAD_JSON,
-            )
-
-        nonce = body["nonce"]
-
-        if nonce not in self.nonces:
-            raise SynapseError(
-                400, "unrecognised nonce",
-            )
-
-        # Delete the nonce, so it can't be reused, even if it's invalid
-        del self.nonces[nonce]
-
-        if "username" not in body:
-            raise SynapseError(
-                400, "username must be specified", errcode=Codes.BAD_JSON,
-            )
-        else:
-            if (
-                not isinstance(body['username'], text_type)
-                or len(body['username']) > 512
-            ):
-                raise SynapseError(400, "Invalid username")
-
-            username = body["username"].encode("utf-8")
-            if b"\x00" in username:
-                raise SynapseError(400, "Invalid username")
-
-        if "password" not in body:
-            raise SynapseError(
-                400, "password must be specified", errcode=Codes.BAD_JSON,
-            )
-        else:
-            if (
-                not isinstance(body['password'], text_type)
-                or len(body['password']) > 512
-            ):
-                raise SynapseError(400, "Invalid password")
-
-            password = body["password"].encode("utf-8")
-            if b"\x00" in password:
-                raise SynapseError(400, "Invalid password")
-
-        admin = body.get("admin", None)
-        user_type = body.get("user_type", None)
-
-        if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES:
-            raise SynapseError(400, "Invalid user type")
-
-        got_mac = body["mac"]
-
-        want_mac = hmac.new(
-            key=self.hs.config.registration_shared_secret.encode(),
-            digestmod=hashlib.sha1,
-        )
-        want_mac.update(nonce.encode('utf8'))
-        want_mac.update(b"\x00")
-        want_mac.update(username)
-        want_mac.update(b"\x00")
-        want_mac.update(password)
-        want_mac.update(b"\x00")
-        want_mac.update(b"admin" if admin else b"notadmin")
-        if user_type:
-            want_mac.update(b"\x00")
-            want_mac.update(user_type.encode('utf8'))
-        want_mac = want_mac.hexdigest()
-
-        if not hmac.compare_digest(
-                want_mac.encode('ascii'),
-                got_mac.encode('ascii')
-        ):
-            raise SynapseError(403, "HMAC incorrect")
-
-        # Reuse the parts of RegisterRestServlet to reduce code duplication
-        from synapse.rest.client.v2_alpha.register import RegisterRestServlet
-
-        register = RegisterRestServlet(self.hs)
-
-        (user_id, _) = yield register.registration_handler.register(
-            localpart=body['username'].lower(),
-            password=body["password"],
-            admin=bool(admin),
-            generate_token=False,
-            user_type=user_type,
-        )
-
-        result = yield register._create_registration_details(user_id, body)
-        defer.returnValue((200, result))
-
-
-class WhoisRestServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/whois/(?P<user_id>[^/]*)")
-
-    def __init__(self, hs):
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self.handlers = hs.get_handlers()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request, user_id):
-        target_user = UserID.from_string(user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        auth_user = requester.user
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin and target_user != auth_user:
-            raise AuthError(403, "You are not a server admin")
-
-        if not self.hs.is_mine(target_user):
-            raise SynapseError(400, "Can only whois a local user")
-
-        ret = yield self.handlers.admin_handler.get_whois(target_user)
-
-        defer.returnValue((200, ret))
-
-
-class PurgeMediaCacheRestServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/purge_media_cache")
-
-    def __init__(self, hs):
-        self.media_repository = hs.get_media_repository()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        before_ts = parse_integer(request, "before_ts", required=True)
-        logger.info("before_ts: %r", before_ts)
-
-        ret = yield self.media_repository.delete_old_remote_media(before_ts)
-
-        defer.returnValue((200, ret))
-
-
-class PurgeHistoryRestServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns(
-        "/purge_history/(?P<room_id>[^/]*)(/(?P<event_id>[^/]+))?"
-    )
-
-    def __init__(self, hs):
-        """
-
-        Args:
-            hs (synapse.server.HomeServer)
-        """
-        self.pagination_handler = hs.get_pagination_handler()
-        self.store = hs.get_datastore()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, room_id, event_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        body = parse_json_object_from_request(request, allow_empty_body=True)
-
-        delete_local_events = bool(body.get("delete_local_events", False))
-
-        # establish the topological ordering we should keep events from. The
-        # user can provide an event_id in the URL or the request body, or can
-        # provide a timestamp in the request body.
-        if event_id is None:
-            event_id = body.get('purge_up_to_event_id')
-
-        if event_id is not None:
-            event = yield self.store.get_event(event_id)
-
-            if event.room_id != room_id:
-                raise SynapseError(400, "Event is for wrong room.")
-
-            token = yield self.store.get_topological_token_for_event(event_id)
-
-            logger.info(
-                "[purge] purging up to token %s (event_id %s)",
-                token, event_id,
-            )
-        elif 'purge_up_to_ts' in body:
-            ts = body['purge_up_to_ts']
-            if not isinstance(ts, int):
-                raise SynapseError(
-                    400, "purge_up_to_ts must be an int",
-                    errcode=Codes.BAD_JSON,
-                )
-
-            stream_ordering = (
-                yield self.store.find_first_stream_ordering_after_ts(ts)
-            )
-
-            r = (
-                yield self.store.get_room_event_after_stream_ordering(
-                    room_id, stream_ordering,
-                )
-            )
-            if not r:
-                logger.warn(
-                    "[purge] purging events not possible: No event found "
-                    "(received_ts %i => stream_ordering %i)",
-                    ts, stream_ordering,
-                )
-                raise SynapseError(
-                    404,
-                    "there is no event to be purged",
-                    errcode=Codes.NOT_FOUND,
-                )
-            (stream, topo, _event_id) = r
-            token = "t%d-%d" % (topo, stream)
-            logger.info(
-                "[purge] purging up to token %s (received_ts %i => "
-                "stream_ordering %i)",
-                token, ts, stream_ordering,
-            )
-        else:
-            raise SynapseError(
-                400,
-                "must specify purge_up_to_event_id or purge_up_to_ts",
-                errcode=Codes.BAD_JSON,
-            )
-
-        purge_id = yield self.pagination_handler.start_purge_history(
-            room_id, token,
-            delete_local_events=delete_local_events,
-        )
-
-        defer.returnValue((200, {
-            "purge_id": purge_id,
-        }))
-
-
-class PurgeHistoryStatusRestServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns(
-        "/purge_history_status/(?P<purge_id>[^/]+)"
-    )
-
-    def __init__(self, hs):
-        """
-
-        Args:
-            hs (synapse.server.HomeServer)
-        """
-        self.pagination_handler = hs.get_pagination_handler()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request, purge_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        purge_status = self.pagination_handler.get_purge_status(purge_id)
-        if purge_status is None:
-            raise NotFoundError("purge id '%s' not found" % purge_id)
-
-        defer.returnValue((200, purge_status.asdict()))
-
-
-class DeactivateAccountRestServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/deactivate/(?P<target_user_id>[^/]*)")
-
-    def __init__(self, hs):
-        self._deactivate_account_handler = hs.get_deactivate_account_handler()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, target_user_id):
-        body = parse_json_object_from_request(request, allow_empty_body=True)
-        erase = body.get("erase", False)
-        if not isinstance(erase, bool):
-            raise SynapseError(
-                http_client.BAD_REQUEST,
-                "Param 'erase' must be a boolean, if given",
-                Codes.BAD_JSON,
-            )
-
-        UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        result = yield self._deactivate_account_handler.deactivate_account(
-            target_user_id, erase,
-        )
-        if result:
-            id_server_unbind_result = "success"
-        else:
-            id_server_unbind_result = "no-support"
-
-        defer.returnValue((200, {
-            "id_server_unbind_result": id_server_unbind_result,
-        }))
-
-
-class ShutdownRoomRestServlet(RestServlet):
-    """Shuts down a room by removing all local users from the room and blocking
-    all future invites and joins to the room. Any local aliases will be repointed
-    to a new room created by `new_room_user_id` and kicked users will be auto
-    joined to the new room.
-    """
-    PATTERNS = historical_admin_path_patterns("/shutdown_room/(?P<room_id>[^/]+)")
-
-    DEFAULT_MESSAGE = (
-        "Sharing illegal content on this server is not permitted and rooms in"
-        " violation will be blocked."
-    )
-
-    def __init__(self, hs):
-        self.hs = hs
-        self.store = hs.get_datastore()
-        self.state = hs.get_state_handler()
-        self._room_creation_handler = hs.get_room_creation_handler()
-        self.event_creation_handler = hs.get_event_creation_handler()
-        self.room_member_handler = hs.get_room_member_handler()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, room_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        content = parse_json_object_from_request(request)
-        assert_params_in_dict(content, ["new_room_user_id"])
-        new_room_user_id = content["new_room_user_id"]
-
-        room_creator_requester = create_requester(new_room_user_id)
-
-        message = content.get("message", self.DEFAULT_MESSAGE)
-        room_name = content.get("room_name", "Content Violation Notification")
-
-        info = yield self._room_creation_handler.create_room(
-            room_creator_requester,
-            config={
-                "preset": "public_chat",
-                "name": room_name,
-                "power_level_content_override": {
-                    "users_default": -10,
-                },
-            },
-            ratelimit=False,
-        )
-        new_room_id = info["room_id"]
-
-        requester_user_id = requester.user.to_string()
-
-        logger.info(
-            "Shutting down room %r, joining to new room: %r",
-            room_id, new_room_id,
-        )
-
-        # This will work even if the room is already blocked, but that is
-        # desirable in case the first attempt at blocking the room failed below.
-        yield self.store.block_room(room_id, requester_user_id)
-
-        users = yield self.state.get_current_users_in_room(room_id)
-        kicked_users = []
-        failed_to_kick_users = []
-        for user_id in users:
-            if not self.hs.is_mine_id(user_id):
-                continue
-
-            logger.info("Kicking %r from %r...", user_id, room_id)
-
-            try:
-                target_requester = create_requester(user_id)
-                yield self.room_member_handler.update_membership(
-                    requester=target_requester,
-                    target=target_requester.user,
-                    room_id=room_id,
-                    action=Membership.LEAVE,
-                    content={},
-                    ratelimit=False,
-                    require_consent=False,
-                )
-
-                yield self.room_member_handler.forget(target_requester.user, room_id)
-
-                yield self.room_member_handler.update_membership(
-                    requester=target_requester,
-                    target=target_requester.user,
-                    room_id=new_room_id,
-                    action=Membership.JOIN,
-                    content={},
-                    ratelimit=False,
-                    require_consent=False,
-                )
-
-                kicked_users.append(user_id)
-            except Exception:
-                logger.exception(
-                    "Failed to leave old room and join new room for %r", user_id,
-                )
-                failed_to_kick_users.append(user_id)
-
-        yield self.event_creation_handler.create_and_send_nonmember_event(
-            room_creator_requester,
-            {
-                "type": "m.room.message",
-                "content": {"body": message, "msgtype": "m.text"},
-                "room_id": new_room_id,
-                "sender": new_room_user_id,
-            },
-            ratelimit=False,
-        )
-
-        aliases_for_room = yield self.store.get_aliases_for_room(room_id)
-
-        yield self.store.update_aliases_for_room(
-            room_id, new_room_id, requester_user_id
-        )
-
-        defer.returnValue((200, {
-            "kicked_users": kicked_users,
-            "failed_to_kick_users": failed_to_kick_users,
-            "local_aliases": aliases_for_room,
-            "new_room_id": new_room_id,
-        }))
-
-
-class QuarantineMediaInRoom(RestServlet):
-    """Quarantines all media in a room so that no one can download it via
-    this server.
-    """
-    PATTERNS = historical_admin_path_patterns("/quarantine_media/(?P<room_id>[^/]+)")
-
-    def __init__(self, hs):
-        self.store = hs.get_datastore()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, room_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        num_quarantined = yield self.store.quarantine_media_ids_in_room(
-            room_id, requester.user.to_string(),
-        )
-
-        defer.returnValue((200, {"num_quarantined": num_quarantined}))
-
-
-class ListMediaInRoom(RestServlet):
-    """Lists all of the media in a given room.
-    """
-    PATTERNS = historical_admin_path_patterns("/room/(?P<room_id>[^/]+)/media")
-
-    def __init__(self, hs):
-        self.store = hs.get_datastore()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request, room_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        local_mxcs, remote_mxcs = yield self.store.get_media_mxcs_in_room(room_id)
-
-        defer.returnValue((200, {"local": local_mxcs, "remote": remote_mxcs}))
-
-
-class ResetPasswordRestServlet(RestServlet):
-    """Post request to allow an administrator reset password for a user.
-    This needs user to have administrator access in Synapse.
-        Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/reset_password/
-            @user:to_reset_password?access_token=admin_access_token
-        JsonBodyToSend:
-            {
-                "new_password": "secret"
-            }
-        Returns:
-            200 OK with empty object if success otherwise an error.
-        """
-    PATTERNS = historical_admin_path_patterns("/reset_password/(?P<target_user_id>[^/]*)")
-
-    def __init__(self, hs):
-        self.store = hs.get_datastore()
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self._set_password_handler = hs.get_set_password_handler()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, target_user_id):
-        """Post request to allow an administrator reset password for a user.
-        This needs user to have administrator access in Synapse.
-        """
-        UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        params = parse_json_object_from_request(request)
-        assert_params_in_dict(params, ["new_password"])
-        new_password = params['new_password']
-
-        yield self._set_password_handler.set_password(
-            target_user_id, new_password, requester
-        )
-        defer.returnValue((200, {}))
-
-
-class GetUsersPaginatedRestServlet(RestServlet):
-    """Get request to get specific number of users from Synapse.
-    This needs user to have administrator access in Synapse.
-        Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/
-            @admin:user?access_token=admin_access_token&start=0&limit=10
-        Returns:
-            200 OK with json object {list[dict[str, Any]], count} or empty object.
-        """
-    PATTERNS = historical_admin_path_patterns("/users_paginate/(?P<target_user_id>[^/]*)")
-
-    def __init__(self, hs):
-        self.store = hs.get_datastore()
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self.handlers = hs.get_handlers()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request, target_user_id):
-        """Get request to get specific number of users from Synapse.
-        This needs user to have administrator access in Synapse.
-        """
-        target_user = UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        # To allow all users to get the users list
-        # if not is_admin and target_user != auth_user:
-        #     raise AuthError(403, "You are not a server admin")
-
-        if not self.hs.is_mine(target_user):
-            raise SynapseError(400, "Can only users a local user")
-
-        order = "name"  # order by name in user table
-        start = parse_integer(request, "start", required=True)
-        limit = parse_integer(request, "limit", required=True)
-
-        logger.info("limit: %s, start: %s", limit, start)
-
-        ret = yield self.handlers.admin_handler.get_users_paginate(
-            order, start, limit
-        )
-        defer.returnValue((200, ret))
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, target_user_id):
-        """Post request to get specific number of users from Synapse..
-        This needs user to have administrator access in Synapse.
-        Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/
-            @admin:user?access_token=admin_access_token
-        JsonBodyToSend:
-            {
-                "start": "0",
-                "limit": "10
-            }
-        Returns:
-            200 OK with json object {list[dict[str, Any]], count} or empty object.
-        """
-        UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        order = "name"  # order by name in user table
-        params = parse_json_object_from_request(request)
-        assert_params_in_dict(params, ["limit", "start"])
-        limit = params['limit']
-        start = params['start']
-        logger.info("limit: %s, start: %s", limit, start)
-
-        ret = yield self.handlers.admin_handler.get_users_paginate(
-            order, start, limit
-        )
-        defer.returnValue((200, ret))
-
-
-class SearchUsersRestServlet(RestServlet):
-    """Get request to search user table for specific users according to
-    search term.
-    This needs user to have administrator access in Synapse.
-        Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/search_users/
-            @admin:user?access_token=admin_access_token&term=alice
-        Returns:
-            200 OK with json object {list[dict[str, Any]], count} or empty object.
-    """
-    PATTERNS = historical_admin_path_patterns("/search_users/(?P<target_user_id>[^/]*)")
-
-    def __init__(self, hs):
-        self.store = hs.get_datastore()
-        self.hs = hs
-        self.auth = hs.get_auth()
-        self.handlers = hs.get_handlers()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request, target_user_id):
-        """Get request to search user table for specific users according to
-        search term.
-        This needs user to have a administrator access in Synapse.
-        """
-        target_user = UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        # To allow all users to get the users list
-        # if not is_admin and target_user != auth_user:
-        #     raise AuthError(403, "You are not a server admin")
-
-        if not self.hs.is_mine(target_user):
-            raise SynapseError(400, "Can only users a local user")
-
-        term = parse_string(request, "term", required=True)
-        logger.info("term: %s ", term)
-
-        ret = yield self.handlers.admin_handler.search_users(
-            term
-        )
-        defer.returnValue((200, ret))
-
-
-class DeleteGroupAdminRestServlet(RestServlet):
-    """Allows deleting of local groups
-    """
-    PATTERNS = historical_admin_path_patterns("/delete_group/(?P<group_id>[^/]*)")
-
-    def __init__(self, hs):
-        self.group_server = hs.get_groups_server_handler()
-        self.is_mine_id = hs.is_mine_id
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request, group_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        if not self.is_mine_id(group_id):
-            raise SynapseError(400, "Can only delete local groups")
-
-        yield self.group_server.delete_group(group_id, requester.user.to_string())
-        defer.returnValue((200, {}))
-
-
-class AccountValidityRenewServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/account_validity/validity$")
-
-    def __init__(self, hs):
-        """
-        Args:
-            hs (synapse.server.HomeServer): server
-        """
-        self.hs = hs
-        self.account_activity_handler = hs.get_account_validity_handler()
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_POST(self, request):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        body = parse_json_object_from_request(request)
-
-        if "user_id" not in body:
-            raise SynapseError(400, "Missing property 'user_id' in the request body")
-
-        expiration_ts = yield self.account_activity_handler.renew_account_for_user(
-            body["user_id"], body.get("expiration_ts"),
-            not body.get("enable_renewal_emails", True),
-        )
-
-        res = {
-            "expiration_ts": expiration_ts,
-        }
-        defer.returnValue((200, res))
-
-
-class AdminRestResource(JsonResource):
-    """The REST resource which gets mounted at /_synapse/admin"""
-
-    def __init__(self, hs):
-        JsonResource.__init__(self, hs, canonical_json=False)
-        register_servlets(hs, self)
-
-
-def register_servlets(hs, http_server):
-    WhoisRestServlet(hs).register(http_server)
-    PurgeMediaCacheRestServlet(hs).register(http_server)
-    PurgeHistoryStatusRestServlet(hs).register(http_server)
-    DeactivateAccountRestServlet(hs).register(http_server)
-    PurgeHistoryRestServlet(hs).register(http_server)
-    UsersRestServlet(hs).register(http_server)
-    ResetPasswordRestServlet(hs).register(http_server)
-    GetUsersPaginatedRestServlet(hs).register(http_server)
-    SearchUsersRestServlet(hs).register(http_server)
-    ShutdownRoomRestServlet(hs).register(http_server)
-    QuarantineMediaInRoom(hs).register(http_server)
-    ListMediaInRoom(hs).register(http_server)
-    UserRegisterServlet(hs).register(http_server)
-    VersionServlet(hs).register(http_server)
-    DeleteGroupAdminRestServlet(hs).register(http_server)
-    AccountValidityRenewServlet(hs).register(http_server)
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index f1d0aa42b6..32ef83e9c0 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -14,8 +14,9 @@
 # limitations under the License.
 from mock import Mock
 
+import synapse.rest.admin
 from synapse.api.constants import UserTypes
-from synapse.rest.client.v1 import admin, login, room
+from synapse.rest.client.v1 import login, room
 from synapse.rest.client.v2_alpha import user_directory
 from synapse.storage.roommember import ProfileInfo
 
@@ -29,7 +30,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
 
     servlets = [
         login.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
     ]
 
@@ -327,7 +328,7 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase):
         user_directory.register_servlets,
         room.register_servlets,
         login.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
     ]
 
     def make_homeserver(self, reactor, clock):
diff --git a/tests/push/test_email.py b/tests/push/test_email.py
index be3fed8de3..e29bd18ad7 100644
--- a/tests/push/test_email.py
+++ b/tests/push/test_email.py
@@ -19,7 +19,8 @@ import pkg_resources
 
 from twisted.internet.defer import Deferred
 
-from synapse.rest.client.v1 import admin, login, room
+import synapse.rest.admin
+from synapse.rest.client.v1 import login, room
 
 from tests.unittest import HomeserverTestCase
 
@@ -33,7 +34,7 @@ class EmailPusherTests(HomeserverTestCase):
 
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/push/test_http.py b/tests/push/test_http.py
index 6dc45e8506..3f9f56bb79 100644
--- a/tests/push/test_http.py
+++ b/tests/push/test_http.py
@@ -17,7 +17,8 @@ from mock import Mock
 
 from twisted.internet.defer import Deferred
 
-from synapse.rest.client.v1 import admin, login, room
+import synapse.rest.admin
+from synapse.rest.client.v1 import login, room
 from synapse.util.logcontext import make_deferred_yieldable
 
 from tests.unittest import HomeserverTestCase
@@ -32,7 +33,7 @@ class HTTPPusherTests(HomeserverTestCase):
 
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/admin/__init__.py b/tests/rest/admin/__init__.py
new file mode 100644
index 0000000000..1453d04571
--- /dev/null
+++ b/tests/rest/admin/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py
new file mode 100644
index 0000000000..42858b5fea
--- /dev/null
+++ b/tests/rest/admin/test_admin.py
@@ -0,0 +1,617 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import hmac
+import json
+
+from mock import Mock
+
+import synapse.rest.admin
+from synapse.api.constants import UserTypes
+from synapse.rest.client.v1 import events, login, room
+from synapse.rest.client.v2_alpha import groups
+
+from tests import unittest
+
+
+class VersionTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+    ]
+
+    url = '/_matrix/client/r0/admin/server_version'
+
+    def test_version_string(self):
+        self.register_user("admin", "pass", admin=True)
+        self.admin_token = self.login("admin", "pass")
+
+        request, channel = self.make_request("GET", self.url,
+                                             access_token=self.admin_token)
+        self.render(request)
+
+        self.assertEqual(200, int(channel.result["code"]),
+                         msg=channel.result["body"])
+        self.assertEqual({'server_version', 'python_version'},
+                         set(channel.json_body.keys()))
+
+    def test_inaccessible_to_non_admins(self):
+        self.register_user("unprivileged-user", "pass", admin=False)
+        user_token = self.login("unprivileged-user", "pass")
+
+        request, channel = self.make_request("GET", self.url,
+                                             access_token=user_token)
+        self.render(request)
+
+        self.assertEqual(403, int(channel.result['code']),
+                         msg=channel.result['body'])
+
+
+class UserRegisterTestCase(unittest.HomeserverTestCase):
+
+    servlets = [synapse.rest.admin.register_servlets]
+
+    def make_homeserver(self, reactor, clock):
+
+        self.url = "/_matrix/client/r0/admin/register"
+
+        self.registration_handler = Mock()
+        self.identity_handler = Mock()
+        self.login_handler = Mock()
+        self.device_handler = Mock()
+        self.device_handler.check_device_registered = Mock(return_value="FAKE")
+
+        self.datastore = Mock(return_value=Mock())
+        self.datastore.get_current_state_deltas = Mock(return_value=[])
+
+        self.secrets = Mock()
+
+        self.hs = self.setup_test_homeserver()
+
+        self.hs.config.registration_shared_secret = u"shared"
+
+        self.hs.get_media_repository = Mock()
+        self.hs.get_deactivate_account_handler = Mock()
+
+        return self.hs
+
+    def test_disabled(self):
+        """
+        If there is no shared secret, registration through this method will be
+        prevented.
+        """
+        self.hs.config.registration_shared_secret = None
+
+        request, channel = self.make_request("POST", self.url, b'{}')
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual(
+            'Shared secret registration is not enabled', channel.json_body["error"]
+        )
+
+    def test_get_nonce(self):
+        """
+        Calling GET on the endpoint will return a randomised nonce, using the
+        homeserver's secrets provider.
+        """
+        secrets = Mock()
+        secrets.token_hex = Mock(return_value="abcd")
+
+        self.hs.get_secrets = Mock(return_value=secrets)
+
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+
+        self.assertEqual(channel.json_body, {"nonce": "abcd"})
+
+    def test_expired_nonce(self):
+        """
+        Calling GET on the endpoint will return a randomised nonce, which will
+        only last for SALT_TIMEOUT (60s).
+        """
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+        nonce = channel.json_body["nonce"]
+
+        # 59 seconds
+        self.reactor.advance(59)
+
+        body = json.dumps({"nonce": nonce})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('username must be specified', channel.json_body["error"])
+
+        # 61 seconds
+        self.reactor.advance(2)
+
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('unrecognised nonce', channel.json_body["error"])
+
+    def test_register_incorrect_nonce(self):
+        """
+        Only the provided nonce can be used, as it's checked in the MAC.
+        """
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(b"notthenonce\x00bob\x00abc123\x00admin")
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "mac": want_mac,
+            }
+        )
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual("HMAC incorrect", channel.json_body["error"])
+
+    def test_register_correct_nonce(self):
+        """
+        When the correct nonce is provided, and the right key is provided, the
+        user is registered.
+        """
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(
+            nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin\x00support"
+        )
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "user_type": UserTypes.SUPPORT,
+                "mac": want_mac,
+            }
+        )
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual("@bob:test", channel.json_body["user_id"])
+
+    def test_nonce_reuse(self):
+        """
+        A valid unrecognised nonce.
+        """
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+        nonce = channel.json_body["nonce"]
+
+        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
+        want_mac.update(
+            nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin"
+        )
+        want_mac = want_mac.hexdigest()
+
+        body = json.dumps(
+            {
+                "nonce": nonce,
+                "username": "bob",
+                "password": "abc123",
+                "admin": True,
+                "mac": want_mac,
+            }
+        )
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual("@bob:test", channel.json_body["user_id"])
+
+        # Now, try and reuse it
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('unrecognised nonce', channel.json_body["error"])
+
+    def test_missing_parts(self):
+        """
+        Synapse will complain if you don't give nonce, username, password, and
+        mac.  Admin and user_types are optional.  Additional checks are done for length
+        and type.
+        """
+
+        def nonce():
+            request, channel = self.make_request("GET", self.url)
+            self.render(request)
+            return channel.json_body["nonce"]
+
+        #
+        # Nonce check
+        #
+
+        # Must be present
+        body = json.dumps({})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('nonce must be specified', channel.json_body["error"])
+
+        #
+        # Username checks
+        #
+
+        # Must be present
+        body = json.dumps({"nonce": nonce()})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('username must be specified', channel.json_body["error"])
+
+        # Must be a string
+        body = json.dumps({"nonce": nonce(), "username": 1234})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid username', channel.json_body["error"])
+
+        # Must not have null bytes
+        body = json.dumps({"nonce": nonce(), "username": u"abcd\u0000"})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid username', channel.json_body["error"])
+
+        # Must not have null bytes
+        body = json.dumps({"nonce": nonce(), "username": "a" * 1000})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid username', channel.json_body["error"])
+
+        #
+        # Password checks
+        #
+
+        # Must be present
+        body = json.dumps({"nonce": nonce(), "username": "a"})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('password must be specified', channel.json_body["error"])
+
+        # Must be a string
+        body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid password', channel.json_body["error"])
+
+        # Must not have null bytes
+        body = json.dumps(
+            {"nonce": nonce(), "username": "a", "password": u"abcd\u0000"}
+        )
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid password', channel.json_body["error"])
+
+        # Super long
+        body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000})
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid password', channel.json_body["error"])
+
+        #
+        # user_type check
+        #
+
+        # Invalid user_type
+        body = json.dumps({
+            "nonce": nonce(),
+            "username": "a",
+            "password": "1234",
+            "user_type": "invalid"}
+        )
+        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
+        self.render(request)
+
+        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
+        self.assertEqual('Invalid user type', channel.json_body["error"])
+
+
+class ShutdownRoomTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        events.register_servlets,
+        room.register_servlets,
+        room.register_deprecated_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.event_creation_handler = hs.get_event_creation_handler()
+        hs.config.user_consent_version = "1"
+
+        consent_uri_builder = Mock()
+        consent_uri_builder.build_user_consent_uri.return_value = (
+            "http://example.com"
+        )
+        self.event_creation_handler._consent_uri_builder = consent_uri_builder
+
+        self.store = hs.get_datastore()
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_token = self.login("user", "pass")
+
+        # Mark the admin user as having consented
+        self.get_success(
+            self.store.user_set_consent_version(self.admin_user, "1"),
+        )
+
+    def test_shutdown_room_consent(self):
+        """Test that we can shutdown rooms with local users who have not
+        yet accepted the privacy policy. This used to fail when we tried to
+        force part the user from the old room.
+        """
+        self.event_creation_handler._block_events_without_consent_error = None
+
+        room_id = self.helper.create_room_as(self.other_user, tok=self.other_user_token)
+
+        # Assert one user in room
+        users_in_room = self.get_success(
+            self.store.get_users_in_room(room_id),
+        )
+        self.assertEqual([self.other_user], users_in_room)
+
+        # Enable require consent to send events
+        self.event_creation_handler._block_events_without_consent_error = "Error"
+
+        # Assert that the user is getting consent error
+        self.helper.send(
+            room_id,
+            body="foo", tok=self.other_user_token, expect_code=403,
+        )
+
+        # Test that the admin can still send shutdown
+        url = "admin/shutdown_room/" + room_id
+        request, channel = self.make_request(
+            "POST",
+            url.encode('ascii'),
+            json.dumps({"new_room_user_id": self.admin_user}),
+            access_token=self.admin_user_tok,
+        )
+        self.render(request)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+
+        # Assert there is now no longer anyone in the room
+        users_in_room = self.get_success(
+            self.store.get_users_in_room(room_id),
+        )
+        self.assertEqual([], users_in_room)
+
+    @unittest.DEBUG
+    def test_shutdown_room_block_peek(self):
+        """Test that a world_readable room can no longer be peeked into after
+        it has been shut down.
+        """
+
+        self.event_creation_handler._block_events_without_consent_error = None
+
+        room_id = self.helper.create_room_as(self.other_user, tok=self.other_user_token)
+
+        # Enable world readable
+        url = "rooms/%s/state/m.room.history_visibility" % (room_id,)
+        request, channel = self.make_request(
+            "PUT",
+            url.encode('ascii'),
+            json.dumps({"history_visibility": "world_readable"}),
+            access_token=self.other_user_token,
+        )
+        self.render(request)
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+
+        # Test that the admin can still send shutdown
+        url = "admin/shutdown_room/" + room_id
+        request, channel = self.make_request(
+            "POST",
+            url.encode('ascii'),
+            json.dumps({"new_room_user_id": self.admin_user}),
+            access_token=self.admin_user_tok,
+        )
+        self.render(request)
+
+        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
+
+        # Assert we can no longer peek into the room
+        self._assert_peek(room_id, expect_code=403)
+
+    def _assert_peek(self, room_id, expect_code):
+        """Assert that the admin user can (or cannot) peek into the room.
+        """
+
+        url = "rooms/%s/initialSync" % (room_id,)
+        request, channel = self.make_request(
+            "GET",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+        )
+        self.render(request)
+        self.assertEqual(
+            expect_code, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        url = "events?timeout=0&room_id=" + room_id
+        request, channel = self.make_request(
+            "GET",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+        )
+        self.render(request)
+        self.assertEqual(
+            expect_code, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+
+class DeleteGroupTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        groups.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+        self.store = hs.get_datastore()
+
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_token = self.login("user", "pass")
+
+    def test_delete_group(self):
+        # Create a new group
+        request, channel = self.make_request(
+            "POST",
+            "/create_group".encode('ascii'),
+            access_token=self.admin_user_tok,
+            content={
+                "localpart": "test",
+            }
+        )
+
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        group_id = channel.json_body["group_id"]
+
+        self._check_group(group_id, expect_code=200)
+
+        # Invite/join another user
+
+        url = "/groups/%s/admin/users/invite/%s" % (group_id, self.other_user)
+        request, channel = self.make_request(
+            "PUT",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+            content={}
+        )
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        url = "/groups/%s/self/accept_invite" % (group_id,)
+        request, channel = self.make_request(
+            "PUT",
+            url.encode('ascii'),
+            access_token=self.other_user_token,
+            content={}
+        )
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        # Check other user knows they're in the group
+        self.assertIn(group_id, self._get_groups_user_is_in(self.admin_user_tok))
+        self.assertIn(group_id, self._get_groups_user_is_in(self.other_user_token))
+
+        # Now delete the group
+        url = "/admin/delete_group/" + group_id
+        request, channel = self.make_request(
+            "POST",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+            content={
+                "localpart": "test",
+            }
+        )
+
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        # Check group returns 404
+        self._check_group(group_id, expect_code=404)
+
+        # Check users don't think they're in the group
+        self.assertNotIn(group_id, self._get_groups_user_is_in(self.admin_user_tok))
+        self.assertNotIn(group_id, self._get_groups_user_is_in(self.other_user_token))
+
+    def _check_group(self, group_id, expect_code):
+        """Assert that trying to fetch the given group results in the given
+        HTTP status code
+        """
+
+        url = "/groups/%s/profile" % (group_id,)
+        request, channel = self.make_request(
+            "GET",
+            url.encode('ascii'),
+            access_token=self.admin_user_tok,
+        )
+
+        self.render(request)
+        self.assertEqual(
+            expect_code, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+    def _get_groups_user_is_in(self, access_token):
+        """Returns the list of groups the user is in (given their access token)
+        """
+        request, channel = self.make_request(
+            "GET",
+            "/joined_groups".encode('ascii'),
+            access_token=access_token,
+        )
+
+        self.render(request)
+        self.assertEqual(
+            200, int(channel.result["code"]), msg=channel.result["body"],
+        )
+
+        return channel.json_body["groups"]
diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py
index 4294bbec2a..36e6c1c67d 100644
--- a/tests/rest/client/test_consent.py
+++ b/tests/rest/client/test_consent.py
@@ -15,8 +15,9 @@
 
 import os
 
+import synapse.rest.admin
 from synapse.api.urls import ConsentURIBuilder
-from synapse.rest.client.v1 import admin, login, room
+from synapse.rest.client.v1 import login, room
 from synapse.rest.consent import consent_resource
 
 from tests import unittest
@@ -31,7 +32,7 @@ except Exception:
 class ConsentResourceTestCase(unittest.HomeserverTestCase):
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py
index ca63b2e6ed..d4fe0aee7d 100644
--- a/tests/rest/client/test_identity.py
+++ b/tests/rest/client/test_identity.py
@@ -15,7 +15,8 @@
 
 import json
 
-from synapse.rest.client.v1 import admin, login, room
+import synapse.rest.admin
+from synapse.rest.client.v1 import login, room
 
 from tests import unittest
 
@@ -23,7 +24,7 @@ from tests import unittest
 class IdentityTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
deleted file mode 100644
index c00ef21d75..0000000000
--- a/tests/rest/client/v1/test_admin.py
+++ /dev/null
@@ -1,616 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import hashlib
-import hmac
-import json
-
-from mock import Mock
-
-from synapse.api.constants import UserTypes
-from synapse.rest.client.v1 import admin, events, login, room
-from synapse.rest.client.v2_alpha import groups
-
-from tests import unittest
-
-
-class VersionTestCase(unittest.HomeserverTestCase):
-
-    servlets = [
-        admin.register_servlets,
-        login.register_servlets,
-    ]
-
-    url = '/_matrix/client/r0/admin/server_version'
-
-    def test_version_string(self):
-        self.register_user("admin", "pass", admin=True)
-        self.admin_token = self.login("admin", "pass")
-
-        request, channel = self.make_request("GET", self.url,
-                                             access_token=self.admin_token)
-        self.render(request)
-
-        self.assertEqual(200, int(channel.result["code"]),
-                         msg=channel.result["body"])
-        self.assertEqual({'server_version', 'python_version'},
-                         set(channel.json_body.keys()))
-
-    def test_inaccessible_to_non_admins(self):
-        self.register_user("unprivileged-user", "pass", admin=False)
-        user_token = self.login("unprivileged-user", "pass")
-
-        request, channel = self.make_request("GET", self.url,
-                                             access_token=user_token)
-        self.render(request)
-
-        self.assertEqual(403, int(channel.result['code']),
-                         msg=channel.result['body'])
-
-
-class UserRegisterTestCase(unittest.HomeserverTestCase):
-
-    servlets = [admin.register_servlets]
-
-    def make_homeserver(self, reactor, clock):
-
-        self.url = "/_matrix/client/r0/admin/register"
-
-        self.registration_handler = Mock()
-        self.identity_handler = Mock()
-        self.login_handler = Mock()
-        self.device_handler = Mock()
-        self.device_handler.check_device_registered = Mock(return_value="FAKE")
-
-        self.datastore = Mock(return_value=Mock())
-        self.datastore.get_current_state_deltas = Mock(return_value=[])
-
-        self.secrets = Mock()
-
-        self.hs = self.setup_test_homeserver()
-
-        self.hs.config.registration_shared_secret = u"shared"
-
-        self.hs.get_media_repository = Mock()
-        self.hs.get_deactivate_account_handler = Mock()
-
-        return self.hs
-
-    def test_disabled(self):
-        """
-        If there is no shared secret, registration through this method will be
-        prevented.
-        """
-        self.hs.config.registration_shared_secret = None
-
-        request, channel = self.make_request("POST", self.url, b'{}')
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual(
-            'Shared secret registration is not enabled', channel.json_body["error"]
-        )
-
-    def test_get_nonce(self):
-        """
-        Calling GET on the endpoint will return a randomised nonce, using the
-        homeserver's secrets provider.
-        """
-        secrets = Mock()
-        secrets.token_hex = Mock(return_value="abcd")
-
-        self.hs.get_secrets = Mock(return_value=secrets)
-
-        request, channel = self.make_request("GET", self.url)
-        self.render(request)
-
-        self.assertEqual(channel.json_body, {"nonce": "abcd"})
-
-    def test_expired_nonce(self):
-        """
-        Calling GET on the endpoint will return a randomised nonce, which will
-        only last for SALT_TIMEOUT (60s).
-        """
-        request, channel = self.make_request("GET", self.url)
-        self.render(request)
-        nonce = channel.json_body["nonce"]
-
-        # 59 seconds
-        self.reactor.advance(59)
-
-        body = json.dumps({"nonce": nonce})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('username must be specified', channel.json_body["error"])
-
-        # 61 seconds
-        self.reactor.advance(2)
-
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('unrecognised nonce', channel.json_body["error"])
-
-    def test_register_incorrect_nonce(self):
-        """
-        Only the provided nonce can be used, as it's checked in the MAC.
-        """
-        request, channel = self.make_request("GET", self.url)
-        self.render(request)
-        nonce = channel.json_body["nonce"]
-
-        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
-        want_mac.update(b"notthenonce\x00bob\x00abc123\x00admin")
-        want_mac = want_mac.hexdigest()
-
-        body = json.dumps(
-            {
-                "nonce": nonce,
-                "username": "bob",
-                "password": "abc123",
-                "admin": True,
-                "mac": want_mac,
-            }
-        )
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual("HMAC incorrect", channel.json_body["error"])
-
-    def test_register_correct_nonce(self):
-        """
-        When the correct nonce is provided, and the right key is provided, the
-        user is registered.
-        """
-        request, channel = self.make_request("GET", self.url)
-        self.render(request)
-        nonce = channel.json_body["nonce"]
-
-        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
-        want_mac.update(
-            nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin\x00support"
-        )
-        want_mac = want_mac.hexdigest()
-
-        body = json.dumps(
-            {
-                "nonce": nonce,
-                "username": "bob",
-                "password": "abc123",
-                "admin": True,
-                "user_type": UserTypes.SUPPORT,
-                "mac": want_mac,
-            }
-        )
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual("@bob:test", channel.json_body["user_id"])
-
-    def test_nonce_reuse(self):
-        """
-        A valid unrecognised nonce.
-        """
-        request, channel = self.make_request("GET", self.url)
-        self.render(request)
-        nonce = channel.json_body["nonce"]
-
-        want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1)
-        want_mac.update(
-            nonce.encode('ascii') + b"\x00bob\x00abc123\x00admin"
-        )
-        want_mac = want_mac.hexdigest()
-
-        body = json.dumps(
-            {
-                "nonce": nonce,
-                "username": "bob",
-                "password": "abc123",
-                "admin": True,
-                "mac": want_mac,
-            }
-        )
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual("@bob:test", channel.json_body["user_id"])
-
-        # Now, try and reuse it
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('unrecognised nonce', channel.json_body["error"])
-
-    def test_missing_parts(self):
-        """
-        Synapse will complain if you don't give nonce, username, password, and
-        mac.  Admin and user_types are optional.  Additional checks are done for length
-        and type.
-        """
-
-        def nonce():
-            request, channel = self.make_request("GET", self.url)
-            self.render(request)
-            return channel.json_body["nonce"]
-
-        #
-        # Nonce check
-        #
-
-        # Must be present
-        body = json.dumps({})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('nonce must be specified', channel.json_body["error"])
-
-        #
-        # Username checks
-        #
-
-        # Must be present
-        body = json.dumps({"nonce": nonce()})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('username must be specified', channel.json_body["error"])
-
-        # Must be a string
-        body = json.dumps({"nonce": nonce(), "username": 1234})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid username', channel.json_body["error"])
-
-        # Must not have null bytes
-        body = json.dumps({"nonce": nonce(), "username": u"abcd\u0000"})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid username', channel.json_body["error"])
-
-        # Must not have null bytes
-        body = json.dumps({"nonce": nonce(), "username": "a" * 1000})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid username', channel.json_body["error"])
-
-        #
-        # Password checks
-        #
-
-        # Must be present
-        body = json.dumps({"nonce": nonce(), "username": "a"})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('password must be specified', channel.json_body["error"])
-
-        # Must be a string
-        body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid password', channel.json_body["error"])
-
-        # Must not have null bytes
-        body = json.dumps(
-            {"nonce": nonce(), "username": "a", "password": u"abcd\u0000"}
-        )
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid password', channel.json_body["error"])
-
-        # Super long
-        body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000})
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid password', channel.json_body["error"])
-
-        #
-        # user_type check
-        #
-
-        # Invalid user_type
-        body = json.dumps({
-            "nonce": nonce(),
-            "username": "a",
-            "password": "1234",
-            "user_type": "invalid"}
-        )
-        request, channel = self.make_request("POST", self.url, body.encode('utf8'))
-        self.render(request)
-
-        self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
-        self.assertEqual('Invalid user type', channel.json_body["error"])
-
-
-class ShutdownRoomTestCase(unittest.HomeserverTestCase):
-    servlets = [
-        admin.register_servlets,
-        login.register_servlets,
-        events.register_servlets,
-        room.register_servlets,
-        room.register_deprecated_servlets,
-    ]
-
-    def prepare(self, reactor, clock, hs):
-        self.event_creation_handler = hs.get_event_creation_handler()
-        hs.config.user_consent_version = "1"
-
-        consent_uri_builder = Mock()
-        consent_uri_builder.build_user_consent_uri.return_value = (
-            "http://example.com"
-        )
-        self.event_creation_handler._consent_uri_builder = consent_uri_builder
-
-        self.store = hs.get_datastore()
-
-        self.admin_user = self.register_user("admin", "pass", admin=True)
-        self.admin_user_tok = self.login("admin", "pass")
-
-        self.other_user = self.register_user("user", "pass")
-        self.other_user_token = self.login("user", "pass")
-
-        # Mark the admin user as having consented
-        self.get_success(
-            self.store.user_set_consent_version(self.admin_user, "1"),
-        )
-
-    def test_shutdown_room_consent(self):
-        """Test that we can shutdown rooms with local users who have not
-        yet accepted the privacy policy. This used to fail when we tried to
-        force part the user from the old room.
-        """
-        self.event_creation_handler._block_events_without_consent_error = None
-
-        room_id = self.helper.create_room_as(self.other_user, tok=self.other_user_token)
-
-        # Assert one user in room
-        users_in_room = self.get_success(
-            self.store.get_users_in_room(room_id),
-        )
-        self.assertEqual([self.other_user], users_in_room)
-
-        # Enable require consent to send events
-        self.event_creation_handler._block_events_without_consent_error = "Error"
-
-        # Assert that the user is getting consent error
-        self.helper.send(
-            room_id,
-            body="foo", tok=self.other_user_token, expect_code=403,
-        )
-
-        # Test that the admin can still send shutdown
-        url = "admin/shutdown_room/" + room_id
-        request, channel = self.make_request(
-            "POST",
-            url.encode('ascii'),
-            json.dumps({"new_room_user_id": self.admin_user}),
-            access_token=self.admin_user_tok,
-        )
-        self.render(request)
-
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
-
-        # Assert there is now no longer anyone in the room
-        users_in_room = self.get_success(
-            self.store.get_users_in_room(room_id),
-        )
-        self.assertEqual([], users_in_room)
-
-    @unittest.DEBUG
-    def test_shutdown_room_block_peek(self):
-        """Test that a world_readable room can no longer be peeked into after
-        it has been shut down.
-        """
-
-        self.event_creation_handler._block_events_without_consent_error = None
-
-        room_id = self.helper.create_room_as(self.other_user, tok=self.other_user_token)
-
-        # Enable world readable
-        url = "rooms/%s/state/m.room.history_visibility" % (room_id,)
-        request, channel = self.make_request(
-            "PUT",
-            url.encode('ascii'),
-            json.dumps({"history_visibility": "world_readable"}),
-            access_token=self.other_user_token,
-        )
-        self.render(request)
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
-
-        # Test that the admin can still send shutdown
-        url = "admin/shutdown_room/" + room_id
-        request, channel = self.make_request(
-            "POST",
-            url.encode('ascii'),
-            json.dumps({"new_room_user_id": self.admin_user}),
-            access_token=self.admin_user_tok,
-        )
-        self.render(request)
-
-        self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
-
-        # Assert we can no longer peek into the room
-        self._assert_peek(room_id, expect_code=403)
-
-    def _assert_peek(self, room_id, expect_code):
-        """Assert that the admin user can (or cannot) peek into the room.
-        """
-
-        url = "rooms/%s/initialSync" % (room_id,)
-        request, channel = self.make_request(
-            "GET",
-            url.encode('ascii'),
-            access_token=self.admin_user_tok,
-        )
-        self.render(request)
-        self.assertEqual(
-            expect_code, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-        url = "events?timeout=0&room_id=" + room_id
-        request, channel = self.make_request(
-            "GET",
-            url.encode('ascii'),
-            access_token=self.admin_user_tok,
-        )
-        self.render(request)
-        self.assertEqual(
-            expect_code, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-
-class DeleteGroupTestCase(unittest.HomeserverTestCase):
-    servlets = [
-        admin.register_servlets,
-        login.register_servlets,
-        groups.register_servlets,
-    ]
-
-    def prepare(self, reactor, clock, hs):
-        self.store = hs.get_datastore()
-
-        self.admin_user = self.register_user("admin", "pass", admin=True)
-        self.admin_user_tok = self.login("admin", "pass")
-
-        self.other_user = self.register_user("user", "pass")
-        self.other_user_token = self.login("user", "pass")
-
-    def test_delete_group(self):
-        # Create a new group
-        request, channel = self.make_request(
-            "POST",
-            "/create_group".encode('ascii'),
-            access_token=self.admin_user_tok,
-            content={
-                "localpart": "test",
-            }
-        )
-
-        self.render(request)
-        self.assertEqual(
-            200, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-        group_id = channel.json_body["group_id"]
-
-        self._check_group(group_id, expect_code=200)
-
-        # Invite/join another user
-
-        url = "/groups/%s/admin/users/invite/%s" % (group_id, self.other_user)
-        request, channel = self.make_request(
-            "PUT",
-            url.encode('ascii'),
-            access_token=self.admin_user_tok,
-            content={}
-        )
-        self.render(request)
-        self.assertEqual(
-            200, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-        url = "/groups/%s/self/accept_invite" % (group_id,)
-        request, channel = self.make_request(
-            "PUT",
-            url.encode('ascii'),
-            access_token=self.other_user_token,
-            content={}
-        )
-        self.render(request)
-        self.assertEqual(
-            200, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-        # Check other user knows they're in the group
-        self.assertIn(group_id, self._get_groups_user_is_in(self.admin_user_tok))
-        self.assertIn(group_id, self._get_groups_user_is_in(self.other_user_token))
-
-        # Now delete the group
-        url = "/admin/delete_group/" + group_id
-        request, channel = self.make_request(
-            "POST",
-            url.encode('ascii'),
-            access_token=self.admin_user_tok,
-            content={
-                "localpart": "test",
-            }
-        )
-
-        self.render(request)
-        self.assertEqual(
-            200, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-        # Check group returns 404
-        self._check_group(group_id, expect_code=404)
-
-        # Check users don't think they're in the group
-        self.assertNotIn(group_id, self._get_groups_user_is_in(self.admin_user_tok))
-        self.assertNotIn(group_id, self._get_groups_user_is_in(self.other_user_token))
-
-    def _check_group(self, group_id, expect_code):
-        """Assert that trying to fetch the given group results in the given
-        HTTP status code
-        """
-
-        url = "/groups/%s/profile" % (group_id,)
-        request, channel = self.make_request(
-            "GET",
-            url.encode('ascii'),
-            access_token=self.admin_user_tok,
-        )
-
-        self.render(request)
-        self.assertEqual(
-            expect_code, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-    def _get_groups_user_is_in(self, access_token):
-        """Returns the list of groups the user is in (given their access token)
-        """
-        request, channel = self.make_request(
-            "GET",
-            "/joined_groups".encode('ascii'),
-            access_token=access_token,
-        )
-
-        self.render(request)
-        self.assertEqual(
-            200, int(channel.result["code"]), msg=channel.result["body"],
-        )
-
-        return channel.json_body["groups"]
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 36d8547275..5cb1c1ae9f 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -17,7 +17,8 @@
 
 from mock import Mock, NonCallableMock
 
-from synapse.rest.client.v1 import admin, events, login, room
+import synapse.rest.admin
+from synapse.rest.client.v1 import events, login, room
 
 from tests import unittest
 
@@ -28,7 +29,7 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
     servlets = [
         events.register_servlets,
         room.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         login.register_servlets,
     ]
 
diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py
index 86312f1096..8d9ef877f6 100644
--- a/tests/rest/client/v1/test_login.py
+++ b/tests/rest/client/v1/test_login.py
@@ -1,6 +1,7 @@
 import json
 
-from synapse.rest.client.v1 import admin, login
+import synapse.rest.admin
+from synapse.rest.client.v1 import login
 
 from tests import unittest
 
@@ -10,7 +11,7 @@ LOGIN_URL = b"/_matrix/client/r0/login"
 class LoginRestServletTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         login.register_servlets,
     ]
 
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 015c144248..1a34924f3e 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -22,8 +22,9 @@ from six.moves.urllib import parse as urlparse
 
 from twisted.internet import defer
 
+import synapse.rest.admin
 from synapse.api.constants import Membership
-from synapse.rest.client.v1 import admin, login, room
+from synapse.rest.client.v1 import login, room
 
 from tests import unittest
 
@@ -803,7 +804,7 @@ class RoomMessageListTestCase(RoomBase):
 
 class RoomSearchTestCase(unittest.HomeserverTestCase):
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py
index 7fa120a10f..67021185d0 100644
--- a/tests/rest/client/v2_alpha/test_auth.py
+++ b/tests/rest/client/v2_alpha/test_auth.py
@@ -16,8 +16,8 @@
 
 from twisted.internet.defer import succeed
 
+import synapse.rest.admin
 from synapse.api.constants import LoginType
-from synapse.rest.client.v1 import admin
 from synapse.rest.client.v2_alpha import auth, register
 
 from tests import unittest
@@ -27,7 +27,7 @@ class FallbackAuthTests(unittest.HomeserverTestCase):
 
     servlets = [
         auth.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         register.register_servlets,
     ]
     hijack_auth = False
diff --git a/tests/rest/client/v2_alpha/test_capabilities.py b/tests/rest/client/v2_alpha/test_capabilities.py
index bbfc77e829..8134163e20 100644
--- a/tests/rest/client/v2_alpha/test_capabilities.py
+++ b/tests/rest/client/v2_alpha/test_capabilities.py
@@ -12,9 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import synapse.rest.admin
 from synapse.api.room_versions import DEFAULT_ROOM_VERSION, KNOWN_ROOM_VERSIONS
-from synapse.rest.client.v1 import admin, login
+from synapse.rest.client.v1 import login
 from synapse.rest.client.v2_alpha import capabilities
 
 from tests import unittest
@@ -23,7 +23,7 @@ from tests import unittest
 class CapabilitiesTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         capabilities.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 3d44667489..4d698af03a 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -4,10 +4,11 @@ import os
 
 import pkg_resources
 
+import synapse.rest.admin
 from synapse.api.constants import LoginType
 from synapse.api.errors import Codes
 from synapse.appservice import ApplicationService
-from synapse.rest.client.v1 import admin, login
+from synapse.rest.client.v1 import login
 from synapse.rest.client.v2_alpha import account_validity, register, sync
 
 from tests import unittest
@@ -198,7 +199,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
 
     servlets = [
         register.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         login.register_servlets,
         sync.register_servlets,
         account_validity.register_servlets,
@@ -307,7 +308,7 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
         register.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         login.register_servlets,
         sync.register_servlets,
         account_validity.register_servlets,
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 99b716f00a..65fac1d5ce 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -15,7 +15,8 @@
 
 from mock import Mock
 
-from synapse.rest.client.v1 import admin, login, room
+import synapse.rest.admin
+from synapse.rest.client.v1 import login, room
 from synapse.rest.client.v2_alpha import sync
 
 from tests import unittest
@@ -72,7 +73,7 @@ class FilterTestCase(unittest.HomeserverTestCase):
 class SyncTypingTests(unittest.HomeserverTestCase):
 
     servlets = [
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         room.register_servlets,
         login.register_servlets,
         sync.register_servlets,
diff --git a/tests/server_notices/test_consent.py b/tests/server_notices/test_consent.py
index 95badc985e..e8b8ac5725 100644
--- a/tests/server_notices/test_consent.py
+++ b/tests/server_notices/test_consent.py
@@ -12,8 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from synapse.rest.client.v1 import admin, login, room
+import synapse.rest.admin
+from synapse.rest.client.v1 import login, room
 from synapse.rest.client.v2_alpha import sync
 
 from tests import unittest
@@ -23,7 +23,7 @@ class ConsentNoticesTests(unittest.HomeserverTestCase):
 
     servlets = [
         sync.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets,
         login.register_servlets,
         room.register_servlets,
     ]
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index 858efe4992..b0f6fd34d8 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -18,8 +18,9 @@ from mock import Mock
 
 from twisted.internet import defer
 
+import synapse.rest.admin
 from synapse.http.site import XForwardedForRequest
-from synapse.rest.client.v1 import admin, login
+from synapse.rest.client.v1 import login
 
 from tests import unittest
 
@@ -205,7 +206,7 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
 
 class ClientIpAuthTestCase(unittest.HomeserverTestCase):
 
-    servlets = [admin.register_servlets, login.register_servlets]
+    servlets = [synapse.rest.admin.register_servlets, login.register_servlets]
 
     def make_homeserver(self, reactor, clock):
         hs = self.setup_test_homeserver()
-- 
cgit 1.5.1


From f203c98794aa5344c1cbc26e526bd334fd80a1c2 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 1 May 2019 17:49:11 +0100
Subject: fix examples

---
 synapse/rest/admin/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index ddf9ced1a3..afac8b1724 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -640,7 +640,7 @@ class ResetPasswordRestServlet(RestServlet):
     """Post request to allow an administrator reset password for a user.
     This needs user to have administrator access in Synapse.
         Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/reset_password/
+            http://localhost:8008/_synapse/admin/v1/reset_password/
             @user:to_reset_password?access_token=admin_access_token
         JsonBodyToSend:
             {
@@ -683,7 +683,7 @@ class GetUsersPaginatedRestServlet(RestServlet):
     """Get request to get specific number of users from Synapse.
     This needs user to have administrator access in Synapse.
         Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/
+            http://localhost:8008/_synapse/admin/v1/users_paginate/
             @admin:user?access_token=admin_access_token&start=0&limit=10
         Returns:
             200 OK with json object {list[dict[str, Any]], count} or empty object.
@@ -731,7 +731,7 @@ class GetUsersPaginatedRestServlet(RestServlet):
         """Post request to get specific number of users from Synapse..
         This needs user to have administrator access in Synapse.
         Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/users_paginate/
+            http://localhost:8008/_synapse/admin/v1/users_paginate/
             @admin:user?access_token=admin_access_token
         JsonBodyToSend:
             {
@@ -766,7 +766,7 @@ class SearchUsersRestServlet(RestServlet):
     search term.
     This needs user to have administrator access in Synapse.
         Example:
-            http://localhost:8008/_matrix/client/api/v1/admin/search_users/
+            http://localhost:8008/_synapse/admin/v1/search_users/
             @admin:user?access_token=admin_access_token&term=alice
         Returns:
             200 OK with json object {list[dict[str, Any]], count} or empty object.
-- 
cgit 1.5.1


From 0836cbb9f5aaf9260161fa998b2fe4c0ea7e692b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 2 May 2019 10:45:52 +0100
Subject: Factor out an "assert_requester_is_admin" function (#5120)

Rather than copying-and-pasting the same four lines hundreds of times
---
 changelog.d/5120.misc          |  1 +
 synapse/api/auth.py            |  2 +-
 synapse/rest/admin/__init__.py | 95 +++++++++---------------------------------
 synapse/rest/admin/_base.py    | 59 ++++++++++++++++++++++++++
 4 files changed, 81 insertions(+), 76 deletions(-)
 create mode 100644 changelog.d/5120.misc
 create mode 100644 synapse/rest/admin/_base.py

(limited to 'synapse/rest')

diff --git a/changelog.d/5120.misc b/changelog.d/5120.misc
new file mode 100644
index 0000000000..6575f29322
--- /dev/null
+++ b/changelog.d/5120.misc
@@ -0,0 +1 @@
+Factor out an "assert_requester_is_admin" function.
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 960e66dbdc..0c6c93a87b 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -556,7 +556,7 @@ class Auth(object):
         """ Check if the given user is a local server admin.
 
         Args:
-            user (str): mxid of user to check
+            user (UserID): user to check
 
         Returns:
             bool: True if the user is an admin
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index afac8b1724..d02f5198b8 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -36,6 +36,7 @@ from synapse.http.servlet import (
     parse_json_object_from_request,
     parse_string,
 )
+from synapse.rest.admin._base import assert_requester_is_admin, assert_user_is_admin
 from synapse.types import UserID, create_requester
 from synapse.util.versionstring import get_version_string
 
@@ -75,15 +76,7 @@ class UsersRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         target_user = UserID.from_string(user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        # To allow all users to get the users list
-        # if not is_admin and target_user != auth_user:
-        #     raise AuthError(403, "You are not a server admin")
+        yield assert_requester_is_admin(self.auth, request)
 
         if not self.hs.is_mine(target_user):
             raise SynapseError(400, "Can only users a local user")
@@ -101,11 +94,7 @@ class VersionServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_requester_is_admin(self.auth, request)
 
         ret = {
             'server_version': get_version_string(synapse),
@@ -265,10 +254,9 @@ class WhoisRestServlet(RestServlet):
         target_user = UserID.from_string(user_id)
         requester = yield self.auth.get_user_by_req(request)
         auth_user = requester.user
-        is_admin = yield self.auth.is_server_admin(requester.user)
 
-        if not is_admin and target_user != auth_user:
-            raise AuthError(403, "You are not a server admin")
+        if target_user != auth_user:
+            yield assert_user_is_admin(self.auth, auth_user)
 
         if not self.hs.is_mine(target_user):
             raise SynapseError(400, "Can only whois a local user")
@@ -287,11 +275,7 @@ class PurgeMediaCacheRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_requester_is_admin(self.auth, request)
 
         before_ts = parse_integer(request, "before_ts", required=True)
         logger.info("before_ts: %r", before_ts)
@@ -318,11 +302,7 @@ class PurgeHistoryRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, event_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_requester_is_admin(self.auth, request)
 
         body = parse_json_object_from_request(request, allow_empty_body=True)
 
@@ -414,11 +394,7 @@ class PurgeHistoryStatusRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, purge_id):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_requester_is_admin(self.auth, request)
 
         purge_status = self.pagination_handler.get_purge_status(purge_id)
         if purge_status is None:
@@ -436,6 +412,7 @@ class DeactivateAccountRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, target_user_id):
+        yield assert_requester_is_admin(self.auth, request)
         body = parse_json_object_from_request(request, allow_empty_body=True)
         erase = body.get("erase", False)
         if not isinstance(erase, bool):
@@ -446,11 +423,6 @@ class DeactivateAccountRestServlet(RestServlet):
             )
 
         UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
 
         result = yield self._deactivate_account_handler.deactivate_account(
             target_user_id, erase,
@@ -490,9 +462,7 @@ class ShutdownRoomRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request, room_id):
         requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_user_is_admin(self.auth, requester.user)
 
         content = parse_json_object_from_request(request)
         assert_params_in_dict(content, ["new_room_user_id"])
@@ -605,9 +575,7 @@ class QuarantineMediaInRoom(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request, room_id):
         requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_user_is_admin(self.auth, requester.user)
 
         num_quarantined = yield self.store.quarantine_media_ids_in_room(
             room_id, requester.user.to_string(),
@@ -662,12 +630,10 @@ class ResetPasswordRestServlet(RestServlet):
         """Post request to allow an administrator reset password for a user.
         This needs user to have administrator access in Synapse.
         """
-        UserID.from_string(target_user_id)
         requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
+        yield assert_user_is_admin(self.auth, requester.user)
 
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        UserID.from_string(target_user_id)
 
         params = parse_json_object_from_request(request)
         assert_params_in_dict(params, ["new_password"])
@@ -701,16 +667,9 @@ class GetUsersPaginatedRestServlet(RestServlet):
         """Get request to get specific number of users from Synapse.
         This needs user to have administrator access in Synapse.
         """
-        target_user = UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
+        yield assert_requester_is_admin(self.auth, request)
 
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
-
-        # To allow all users to get the users list
-        # if not is_admin and target_user != auth_user:
-        #     raise AuthError(403, "You are not a server admin")
+        target_user = UserID.from_string(target_user_id)
 
         if not self.hs.is_mine(target_user):
             raise SynapseError(400, "Can only users a local user")
@@ -741,12 +700,8 @@ class GetUsersPaginatedRestServlet(RestServlet):
         Returns:
             200 OK with json object {list[dict[str, Any]], count} or empty object.
         """
+        yield assert_requester_is_admin(self.auth, request)
         UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
 
         order = "name"  # order by name in user table
         params = parse_json_object_from_request(request)
@@ -785,12 +740,9 @@ class SearchUsersRestServlet(RestServlet):
         search term.
         This needs user to have a administrator access in Synapse.
         """
-        target_user = UserID.from_string(target_user_id)
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
+        yield assert_requester_is_admin(self.auth, request)
 
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        target_user = UserID.from_string(target_user_id)
 
         # To allow all users to get the users list
         # if not is_admin and target_user != auth_user:
@@ -821,10 +773,7 @@ class DeleteGroupAdminRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request, group_id):
         requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_user_is_admin(self.auth, requester.user)
 
         if not self.is_mine_id(group_id):
             raise SynapseError(400, "Can only delete local groups")
@@ -847,11 +796,7 @@ class AccountValidityRenewServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        requester = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(requester.user)
-
-        if not is_admin:
-            raise AuthError(403, "You are not a server admin")
+        yield assert_requester_is_admin(self.auth, request)
 
         body = parse_json_object_from_request(request)
 
diff --git a/synapse/rest/admin/_base.py b/synapse/rest/admin/_base.py
new file mode 100644
index 0000000000..881d67b89c
--- /dev/null
+++ b/synapse/rest/admin/_base.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from twisted.internet import defer
+
+from synapse.api.errors import AuthError
+
+
+@defer.inlineCallbacks
+def assert_requester_is_admin(auth, request):
+    """Verify that the requester is an admin user
+
+    WARNING: MAKE SURE YOU YIELD ON THE RESULT!
+
+    Args:
+        auth (synapse.api.auth.Auth):
+        request (twisted.web.server.Request): incoming request
+
+    Returns:
+        Deferred
+
+    Raises:
+        AuthError if the requester is not an admin
+    """
+    requester = yield auth.get_user_by_req(request)
+    yield assert_user_is_admin(auth, requester.user)
+
+
+@defer.inlineCallbacks
+def assert_user_is_admin(auth, user_id):
+    """Verify that the given user is an admin user
+
+    WARNING: MAKE SURE YOU YIELD ON THE RESULT!
+
+    Args:
+        auth (synapse.api.auth.Auth):
+        user_id (UserID):
+
+    Returns:
+        Deferred
+
+    Raises:
+        AuthError if the user is not an admin
+    """
+
+    is_admin = yield auth.is_server_admin(user_id)
+    if not is_admin:
+        raise AuthError(403, "You are not a server admin")
-- 
cgit 1.5.1


From 12f9d51e826058998cb11759e068de8977ddd3d5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 2 May 2019 11:59:16 +0100
Subject: Add admin api for sending server_notices (#5121)

---
 changelog.d/5121.feature                        |   1 +
 docs/admin_api/server_notices.md                |  48 ++++++++++++
 docs/server_notices.md                          |  25 ++----
 synapse/rest/__init__.py                        |   4 +-
 synapse/rest/admin/__init__.py                  |  17 +++-
 synapse/rest/admin/server_notice_servlet.py     | 100 ++++++++++++++++++++++++
 tests/handlers/test_user_directory.py           |   4 +-
 tests/push/test_email.py                        |   2 +-
 tests/push/test_http.py                         |   2 +-
 tests/rest/admin/test_admin.py                  |   8 +-
 tests/rest/client/test_consent.py               |   2 +-
 tests/rest/client/test_identity.py              |   2 +-
 tests/rest/client/v1/test_events.py             |   2 +-
 tests/rest/client/v1/test_login.py              |   2 +-
 tests/rest/client/v1/test_rooms.py              |   2 +-
 tests/rest/client/v2_alpha/test_auth.py         |   2 +-
 tests/rest/client/v2_alpha/test_capabilities.py |   2 +-
 tests/rest/client/v2_alpha/test_register.py     |   4 +-
 tests/rest/client/v2_alpha/test_sync.py         |   2 +-
 tests/server_notices/test_consent.py            |   2 +-
 tests/storage/test_client_ips.py                |   5 +-
 21 files changed, 196 insertions(+), 42 deletions(-)
 create mode 100644 changelog.d/5121.feature
 create mode 100644 docs/admin_api/server_notices.md
 create mode 100644 synapse/rest/admin/server_notice_servlet.py

(limited to 'synapse/rest')

diff --git a/changelog.d/5121.feature b/changelog.d/5121.feature
new file mode 100644
index 0000000000..54b228680d
--- /dev/null
+++ b/changelog.d/5121.feature
@@ -0,0 +1 @@
+Implement an admin API for sending server notices. Many thanks to @krombel who provided a foundation for this work.
diff --git a/docs/admin_api/server_notices.md b/docs/admin_api/server_notices.md
new file mode 100644
index 0000000000..5ddd21cfb2
--- /dev/null
+++ b/docs/admin_api/server_notices.md
@@ -0,0 +1,48 @@
+# Server Notices
+
+The API to send notices is as follows:
+
+```
+POST /_synapse/admin/v1/send_server_notice
+```
+
+or:
+
+```
+PUT /_synapse/admin/v1/send_server_notice/{txnId}
+```
+
+You will need to authenticate with an access token for an admin user.
+
+When using the `PUT` form, retransmissions with the same transaction ID will be
+ignored in the same way as with `PUT
+/_matrix/client/r0/rooms/{roomId}/send/{eventType}/{txnId}`.
+
+The request body should look something like the following:
+
+```json
+{
+    "user_id": "@target_user:server_name",
+    "content": {
+        "msgtype": "m.text",
+        "body": "This is my message"
+    }
+}
+```
+
+You can optionally include the following additional parameters:
+
+* `type`: the type of event. Defaults to `m.room.message`.
+* `state_key`: Setting this will result in a state event being sent.
+
+
+Once the notice has been sent, the APU will return the following response:
+
+```json
+{
+    "event_id": "<event_id>"
+}
+```
+
+Note that server notices must be enabled in `homeserver.yaml` before this API
+can be used. See [server_notices.md](../server_notices.md) for more information.
diff --git a/docs/server_notices.md b/docs/server_notices.md
index 58f8776319..950a6608e9 100644
--- a/docs/server_notices.md
+++ b/docs/server_notices.md
@@ -1,5 +1,4 @@
-Server Notices
-==============
+# Server Notices
 
 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a
 channel whereby server administrators can send messages to users on the server.
@@ -11,8 +10,7 @@ they may also find a use for features such as "Message of the day".
 This is a feature specific to Synapse, but it uses standard Matrix
 communication mechanisms, so should work with any Matrix client.
 
-User experience
----------------
+## User experience
 
 When the user is first sent a server notice, they will get an invitation to a
 room (typically called 'Server Notices', though this is configurable in
@@ -29,8 +27,7 @@ levels.
 Having joined the room, the user can leave the room if they want. Subsequent
 server notices will then cause a new room to be created.
 
-Synapse configuration
----------------------
+## Synapse configuration
 
 Server notices come from a specific user id on the server. Server
 administrators are free to choose the user id - something like `server` is
@@ -58,17 +55,7 @@ room which will be created.
 `system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
 displayname and avatar of the Server Notices user.
 
-Sending notices
----------------
+## Sending notices
 
-As of the current version of synapse, there is no convenient interface for
-sending notices (other than the automated ones sent as part of consent
-tracking).
-
-In the meantime, it is possible to test this feature using the manhole. Having
-gone into the manhole as described in [manhole.md](manhole.md), a notice can be
-sent with something like:
-
-```
->>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'})
-```
+To send server notices to users you can use the
+[admin_api](admin_api/server_notices.md).
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index e8e1bcddea..3a24d31d1b 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -117,4 +117,6 @@ class ClientRestResource(JsonResource):
         account_validity.register_servlets(hs, client_resource)
 
         # moving to /_synapse/admin
-        synapse.rest.admin.register_servlets(hs, client_resource)
+        synapse.rest.admin.register_servlets_for_client_rest_resource(
+            hs, client_resource
+        )
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index d02f5198b8..0ce89741f0 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -37,6 +37,7 @@ from synapse.http.servlet import (
     parse_string,
 )
 from synapse.rest.admin._base import assert_requester_is_admin, assert_user_is_admin
+from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet
 from synapse.types import UserID, create_requester
 from synapse.util.versionstring import get_version_string
 
@@ -813,16 +814,26 @@ class AccountValidityRenewServlet(RestServlet):
         }
         defer.returnValue((200, res))
 
+########################################################################################
+#
+# please don't add more servlets here: this file is already long and unwieldy. Put
+# them in separate files within the 'admin' package.
+#
+########################################################################################
+
 
 class AdminRestResource(JsonResource):
     """The REST resource which gets mounted at /_synapse/admin"""
 
     def __init__(self, hs):
         JsonResource.__init__(self, hs, canonical_json=False)
-        register_servlets(hs, self)
+
+        register_servlets_for_client_rest_resource(hs, self)
+        SendServerNoticeServlet(hs).register(self)
 
 
-def register_servlets(hs, http_server):
+def register_servlets_for_client_rest_resource(hs, http_server):
+    """Register only the servlets which need to be exposed on /_matrix/client/xxx"""
     WhoisRestServlet(hs).register(http_server)
     PurgeMediaCacheRestServlet(hs).register(http_server)
     PurgeHistoryStatusRestServlet(hs).register(http_server)
@@ -839,3 +850,5 @@ def register_servlets(hs, http_server):
     VersionServlet(hs).register(http_server)
     DeleteGroupAdminRestServlet(hs).register(http_server)
     AccountValidityRenewServlet(hs).register(http_server)
+    # don't add more things here: new servlets should only be exposed on
+    # /_synapse/admin so should not go here. Instead register them in AdminRestResource.
diff --git a/synapse/rest/admin/server_notice_servlet.py b/synapse/rest/admin/server_notice_servlet.py
new file mode 100644
index 0000000000..ae5aca9dac
--- /dev/null
+++ b/synapse/rest/admin/server_notice_servlet.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+
+from twisted.internet import defer
+
+from synapse.api.constants import EventTypes
+from synapse.api.errors import SynapseError
+from synapse.http.servlet import (
+    RestServlet,
+    assert_params_in_dict,
+    parse_json_object_from_request,
+)
+from synapse.rest.admin import assert_requester_is_admin
+from synapse.rest.client.transactions import HttpTransactionCache
+from synapse.types import UserID
+
+
+class SendServerNoticeServlet(RestServlet):
+    """Servlet which will send a server notice to a given user
+
+    POST /_synapse/admin/v1/send_server_notice
+    {
+        "user_id": "@target_user:server_name",
+        "content": {
+            "msgtype": "m.text",
+            "body": "This is my message"
+        }
+    }
+
+    returns:
+
+    {
+        "event_id": "$1895723857jgskldgujpious"
+    }
+    """
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.txns = HttpTransactionCache(hs)
+        self.snm = hs.get_server_notices_manager()
+
+    def register(self, json_resource):
+        PATTERN = "^/_synapse/admin/v1/send_server_notice"
+        json_resource.register_paths(
+            "POST",
+            (re.compile(PATTERN + "$"), ),
+            self.on_POST,
+        )
+        json_resource.register_paths(
+            "PUT",
+            (re.compile(PATTERN + "/(?P<txn_id>[^/]*)$",), ),
+            self.on_PUT,
+        )
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, txn_id=None):
+        yield assert_requester_is_admin(self.auth, request)
+        body = parse_json_object_from_request(request)
+        assert_params_in_dict(body, ("user_id", "content"))
+        event_type = body.get("type", EventTypes.Message)
+        state_key = body.get("state_key")
+
+        if not self.snm.is_enabled():
+            raise SynapseError(400, "Server notices are not enabled on this server")
+
+        user_id = body["user_id"]
+        UserID.from_string(user_id)
+        if not self.hs.is_mine_id(user_id):
+            raise SynapseError(400, "Server notices can only be sent to local users")
+
+        event = yield self.snm.send_notice(
+            user_id=body["user_id"],
+            type=event_type,
+            state_key=state_key,
+            event_content=body["content"],
+        )
+
+        defer.returnValue((200, {"event_id": event.event_id}))
+
+    def on_PUT(self, request, txn_id):
+        return self.txns.fetch_or_execute_request(
+            request, self.on_POST, request, txn_id,
+        )
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index 32ef83e9c0..7dd1a1daf8 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -30,7 +30,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
 
     servlets = [
         login.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
     ]
 
@@ -328,7 +328,7 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase):
         user_directory.register_servlets,
         room.register_servlets,
         login.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
     ]
 
     def make_homeserver(self, reactor, clock):
diff --git a/tests/push/test_email.py b/tests/push/test_email.py
index e29bd18ad7..325ea449ae 100644
--- a/tests/push/test_email.py
+++ b/tests/push/test_email.py
@@ -34,7 +34,7 @@ class EmailPusherTests(HomeserverTestCase):
 
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/push/test_http.py b/tests/push/test_http.py
index 3f9f56bb79..13bd2c8688 100644
--- a/tests/push/test_http.py
+++ b/tests/push/test_http.py
@@ -33,7 +33,7 @@ class HTTPPusherTests(HomeserverTestCase):
 
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py
index 42858b5fea..db4cfd8550 100644
--- a/tests/rest/admin/test_admin.py
+++ b/tests/rest/admin/test_admin.py
@@ -30,7 +30,7 @@ from tests import unittest
 class VersionTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
     ]
 
@@ -63,7 +63,7 @@ class VersionTestCase(unittest.HomeserverTestCase):
 
 class UserRegisterTestCase(unittest.HomeserverTestCase):
 
-    servlets = [synapse.rest.admin.register_servlets]
+    servlets = [synapse.rest.admin.register_servlets_for_client_rest_resource]
 
     def make_homeserver(self, reactor, clock):
 
@@ -359,7 +359,7 @@ class UserRegisterTestCase(unittest.HomeserverTestCase):
 
 class ShutdownRoomTestCase(unittest.HomeserverTestCase):
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
         events.register_servlets,
         room.register_servlets,
@@ -496,7 +496,7 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase):
 
 class DeleteGroupTestCase(unittest.HomeserverTestCase):
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
         groups.register_servlets,
     ]
diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py
index 36e6c1c67d..5528971190 100644
--- a/tests/rest/client/test_consent.py
+++ b/tests/rest/client/test_consent.py
@@ -32,7 +32,7 @@ except Exception:
 class ConsentResourceTestCase(unittest.HomeserverTestCase):
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py
index d4fe0aee7d..2e51ffa418 100644
--- a/tests/rest/client/test_identity.py
+++ b/tests/rest/client/test_identity.py
@@ -24,7 +24,7 @@ from tests import unittest
 class IdentityTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 5cb1c1ae9f..8a9a55a527 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -29,7 +29,7 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
     servlets = [
         events.register_servlets,
         room.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
     ]
 
diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py
index 8d9ef877f6..9ebd91f678 100644
--- a/tests/rest/client/v1/test_login.py
+++ b/tests/rest/client/v1/test_login.py
@@ -11,7 +11,7 @@ LOGIN_URL = b"/_matrix/client/r0/login"
 class LoginRestServletTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
     ]
 
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 1a34924f3e..521ac80f9a 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -804,7 +804,7 @@ class RoomMessageListTestCase(RoomBase):
 
 class RoomSearchTestCase(unittest.HomeserverTestCase):
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py
index 67021185d0..0ca3c4657b 100644
--- a/tests/rest/client/v2_alpha/test_auth.py
+++ b/tests/rest/client/v2_alpha/test_auth.py
@@ -27,7 +27,7 @@ class FallbackAuthTests(unittest.HomeserverTestCase):
 
     servlets = [
         auth.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         register.register_servlets,
     ]
     hijack_auth = False
diff --git a/tests/rest/client/v2_alpha/test_capabilities.py b/tests/rest/client/v2_alpha/test_capabilities.py
index 8134163e20..f3ef977404 100644
--- a/tests/rest/client/v2_alpha/test_capabilities.py
+++ b/tests/rest/client/v2_alpha/test_capabilities.py
@@ -23,7 +23,7 @@ from tests import unittest
 class CapabilitiesTestCase(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         capabilities.register_servlets,
         login.register_servlets,
     ]
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 4d698af03a..1c3a621d26 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -199,7 +199,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase):
 
     servlets = [
         register.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
         sync.register_servlets,
         account_validity.register_servlets,
@@ -308,7 +308,7 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase):
     skip = "No Jinja installed" if not load_jinja2_templates else None
     servlets = [
         register.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
         sync.register_servlets,
         account_validity.register_servlets,
diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py
index 65fac1d5ce..71895094bd 100644
--- a/tests/rest/client/v2_alpha/test_sync.py
+++ b/tests/rest/client/v2_alpha/test_sync.py
@@ -73,7 +73,7 @@ class FilterTestCase(unittest.HomeserverTestCase):
 class SyncTypingTests(unittest.HomeserverTestCase):
 
     servlets = [
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
         login.register_servlets,
         sync.register_servlets,
diff --git a/tests/server_notices/test_consent.py b/tests/server_notices/test_consent.py
index e8b8ac5725..e0b4e0eb63 100644
--- a/tests/server_notices/test_consent.py
+++ b/tests/server_notices/test_consent.py
@@ -23,7 +23,7 @@ class ConsentNoticesTests(unittest.HomeserverTestCase):
 
     servlets = [
         sync.register_servlets,
-        synapse.rest.admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         login.register_servlets,
         room.register_servlets,
     ]
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
index b0f6fd34d8..b62eae7abc 100644
--- a/tests/storage/test_client_ips.py
+++ b/tests/storage/test_client_ips.py
@@ -206,7 +206,10 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase):
 
 class ClientIpAuthTestCase(unittest.HomeserverTestCase):
 
-    servlets = [synapse.rest.admin.register_servlets, login.register_servlets]
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        login.register_servlets,
+    ]
 
     def make_homeserver(self, reactor, clock):
         hs = self.setup_test_homeserver()
-- 
cgit 1.5.1


From 59e2d2694deec13aaa0062e04b5460f978967dc1 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 7 May 2019 09:29:30 +0100
Subject: Remove the requirement to authenticate for /admin/server_version.
 (#5122)

This endpoint isn't much use for its intended purpose if you first need to get
yourself an admin's auth token.

I've restricted it to the `/_synapse/admin` path to make it a bit easier to
lock down for those concerned about exposing this information. I don't imagine
anyone is using it in anger currently.
---
 changelog.d/5122.misc          |  1 +
 docs/admin_api/version_api.rst |  2 --
 synapse/rest/admin/__init__.py | 15 +++++----------
 tests/rest/admin/test_admin.py | 30 ++++++++----------------------
 tests/unittest.py              | 22 ++++++++++++++++++----
 5 files changed, 32 insertions(+), 38 deletions(-)
 create mode 100644 changelog.d/5122.misc

(limited to 'synapse/rest')

diff --git a/changelog.d/5122.misc b/changelog.d/5122.misc
new file mode 100644
index 0000000000..e1be8a6210
--- /dev/null
+++ b/changelog.d/5122.misc
@@ -0,0 +1 @@
+Remove the requirement to authenticate for /admin/server_version.
diff --git a/docs/admin_api/version_api.rst b/docs/admin_api/version_api.rst
index 6d66543b96..833d9028be 100644
--- a/docs/admin_api/version_api.rst
+++ b/docs/admin_api/version_api.rst
@@ -10,8 +10,6 @@ The api is::
 
     GET /_synapse/admin/v1/server_version
 
-including an ``access_token`` of a server admin.
-
 It returns a JSON body like the following:
 
 .. code:: json
diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py
index 0ce89741f0..744d85594f 100644
--- a/synapse/rest/admin/__init__.py
+++ b/synapse/rest/admin/__init__.py
@@ -88,21 +88,16 @@ class UsersRestServlet(RestServlet):
 
 
 class VersionServlet(RestServlet):
-    PATTERNS = historical_admin_path_patterns("/server_version")
+    PATTERNS = (re.compile("^/_synapse/admin/v1/server_version$"), )
 
     def __init__(self, hs):
-        self.auth = hs.get_auth()
-
-    @defer.inlineCallbacks
-    def on_GET(self, request):
-        yield assert_requester_is_admin(self.auth, request)
-
-        ret = {
+        self.res = {
             'server_version': get_version_string(synapse),
             'python_version': platform.python_version(),
         }
 
-        defer.returnValue((200, ret))
+    def on_GET(self, request):
+        return 200, self.res
 
 
 class UserRegisterServlet(RestServlet):
@@ -830,6 +825,7 @@ class AdminRestResource(JsonResource):
 
         register_servlets_for_client_rest_resource(hs, self)
         SendServerNoticeServlet(hs).register(self)
+        VersionServlet(hs).register(self)
 
 
 def register_servlets_for_client_rest_resource(hs, http_server):
@@ -847,7 +843,6 @@ def register_servlets_for_client_rest_resource(hs, http_server):
     QuarantineMediaInRoom(hs).register(http_server)
     ListMediaInRoom(hs).register(http_server)
     UserRegisterServlet(hs).register(http_server)
-    VersionServlet(hs).register(http_server)
     DeleteGroupAdminRestServlet(hs).register(http_server)
     AccountValidityRenewServlet(hs).register(http_server)
     # don't add more things here: new servlets should only be exposed on
diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py
index db4cfd8550..da19a83918 100644
--- a/tests/rest/admin/test_admin.py
+++ b/tests/rest/admin/test_admin.py
@@ -21,6 +21,8 @@ from mock import Mock
 
 import synapse.rest.admin
 from synapse.api.constants import UserTypes
+from synapse.http.server import JsonResource
+from synapse.rest.admin import VersionServlet
 from synapse.rest.client.v1 import events, login, room
 from synapse.rest.client.v2_alpha import groups
 
@@ -28,20 +30,15 @@ from tests import unittest
 
 
 class VersionTestCase(unittest.HomeserverTestCase):
+    url = '/_synapse/admin/v1/server_version'
 
-    servlets = [
-        synapse.rest.admin.register_servlets_for_client_rest_resource,
-        login.register_servlets,
-    ]
-
-    url = '/_matrix/client/r0/admin/server_version'
+    def create_test_json_resource(self):
+        resource = JsonResource(self.hs)
+        VersionServlet(self.hs).register(resource)
+        return resource
 
     def test_version_string(self):
-        self.register_user("admin", "pass", admin=True)
-        self.admin_token = self.login("admin", "pass")
-
-        request, channel = self.make_request("GET", self.url,
-                                             access_token=self.admin_token)
+        request, channel = self.make_request("GET", self.url, shorthand=False)
         self.render(request)
 
         self.assertEqual(200, int(channel.result["code"]),
@@ -49,17 +46,6 @@ class VersionTestCase(unittest.HomeserverTestCase):
         self.assertEqual({'server_version', 'python_version'},
                          set(channel.json_body.keys()))
 
-    def test_inaccessible_to_non_admins(self):
-        self.register_user("unprivileged-user", "pass", admin=False)
-        user_token = self.login("unprivileged-user", "pass")
-
-        request, channel = self.make_request("GET", self.url,
-                                             access_token=user_token)
-        self.render(request)
-
-        self.assertEqual(403, int(channel.result['code']),
-                         msg=channel.result['body'])
-
 
 class UserRegisterTestCase(unittest.HomeserverTestCase):
 
diff --git a/tests/unittest.py b/tests/unittest.py
index 8c65736a51..029a88d770 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -181,10 +181,7 @@ class HomeserverTestCase(TestCase):
             raise Exception("A homeserver wasn't returned, but %r" % (self.hs,))
 
         # Register the resources
-        self.resource = JsonResource(self.hs)
-
-        for servlet in self.servlets:
-            servlet(self.hs, self.resource)
+        self.resource = self.create_test_json_resource()
 
         from tests.rest.client.v1.utils import RestHelper
 
@@ -230,6 +227,23 @@ class HomeserverTestCase(TestCase):
         hs = self.setup_test_homeserver()
         return hs
 
+    def create_test_json_resource(self):
+        """
+        Create a test JsonResource, with the relevant servlets registerd to it
+
+        The default implementation calls each function in `servlets` to do the
+        registration.
+
+        Returns:
+            JsonResource:
+        """
+        resource = JsonResource(self.hs)
+
+        for servlet in self.servlets:
+            servlet(self.hs, resource)
+
+        return resource
+
     def default_config(self, name="test"):
         """
         Get a default HomeServer config object.
-- 
cgit 1.5.1


From c0e0740bef0db661abce352afaf6c958e276f11d Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Wed, 8 May 2019 18:26:56 +0100
Subject: add options to require an access_token to GET /profile and
 /publicRooms on CS API (#5083)

This commit adds two config options:

* `restrict_public_rooms_to_local_users`

Requires auth to fetch the public rooms directory through the CS API and disables fetching it through the federation API.

* `require_auth_for_profile_requests`

When set to `true`, requires that requests to `/profile` over the CS API are authenticated, and only returns the user's profile if the requester shares a room with the profile's owner, as per MSC1301.

MSC1301 also specifies a behaviour for federation (only returning the profile if the server asking for it shares a room with the profile's owner), but that's currently really non-trivial to do in a not too expensive way. Next step is writing down a MSC that allows a HS to specify which user sent the profile query. In this implementation, Synapse won't send a profile query over federation if it doesn't believe it already shares a room with the profile's owner, though.

Groups have been intentionally omitted from this commit.
---
 changelog.d/5083.feature               |  1 +
 docs/sample_config.yaml                | 14 ++++++
 synapse/config/server.py               | 27 ++++++++++
 synapse/federation/transport/server.py | 10 ++++
 synapse/handlers/profile.py            | 43 ++++++++++++++++
 synapse/rest/client/v1/profile.py      | 40 ++++++++++-----
 synapse/rest/client/v1/room.py         |  6 +++
 tests/rest/client/v1/test_profile.py   | 92 +++++++++++++++++++++++++++++++++-
 tests/rest/client/v1/test_rooms.py     | 32 ++++++++++++
 9 files changed, 252 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/5083.feature

(limited to 'synapse/rest')

diff --git a/changelog.d/5083.feature b/changelog.d/5083.feature
new file mode 100644
index 0000000000..55d114b3fe
--- /dev/null
+++ b/changelog.d/5083.feature
@@ -0,0 +1 @@
+Add an configuration option to require authentication on /publicRooms and /profile endpoints.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index b6b9da6e41..bdfc34c6bd 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -69,6 +69,20 @@ pid_file: DATADIR/homeserver.pid
 #
 #use_presence: false
 
+# Whether to require authentication to retrieve profile data (avatars,
+# display names) of other users through the client API. Defaults to
+# 'false'. Note that profile data is also available via the federation
+# API, so this setting is of limited value if federation is enabled on
+# the server.
+#
+#require_auth_for_profile_requests: true
+
+# If set to 'true', requires authentication to access the server's
+# public rooms directory through the client API, and forbids any other
+# homeserver to fetch it via federation. Defaults to 'false'.
+#
+#restrict_public_rooms_to_local_users: true
+
 # The GC threshold parameters to pass to `gc.set_threshold`, if defined
 #
 #gc_thresholds: [700, 10, 10]
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 147a976485..8dce75c56a 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -72,6 +72,19 @@ class ServerConfig(Config):
         # master, potentially causing inconsistency.
         self.enable_media_repo = config.get("enable_media_repo", True)
 
+        # Whether to require authentication to retrieve profile data (avatars,
+        # display names) of other users through the client API.
+        self.require_auth_for_profile_requests = config.get(
+            "require_auth_for_profile_requests", False,
+        )
+
+        # If set to 'True', requires authentication to access the server's
+        # public rooms directory through the client API, and forbids any other
+        # homeserver to fetch it via federation.
+        self.restrict_public_rooms_to_local_users = config.get(
+            "restrict_public_rooms_to_local_users", False,
+        )
+
         # whether to enable search. If disabled, new entries will not be inserted
         # into the search tables and they will not be indexed. Users will receive
         # errors when attempting to search for messages.
@@ -327,6 +340,20 @@ class ServerConfig(Config):
         #
         #use_presence: false
 
+        # Whether to require authentication to retrieve profile data (avatars,
+        # display names) of other users through the client API. Defaults to
+        # 'false'. Note that profile data is also available via the federation
+        # API, so this setting is of limited value if federation is enabled on
+        # the server.
+        #
+        #require_auth_for_profile_requests: true
+
+        # If set to 'true', requires authentication to access the server's
+        # public rooms directory through the client API, and forbids any other
+        # homeserver to fetch it via federation. Defaults to 'false'.
+        #
+        #restrict_public_rooms_to_local_users: true
+
         # The GC threshold parameters to pass to `gc.set_threshold`, if defined
         #
         #gc_thresholds: [700, 10, 10]
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 452599e1a1..9030eb18c5 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -716,8 +716,17 @@ class PublicRoomList(BaseFederationServlet):
 
     PATH = "/publicRooms"
 
+    def __init__(self, handler, authenticator, ratelimiter, server_name, deny_access):
+        super(PublicRoomList, self).__init__(
+            handler, authenticator, ratelimiter, server_name,
+        )
+        self.deny_access = deny_access
+
     @defer.inlineCallbacks
     def on_GET(self, origin, content, query):
+        if self.deny_access:
+            raise FederationDeniedError(origin)
+
         limit = parse_integer_from_args(query, "limit", 0)
         since_token = parse_string_from_args(query, "since", None)
         include_all_networks = parse_boolean_from_args(
@@ -1417,6 +1426,7 @@ def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=N
                 authenticator=authenticator,
                 ratelimiter=ratelimiter,
                 server_name=hs.hostname,
+                deny_access=hs.config.restrict_public_rooms_to_local_users,
             ).register(resource)
 
     if "group_server" in servlet_groups:
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index a65c98ff5c..91fc718ff8 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -53,6 +53,7 @@ class BaseProfileHandler(BaseHandler):
     @defer.inlineCallbacks
     def get_profile(self, user_id):
         target_user = UserID.from_string(user_id)
+
         if self.hs.is_mine(target_user):
             try:
                 displayname = yield self.store.get_profile_displayname(
@@ -283,6 +284,48 @@ class BaseProfileHandler(BaseHandler):
                     room_id, str(e)
                 )
 
+    @defer.inlineCallbacks
+    def check_profile_query_allowed(self, target_user, requester=None):
+        """Checks whether a profile query is allowed. If the
+        'require_auth_for_profile_requests' config flag is set to True and a
+        'requester' is provided, the query is only allowed if the two users
+        share a room.
+
+        Args:
+            target_user (UserID): The owner of the queried profile.
+            requester (None|UserID): The user querying for the profile.
+
+        Raises:
+            SynapseError(403): The two users share no room, or ne user couldn't
+                be found to be in any room the server is in, and therefore the query
+                is denied.
+        """
+        # Implementation of MSC1301: don't allow looking up profiles if the
+        # requester isn't in the same room as the target. We expect requester to
+        # be None when this function is called outside of a profile query, e.g.
+        # when building a membership event. In this case, we must allow the
+        # lookup.
+        if not self.hs.config.require_auth_for_profile_requests or not requester:
+            return
+
+        try:
+            requester_rooms = yield self.store.get_rooms_for_user(
+                requester.to_string()
+            )
+            target_user_rooms = yield self.store.get_rooms_for_user(
+                target_user.to_string(),
+            )
+
+            # Check if the room lists have no elements in common.
+            if requester_rooms.isdisjoint(target_user_rooms):
+                raise SynapseError(403, "Profile isn't available", Codes.FORBIDDEN)
+        except StoreError as e:
+            if e.code == 404:
+                # This likely means that one of the users doesn't exist,
+                # so we act as if we couldn't find the profile.
+                raise SynapseError(403, "Profile isn't available", Codes.FORBIDDEN)
+            raise
+
 
 class MasterProfileHandler(BaseProfileHandler):
     PROFILE_UPDATE_MS = 60 * 1000
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index a23edd8fe5..eac1966c5e 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -31,11 +31,17 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
+        requester_user = None
+
+        if self.hs.config.require_auth_for_profile_requests:
+            requester = yield self.auth.get_user_by_req(request)
+            requester_user = requester.user
+
         user = UserID.from_string(user_id)
 
-        displayname = yield self.profile_handler.get_displayname(
-            user,
-        )
+        yield self.profile_handler.check_profile_query_allowed(user, requester_user)
+
+        displayname = yield self.profile_handler.get_displayname(user)
 
         ret = {}
         if displayname is not None:
@@ -74,11 +80,17 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
+        requester_user = None
+
+        if self.hs.config.require_auth_for_profile_requests:
+            requester = yield self.auth.get_user_by_req(request)
+            requester_user = requester.user
+
         user = UserID.from_string(user_id)
 
-        avatar_url = yield self.profile_handler.get_avatar_url(
-            user,
-        )
+        yield self.profile_handler.check_profile_query_allowed(user, requester_user)
+
+        avatar_url = yield self.profile_handler.get_avatar_url(user)
 
         ret = {}
         if avatar_url is not None:
@@ -116,14 +128,18 @@ class ProfileRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
+        requester_user = None
+
+        if self.hs.config.require_auth_for_profile_requests:
+            requester = yield self.auth.get_user_by_req(request)
+            requester_user = requester.user
+
         user = UserID.from_string(user_id)
 
-        displayname = yield self.profile_handler.get_displayname(
-            user,
-        )
-        avatar_url = yield self.profile_handler.get_avatar_url(
-            user,
-        )
+        yield self.profile_handler.check_profile_query_allowed(user, requester_user)
+
+        displayname = yield self.profile_handler.get_displayname(user)
+        avatar_url = yield self.profile_handler.get_avatar_url(user)
 
         ret = {}
         if displayname is not None:
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 48da4d557f..fab04965cb 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -301,6 +301,12 @@ class PublicRoomListRestServlet(ClientV1RestServlet):
         try:
             yield self.auth.get_user_by_req(request, allow_guest=True)
         except AuthError as e:
+            # Option to allow servers to require auth when accessing
+            # /publicRooms via CS API. This is especially helpful in private
+            # federations.
+            if self.hs.config.restrict_public_rooms_to_local_users:
+                raise
+
             # We allow people to not be authed if they're just looking at our
             # room list, but require auth when we proxy the request.
             # In both cases we call the auth function, as that has the side
diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py
index 1eab9c3bdb..5d13de11e6 100644
--- a/tests/rest/client/v1/test_profile.py
+++ b/tests/rest/client/v1/test_profile.py
@@ -20,7 +20,7 @@ from twisted.internet import defer
 
 import synapse.types
 from synapse.api.errors import AuthError, SynapseError
-from synapse.rest.client.v1 import profile
+from synapse.rest.client.v1 import admin, login, profile, room
 
 from tests import unittest
 
@@ -42,6 +42,7 @@ class ProfileTestCase(unittest.TestCase):
                 "set_displayname",
                 "get_avatar_url",
                 "set_avatar_url",
+                "check_profile_query_allowed",
             ]
         )
 
@@ -155,3 +156,92 @@ class ProfileTestCase(unittest.TestCase):
         self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD")
         self.assertEquals(mocked_set.call_args[0][1].user.localpart, "1234ABCD")
         self.assertEquals(mocked_set.call_args[0][2], "http://my.server/pic.gif")
+
+
+class ProfilesRestrictedTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        profile.register_servlets,
+        room.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+
+        config = self.default_config()
+        config.require_auth_for_profile_requests = True
+        self.hs = self.setup_test_homeserver(config=config)
+
+        return self.hs
+
+    def prepare(self, reactor, clock, hs):
+        # User owning the requested profile.
+        self.owner = self.register_user("owner", "pass")
+        self.owner_tok = self.login("owner", "pass")
+        self.profile_url = "/profile/%s" % (self.owner)
+
+        # User requesting the profile.
+        self.requester = self.register_user("requester", "pass")
+        self.requester_tok = self.login("requester", "pass")
+
+        self.room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok)
+
+    def test_no_auth(self):
+        self.try_fetch_profile(401)
+
+    def test_not_in_shared_room(self):
+        self.ensure_requester_left_room()
+
+        self.try_fetch_profile(403, access_token=self.requester_tok)
+
+    def test_in_shared_room(self):
+        self.ensure_requester_left_room()
+
+        self.helper.join(
+            room=self.room_id,
+            user=self.requester,
+            tok=self.requester_tok,
+        )
+
+        self.try_fetch_profile(200, self.requester_tok)
+
+    def try_fetch_profile(self, expected_code, access_token=None):
+        self.request_profile(
+            expected_code,
+            access_token=access_token
+        )
+
+        self.request_profile(
+            expected_code,
+            url_suffix="/displayname",
+            access_token=access_token,
+        )
+
+        self.request_profile(
+            expected_code,
+            url_suffix="/avatar_url",
+            access_token=access_token,
+        )
+
+    def request_profile(self, expected_code, url_suffix="", access_token=None):
+        request, channel = self.make_request(
+            "GET",
+            self.profile_url + url_suffix,
+            access_token=access_token,
+        )
+        self.render(request)
+        self.assertEqual(channel.code, expected_code, channel.result)
+
+    def ensure_requester_left_room(self):
+        try:
+            self.helper.leave(
+                room=self.room_id,
+                user=self.requester,
+                tok=self.requester_tok,
+            )
+        except AssertionError:
+            # We don't care whether the leave request didn't return a 200 (e.g.
+            # if the user isn't already in the room), because we only want to
+            # make sure the user isn't in the room.
+            pass
diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py
index 521ac80f9a..28fbf6ae52 100644
--- a/tests/rest/client/v1/test_rooms.py
+++ b/tests/rest/client/v1/test_rooms.py
@@ -904,3 +904,35 @@ class RoomSearchTestCase(unittest.HomeserverTestCase):
         self.assertEqual(
             context["profile_info"][self.other_user_id]["displayname"], "otheruser"
         )
+
+
+class PublicRoomsRestrictedTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        admin.register_servlets,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+
+        self.url = b"/_matrix/client/r0/publicRooms"
+
+        config = self.default_config()
+        config.restrict_public_rooms_to_local_users = True
+        self.hs = self.setup_test_homeserver(config=config)
+
+        return self.hs
+
+    def test_restricted_no_auth(self):
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+        self.assertEqual(channel.code, 401, channel.result)
+
+    def test_restricted_auth(self):
+        self.register_user("user", "pass")
+        tok = self.login("user", "pass")
+
+        request, channel = self.make_request("GET", self.url, access_token=tok)
+        self.render(request)
+        self.assertEqual(channel.code, 200, channel.result)
-- 
cgit 1.5.1


From 085ae346ace418e0fc043ac5f568f85ebf80038e Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 10 May 2019 10:52:24 +0100
Subject: Add a DUMMY stage to captcha-only registration flow

This allows the client to complete the email last which is more
natual for the user. Without this stage, if the client would
complete the recaptcha (and terms, if enabled) stages and then the
registration request would complete because you've now completed a
flow, even if you were intending to complete the flow that's the
same except has email auth at the end.

Adding a dummy auth stage to the recaptcha-only flow means it's
always unambiguous which flow the client was trying to complete.
Longer term we should think about changing the protocol so the
client explicitly says which flow it's trying to complete.

https://github.com/vector-im/riot-web/issues/9586
---
 synapse/rest/client/v2_alpha/register.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index dc3e265bcd..95578b76ae 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -345,7 +345,7 @@ class RegisterRestServlet(RestServlet):
         if self.hs.config.enable_registration_captcha:
             # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
-                flows.extend([[LoginType.RECAPTCHA]])
+                flows.extend([[LoginType.RECAPTCHA, LoginType.DUMMY]])
             # only support the email-only flow if we don't require MSISDN 3PIDs
             if not require_msisdn:
                 flows.extend([[LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]])
-- 
cgit 1.5.1


From c2bb7476c93d1f5026a4c8b715b2f70316d3a506 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 10 May 2019 11:08:01 +0100
Subject: Revert 085ae346ace418e0fc043ac5f568f85ebf80038e

Accidentally went straight to develop
---
 synapse/rest/client/v2_alpha/register.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 95578b76ae..dc3e265bcd 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -345,7 +345,7 @@ class RegisterRestServlet(RestServlet):
         if self.hs.config.enable_registration_captcha:
             # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
-                flows.extend([[LoginType.RECAPTCHA, LoginType.DUMMY]])
+                flows.extend([[LoginType.RECAPTCHA]])
             # only support the email-only flow if we don't require MSISDN 3PIDs
             if not require_msisdn:
                 flows.extend([[LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]])
-- 
cgit 1.5.1


From 2f48c4e1ae40869a1bc5dcb36557ad0a0d8a5728 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Fri, 10 May 2019 10:32:44 -0700
Subject: URL preview blacklisting fixes (#5155)

Prevents a SynapseError being raised inside of a IResolutionReceiver and instead opts to just return 0 results. This thus means that we have to lump a failed lookup and a blacklisted lookup together with the same error message, but the substitute should be generic enough to cover both cases.
---
 changelog.d/5155.misc                         |  1 +
 synapse/http/client.py                        | 45 +++++++++++++++------------
 synapse/rest/media/v1/preview_url_resource.py | 10 ++++++
 tests/rest/media/v1/test_url_preview.py       | 22 ++++++-------
 4 files changed, 47 insertions(+), 31 deletions(-)
 create mode 100644 changelog.d/5155.misc

(limited to 'synapse/rest')

diff --git a/changelog.d/5155.misc b/changelog.d/5155.misc
new file mode 100644
index 0000000000..a81dbae67a
--- /dev/null
+++ b/changelog.d/5155.misc
@@ -0,0 +1 @@
+Prevent an exception from being raised in a IResolutionReceiver and use a more generic error message for blacklisted URL previews.
\ No newline at end of file
diff --git a/synapse/http/client.py b/synapse/http/client.py
index ad454f4964..ddbfb72228 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -90,45 +90,50 @@ class IPBlacklistingResolver(object):
     def resolveHostName(self, recv, hostname, portNumber=0):
 
         r = recv()
-        d = defer.Deferred()
         addresses = []
 
-        @provider(IResolutionReceiver)
-        class EndpointReceiver(object):
-            @staticmethod
-            def resolutionBegan(resolutionInProgress):
-                pass
+        def _callback():
+            r.resolutionBegan(None)
 
-            @staticmethod
-            def addressResolved(address):
-                ip_address = IPAddress(address.host)
+            has_bad_ip = False
+            for i in addresses:
+                ip_address = IPAddress(i.host)
 
                 if check_against_blacklist(
                     ip_address, self._ip_whitelist, self._ip_blacklist
                 ):
                     logger.info(
-                        "Dropped %s from DNS resolution to %s" % (ip_address, hostname)
+                        "Dropped %s from DNS resolution to %s due to blacklist" %
+                        (ip_address, hostname)
                     )
-                    raise SynapseError(403, "IP address blocked by IP blacklist entry")
+                    has_bad_ip = True
+
+            # if we have a blacklisted IP, we'd like to raise an error to block the
+            # request, but all we can really do from here is claim that there were no
+            # valid results.
+            if not has_bad_ip:
+                for i in addresses:
+                    r.addressResolved(i)
+            r.resolutionComplete()
 
+        @provider(IResolutionReceiver)
+        class EndpointReceiver(object):
+            @staticmethod
+            def resolutionBegan(resolutionInProgress):
+                pass
+
+            @staticmethod
+            def addressResolved(address):
                 addresses.append(address)
 
             @staticmethod
             def resolutionComplete():
-                d.callback(addresses)
+                _callback()
 
         self._reactor.nameResolver.resolveHostName(
             EndpointReceiver, hostname, portNumber=portNumber
         )
 
-        def _callback(addrs):
-            r.resolutionBegan(None)
-            for i in addrs:
-                r.addressResolved(i)
-            r.resolutionComplete()
-
-        d.addCallback(_callback)
-
         return r
 
 
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index ba3ab1d37d..acf87709f2 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -31,6 +31,7 @@ from six.moves import urllib_parse as urlparse
 from canonicaljson import json
 
 from twisted.internet import defer
+from twisted.internet.error import DNSLookupError
 from twisted.web.resource import Resource
 from twisted.web.server import NOT_DONE_YET
 
@@ -328,9 +329,18 @@ class PreviewUrlResource(Resource):
                 # handler will return a SynapseError to the client instead of
                 # blank data or a 500.
                 raise
+            except DNSLookupError:
+                # DNS lookup returned no results
+                # Note: This will also be the case if one of the resolved IP
+                # addresses is blacklisted
+                raise SynapseError(
+                    502, "DNS resolution failure during URL preview generation",
+                    Codes.UNKNOWN
+                )
             except Exception as e:
                 # FIXME: pass through 404s and other error messages nicely
                 logger.warn("Error downloading %s: %r", url, e)
+
                 raise SynapseError(
                     500, "Failed to download content: %s" % (
                         traceback.format_exception_only(sys.exc_info()[0], e),
diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py
index 650ce95a6f..f696395f3c 100644
--- a/tests/rest/media/v1/test_url_preview.py
+++ b/tests/rest/media/v1/test_url_preview.py
@@ -297,12 +297,12 @@ class URLPreviewTests(unittest.HomeserverTestCase):
 
         # No requests made.
         self.assertEqual(len(self.reactor.tcpClients), 0)
-        self.assertEqual(channel.code, 403)
+        self.assertEqual(channel.code, 502)
         self.assertEqual(
             channel.json_body,
             {
                 'errcode': 'M_UNKNOWN',
-                'error': 'IP address blocked by IP blacklist entry',
+                'error': 'DNS resolution failure during URL preview generation',
             },
         )
 
@@ -318,12 +318,12 @@ class URLPreviewTests(unittest.HomeserverTestCase):
         request.render(self.preview_url)
         self.pump()
 
-        self.assertEqual(channel.code, 403)
+        self.assertEqual(channel.code, 502)
         self.assertEqual(
             channel.json_body,
             {
                 'errcode': 'M_UNKNOWN',
-                'error': 'IP address blocked by IP blacklist entry',
+                'error': 'DNS resolution failure during URL preview generation',
             },
         )
 
@@ -339,7 +339,6 @@ class URLPreviewTests(unittest.HomeserverTestCase):
 
         # No requests made.
         self.assertEqual(len(self.reactor.tcpClients), 0)
-        self.assertEqual(channel.code, 403)
         self.assertEqual(
             channel.json_body,
             {
@@ -347,6 +346,7 @@ class URLPreviewTests(unittest.HomeserverTestCase):
                 'error': 'IP address blocked by IP blacklist entry',
             },
         )
+        self.assertEqual(channel.code, 403)
 
     def test_blacklisted_ip_range_direct(self):
         """
@@ -414,12 +414,12 @@ class URLPreviewTests(unittest.HomeserverTestCase):
         )
         request.render(self.preview_url)
         self.pump()
-        self.assertEqual(channel.code, 403)
+        self.assertEqual(channel.code, 502)
         self.assertEqual(
             channel.json_body,
             {
                 'errcode': 'M_UNKNOWN',
-                'error': 'IP address blocked by IP blacklist entry',
+                'error': 'DNS resolution failure during URL preview generation',
             },
         )
 
@@ -439,12 +439,12 @@ class URLPreviewTests(unittest.HomeserverTestCase):
 
         # No requests made.
         self.assertEqual(len(self.reactor.tcpClients), 0)
-        self.assertEqual(channel.code, 403)
+        self.assertEqual(channel.code, 502)
         self.assertEqual(
             channel.json_body,
             {
                 'errcode': 'M_UNKNOWN',
-                'error': 'IP address blocked by IP blacklist entry',
+                'error': 'DNS resolution failure during URL preview generation',
             },
         )
 
@@ -460,11 +460,11 @@ class URLPreviewTests(unittest.HomeserverTestCase):
         request.render(self.preview_url)
         self.pump()
 
-        self.assertEqual(channel.code, 403)
+        self.assertEqual(channel.code, 502)
         self.assertEqual(
             channel.json_body,
             {
                 'errcode': 'M_UNKNOWN',
-                'error': 'IP address blocked by IP blacklist entry',
+                'error': 'DNS resolution failure during URL preview generation',
             },
         )
-- 
cgit 1.5.1