From 28a6ccb49c57cc686761b9e674b501b3b402e616 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 00:19:58 +0000
Subject: add registrations_require_3pid

lets homeservers specify a whitelist for 3PIDs that users are allowed to associate with.
Typically useful for stopping people from registering with non-work emails
---
 synapse/api/errors.py                    |  1 +
 synapse/config/registration.py           | 13 ++++++
 synapse/rest/client/v2_alpha/_base.py    | 22 ++++++++++
 synapse/rest/client/v2_alpha/account.py  | 14 +++++-
 synapse/rest/client/v2_alpha/register.py | 73 ++++++++++++++++++++++++++------
 5 files changed, 110 insertions(+), 13 deletions(-)

(limited to 'synapse')

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 79b35b3e7c..46b0d7b34c 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -46,6 +46,7 @@ class Codes(object):
     THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED"
     THREEPID_IN_USE = "M_THREEPID_IN_USE"
     THREEPID_NOT_FOUND = "M_THREEPID_NOT_FOUND"
+    THREEPID_DENIED = "M_THREEPID_DENIED"
     INVALID_USERNAME = "M_INVALID_USERNAME"
     SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED"
 
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index ef917fc9f2..e5e4f77872 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -31,6 +31,7 @@ class RegistrationConfig(Config):
                 strtobool(str(config["disable_registration"]))
             )
 
+        self.registrations_require_3pid = config.get("registrations_require_3pid", [])
         self.registration_shared_secret = config.get("registration_shared_secret")
 
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
@@ -52,6 +53,18 @@ class RegistrationConfig(Config):
         # Enable registration for new users.
         enable_registration: False
 
+        # Mandate that registrations require a 3PID which matches one or more
+        # of these 3PIDs.  N.B. regexp escape backslashes are doubled (once for
+        # YAML and once for the regexp itself)
+        #
+        # registrations_require_3pid:
+        #     - medium: email
+        #       pattern: ".*@matrix\\.org"
+        #     - medium: email
+        #       pattern: ".*@vector\\.im"
+        #     - medium: msisdn
+        #       pattern: "\\+44"
+
         # If set, allows registration by anyone who also has the shared
         # secret, even if registration is otherwise disabled.
         registration_shared_secret: "%(registration_shared_secret)s"
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 77434937ff..7c46ef7cab 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -60,6 +60,28 @@ def set_timeline_upper_limit(filter_json, filter_timeline_limit):
             filter_timeline_limit)
 
 
+def check_3pid_allowed(hs, medium, address):
+    # check whether the HS has whitelisted the given 3PID
+
+    allow = False
+    if hs.config.registrations_require_3pid:
+        for constraint in hs.config.registrations_require_3pid:
+            logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
+                address, medium, constraint['pattern'], constraint['medium']
+                )
+            )
+            if (
+                medium == constraint['medium'] and
+                re.match(constraint['pattern'], address)
+            ):
+                allow = True
+                break
+    else:
+        allow = True
+
+    return allow
+
+
 def interactive_auth_handler(orig):
     """Wraps an on_POST method to handle InteractiveAuthIncompleteErrors
 
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 385a3ad2ec..66221e8f00 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -26,7 +26,7 @@ from synapse.http.servlet import (
 )
 from synapse.util.async import run_on_reactor
 from synapse.util.msisdn import phone_number_to_msisdn
-from ._base import client_v2_patterns, interactive_auth_handler
+from ._base import client_v2_patterns, interactive_auth_handler, check_3pid_allowed
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +47,9 @@ class EmailPasswordRequestTokenRestServlet(RestServlet):
             'id_server', 'client_secret', 'email', 'send_attempt'
         ])
 
+        if not check_3pid_allowed(self.hs, "email", body['email']):
+            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'email', body['email']
         )
@@ -78,6 +81,9 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet):
 
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
+        if not check_3pid_allowed(self.hs, "msisdn", msisdn):
+            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'msisdn', msisdn
         )
@@ -217,6 +223,9 @@ class EmailThreepidRequestTokenRestServlet(RestServlet):
         if absent:
             raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
 
+        if not check_3pid_allowed(self.hs, "email", body['email']):
+            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'email', body['email']
         )
@@ -255,6 +264,9 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet):
 
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
+        if not check_3pid_allowed(self.hs, "msisdn", msisdn):
+            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'msisdn', msisdn
         )
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index e9d88a8895..762782c1f0 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -27,9 +27,10 @@ from synapse.http.servlet import (
 )
 from synapse.util.msisdn import phone_number_to_msisdn
 
-from ._base import client_v2_patterns, interactive_auth_handler
+from ._base import client_v2_patterns, interactive_auth_handler, check_3pid_allowed
 
 import logging
+import re
 import hmac
 from hashlib import sha1
 from synapse.util.async import run_on_reactor
@@ -70,6 +71,9 @@ class EmailRegisterRequestTokenRestServlet(RestServlet):
             'id_server', 'client_secret', 'email', 'send_attempt'
         ])
 
+        if not check_3pid_allowed(self.hs, "email", body['email']):
+            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'email', body['email']
         )
@@ -105,6 +109,9 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet):
 
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
+        if not check_3pid_allowed(self.hs, "msisdn", msisdn):
+            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'msisdn', msisdn
         )
@@ -305,31 +312,73 @@ class RegisterRestServlet(RestServlet):
         if 'x_show_msisdn' in body and body['x_show_msisdn']:
             show_msisdn = True
 
+        require_email = False
+        require_msisdn = False
+        for constraint in self.hs.config.registrations_require_3pid:
+            if constraint['medium'] == 'email':
+                require_email = True
+            elif constraint['medium'] == 'msisdn':
+                require_msisdn = True
+            else:
+                logger.warn(
+                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
+                    constraint['medium']
+                )
+
+        flows = []
         if self.hs.config.enable_registration_captcha:
-            flows = [
-                [LoginType.RECAPTCHA],
-                [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
-            ]
+            if not require_email and not require_msisdn:
+                flows.extend([[LoginType.RECAPTCHA]])
+            if require_email or not require_msisdn:
+                flows.extend([[LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]])
+
             if show_msisdn:
+                if not require_email or require_msisdn:
+                    flows.extend([[LoginType.MSISDN, LoginType.RECAPTCHA]])
                 flows.extend([
-                    [LoginType.MSISDN, LoginType.RECAPTCHA],
                     [LoginType.MSISDN, LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
                 ])
         else:
-            flows = [
-                [LoginType.DUMMY],
-                [LoginType.EMAIL_IDENTITY],
-            ]
+            if not require_email and not require_msisdn:
+                flows.extend([[LoginType.DUMMY]])
+            if require_email or not require_msisdn:
+                flows.extend([[LoginType.EMAIL_IDENTITY]])
+
             if show_msisdn:
+                if not require_email or require_msisdn:
+                    flows.extend([[LoginType.MSISDN]])
                 flows.extend([
-                    [LoginType.MSISDN],
-                    [LoginType.MSISDN, LoginType.EMAIL_IDENTITY],
+                    [LoginType.MSISDN, LoginType.EMAIL_IDENTITY]
                 ])
 
         auth_result, params, session_id = yield self.auth_handler.check_auth(
             flows, body, self.hs.get_ip_from_request(request)
         )
 
+        # doublecheck that we're not trying to register an denied 3pid.
+        # the user-facing checks should already have happened when we requested
+        # a 3PID token to validate them in /register/email/requestToken etc
+
+        for constraint in self.hs.config.registrations_require_3pid:
+            if (
+                constraint['medium'] == 'email' and
+                auth_result and LoginType.EMAIL_IDENTITY in auth_result and
+                re.match(
+                    constraint['pattern'],
+                    auth_result[LoginType.EMAIL_IDENTITY].threepid.address
+                )
+            ):
+                raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            elif (
+                constraint['medium'] == 'msisdn' and
+                auth_result and LoginType.MSISDN in auth_result and
+                re.match(
+                    constraint['pattern'],
+                    auth_result[LoginType.MSISDN].threepid.address
+                )
+            ):
+                raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+
         if registered_user_id is not None:
             logger.info(
                 "Already registered user ID %r for this session",
-- 
cgit 1.4.1


From 0af58f14ee351e7d52d7139df9218ff692764f20 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 00:33:51 +0000
Subject: fix pep8

---
 synapse/rest/client/v2_alpha/_base.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'synapse')

diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 7c46ef7cab..b286ff0d95 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -68,8 +68,7 @@ def check_3pid_allowed(hs, medium, address):
         for constraint in hs.config.registrations_require_3pid:
             logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
                 address, medium, constraint['pattern'], constraint['medium']
-                )
-            )
+            ))
             if (
                 medium == constraint['medium'] and
                 re.match(constraint['pattern'], address)
-- 
cgit 1.4.1


From 9d332e0f797e4f302a08b3708df4ac8b42b08216 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 00:53:58 +0000
Subject: fix up v1, and improve errors

---
 synapse/handlers/register.py             | 13 ++++++++++-
 synapse/rest/client/v1/register.py       | 40 +++++++++++++++++++++++---------
 synapse/rest/client/v2_alpha/account.py  | 16 +++++++++----
 synapse/rest/client/v2_alpha/register.py | 16 +++++++++----
 4 files changed, 65 insertions(+), 20 deletions(-)

(limited to 'synapse')

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 5b808beac1..157ebaf251 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -15,6 +15,7 @@
 
 """Contains functions for registering clients."""
 import logging
+import re
 
 from twisted.internet import defer
 
@@ -293,7 +294,7 @@ class RegistrationHandler(BaseHandler):
         """
 
         for c in threepidCreds:
-            logger.info("validating theeepidcred sid %s on id server %s",
+            logger.info("validating threepidcred sid %s on id server %s",
                         c['sid'], c['idServer'])
             try:
                 identity_handler = self.hs.get_handlers().identity_handler
@@ -307,6 +308,16 @@ class RegistrationHandler(BaseHandler):
             logger.info("got threepid with medium '%s' and address '%s'",
                         threepid['medium'], threepid['address'])
 
+            for constraint in self.hs.config.registrations_require_3pid:
+                if (
+                    constraint['medium'] == 'email' and
+                    threepid['medium'] == 'email' and
+                    re.match(constraint['pattern'], threepid['address'])
+                ):
+                    raise RegistrationError(
+                        403, "Third party identifier is not allowed"
+                    )
+
     @defer.inlineCallbacks
     def bind_emails(self, user_id, threepidCreds):
         """Links emails with a user ID and informs an identity server.
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index 32ed1d3ab2..f793542ad6 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -70,10 +70,24 @@ class RegisterRestServlet(ClientV1RestServlet):
         self.handlers = hs.get_handlers()
 
     def on_GET(self, request):
+
+        require_email = False
+        require_msisdn = False
+        for constraint in self.hs.config.registrations_require_3pid:
+            if constraint['medium'] == 'email':
+                require_email = True
+            elif constraint['medium'] == 'msisdn':
+                require_msisdn = True
+            else:
+                logger.warn(
+                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
+                    constraint['medium']
+                )
+
+        flows = []
         if self.hs.config.enable_registration_captcha:
-            return (
-                200,
-                {"flows": [
+            if require_email or not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.RECAPTCHA,
                         "stages": [
@@ -82,27 +96,31 @@ class RegisterRestServlet(ClientV1RestServlet):
                             LoginType.PASSWORD
                         ]
                     },
+                ])
+            if not require_email and not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.RECAPTCHA,
                         "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD]
                     }
-                ]}
-            )
+                ])
         else:
-            return (
-                200,
-                {"flows": [
+            if require_email or not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.EMAIL_IDENTITY,
                         "stages": [
                             LoginType.EMAIL_IDENTITY, LoginType.PASSWORD
                         ]
-                    },
+                    }
+                ])
+            if not require_email and not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.PASSWORD
                     }
-                ]}
-            )
+                ])
+        return (200, {"flows": flows})
 
     @defer.inlineCallbacks
     def on_POST(self, request):
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 66221e8f00..2977ad439f 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -48,7 +48,9 @@ class EmailPasswordRequestTokenRestServlet(RestServlet):
         ])
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'email', body['email']
@@ -82,7 +84,9 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet):
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'msisdn', msisdn
@@ -224,7 +228,9 @@ class EmailThreepidRequestTokenRestServlet(RestServlet):
             raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'email', body['email']
@@ -265,7 +271,9 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet):
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'msisdn', msisdn
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 762782c1f0..898d8b133a 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -72,7 +72,9 @@ class EmailRegisterRequestTokenRestServlet(RestServlet):
         ])
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'email', body['email']
@@ -110,7 +112,9 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet):
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'msisdn', msisdn
@@ -368,7 +372,9 @@ class RegisterRestServlet(RestServlet):
                     auth_result[LoginType.EMAIL_IDENTITY].threepid.address
                 )
             ):
-                raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+                raise SynapseError(
+                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                )
             elif (
                 constraint['medium'] == 'msisdn' and
                 auth_result and LoginType.MSISDN in auth_result and
@@ -377,7 +383,9 @@ class RegisterRestServlet(RestServlet):
                     auth_result[LoginType.MSISDN].threepid.address
                 )
             ):
-                raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+                raise SynapseError(
+                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                )
 
         if registered_user_id is not None:
             logger.info(
-- 
cgit 1.4.1


From 447f4f0d5f136dcadd5fdc286ded2d6e24a3f686 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 15:33:55 +0000
Subject: rewrite based on PR feedback:

 * [ ] split config options into allowed_local_3pids and registrations_require_3pid
 * [ ] simplify and comment logic for picking registration flows
 * [ ] fix docstring and move check_3pid_allowed into a new util module
 * [ ] use check_3pid_allowed everywhere

@erikjohnston PTAL
---
 synapse/config/registration.py           | 12 +++--
 synapse/handlers/register.py             | 15 +++----
 synapse/rest/client/v1/register.py       | 20 +++------
 synapse/rest/client/v2_alpha/_base.py    | 21 ---------
 synapse/rest/client/v2_alpha/account.py  |  3 +-
 synapse/rest/client/v2_alpha/register.py | 75 +++++++++++++++-----------------
 synapse/util/threepids.py                | 45 +++++++++++++++++++
 7 files changed, 102 insertions(+), 89 deletions(-)
 create mode 100644 synapse/util/threepids.py

(limited to 'synapse')

diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index e5e4f77872..336959094b 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -32,6 +32,7 @@ class RegistrationConfig(Config):
             )
 
         self.registrations_require_3pid = config.get("registrations_require_3pid", [])
+        self.allowed_local_3pids = config.get("allowed_local_3pids", [])
         self.registration_shared_secret = config.get("registration_shared_secret")
 
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
@@ -53,11 +54,16 @@ class RegistrationConfig(Config):
         # Enable registration for new users.
         enable_registration: False
 
-        # Mandate that registrations require a 3PID which matches one or more
-        # of these 3PIDs.  N.B. regexp escape backslashes are doubled (once for
-        # YAML and once for the regexp itself)
+        # The user must provide all of the below types of 3PID when registering.
         #
         # registrations_require_3pid:
+        #     - email
+        #     - msisdn
+
+        # Mandate that users are only allowed to associate certain formats of
+        # 3PIDs with accounts on this server.
+        #
+        # allowed_local_3pids:
         #     - medium: email
         #       pattern: ".*@matrix\\.org"
         #     - medium: email
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 157ebaf251..9021d4d57f 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -15,7 +15,6 @@
 
 """Contains functions for registering clients."""
 import logging
-import re
 
 from twisted.internet import defer
 
@@ -26,6 +25,7 @@ from synapse.http.client import CaptchaServerHttpClient
 from synapse import types
 from synapse.types import UserID
 from synapse.util.async import run_on_reactor
+from synapse.util.threepids import check_3pid_allowed
 from ._base import BaseHandler
 
 logger = logging.getLogger(__name__)
@@ -308,15 +308,10 @@ class RegistrationHandler(BaseHandler):
             logger.info("got threepid with medium '%s' and address '%s'",
                         threepid['medium'], threepid['address'])
 
-            for constraint in self.hs.config.registrations_require_3pid:
-                if (
-                    constraint['medium'] == 'email' and
-                    threepid['medium'] == 'email' and
-                    re.match(constraint['pattern'], threepid['address'])
-                ):
-                    raise RegistrationError(
-                        403, "Third party identifier is not allowed"
-                    )
+            if not check_3pid_allowed(self.hs, threepid['medium'], threepid['address']):
+                raise RegistrationError(
+                    403, "Third party identifier is not allowed"
+                )
 
     @defer.inlineCallbacks
     def bind_emails(self, user_id, threepidCreds):
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index f793542ad6..5c5fa8f7ab 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -71,22 +71,13 @@ class RegisterRestServlet(ClientV1RestServlet):
 
     def on_GET(self, request):
 
-        require_email = False
-        require_msisdn = False
-        for constraint in self.hs.config.registrations_require_3pid:
-            if constraint['medium'] == 'email':
-                require_email = True
-            elif constraint['medium'] == 'msisdn':
-                require_msisdn = True
-            else:
-                logger.warn(
-                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
-                    constraint['medium']
-                )
+        require_email = 'email' in self.hs.config.registrations_require_3pid
+        require_msisdn = 'msisdn' in self.hs.config.registrations_require_3pid
 
         flows = []
         if self.hs.config.enable_registration_captcha:
-            if require_email or not require_msisdn:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
+            if not require_msisdn:
                 flows.extend([
                     {
                         "type": LoginType.RECAPTCHA,
@@ -97,6 +88,7 @@ class RegisterRestServlet(ClientV1RestServlet):
                         ]
                     },
                 ])
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([
                     {
@@ -105,6 +97,7 @@ class RegisterRestServlet(ClientV1RestServlet):
                     }
                 ])
         else:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
             if require_email or not require_msisdn:
                 flows.extend([
                     {
@@ -114,6 +107,7 @@ class RegisterRestServlet(ClientV1RestServlet):
                         ]
                     }
                 ])
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([
                     {
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index b286ff0d95..77434937ff 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -60,27 +60,6 @@ def set_timeline_upper_limit(filter_json, filter_timeline_limit):
             filter_timeline_limit)
 
 
-def check_3pid_allowed(hs, medium, address):
-    # check whether the HS has whitelisted the given 3PID
-
-    allow = False
-    if hs.config.registrations_require_3pid:
-        for constraint in hs.config.registrations_require_3pid:
-            logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
-                address, medium, constraint['pattern'], constraint['medium']
-            ))
-            if (
-                medium == constraint['medium'] and
-                re.match(constraint['pattern'], address)
-            ):
-                allow = True
-                break
-    else:
-        allow = True
-
-    return allow
-
-
 def interactive_auth_handler(orig):
     """Wraps an on_POST method to handle InteractiveAuthIncompleteErrors
 
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 2977ad439f..514bb37da1 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -26,7 +26,8 @@ from synapse.http.servlet import (
 )
 from synapse.util.async import run_on_reactor
 from synapse.util.msisdn import phone_number_to_msisdn
-from ._base import client_v2_patterns, interactive_auth_handler, check_3pid_allowed
+from synapse.util.threepids import check_3pid_allowed
+from ._base import client_v2_patterns, interactive_auth_handler
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 898d8b133a..c3479e29de 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -26,11 +26,11 @@ from synapse.http.servlet import (
     RestServlet, parse_json_object_from_request, assert_params_in_request, parse_string
 )
 from synapse.util.msisdn import phone_number_to_msisdn
+from synapse.util.threepids import check_3pid_allowed
 
-from ._base import client_v2_patterns, interactive_auth_handler, check_3pid_allowed
+from ._base import client_v2_patterns, interactive_auth_handler
 
 import logging
-import re
 import hmac
 from hashlib import sha1
 from synapse.util.async import run_on_reactor
@@ -316,41 +316,41 @@ class RegisterRestServlet(RestServlet):
         if 'x_show_msisdn' in body and body['x_show_msisdn']:
             show_msisdn = True
 
-        require_email = False
-        require_msisdn = False
-        for constraint in self.hs.config.registrations_require_3pid:
-            if constraint['medium'] == 'email':
-                require_email = True
-            elif constraint['medium'] == 'msisdn':
-                require_msisdn = True
-            else:
-                logger.warn(
-                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
-                    constraint['medium']
-                )
+        # FIXME: need a better error than "no auth flow found" for scenarios
+        # where we required 3PID for registration but the user didn't give one
+        require_email = 'email' in self.hs.config.registrations_require_3pid
+        require_msisdn = 'msisdn' in self.hs.config.registrations_require_3pid
 
         flows = []
         if self.hs.config.enable_registration_captcha:
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([[LoginType.RECAPTCHA]])
-            if require_email or not require_msisdn:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
+            if not require_msisdn:
                 flows.extend([[LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]])
 
             if show_msisdn:
-                if not require_email or require_msisdn:
+                # only support the MSISDN-only flow if we don't require email 3PIDs
+                if not require_email:
                     flows.extend([[LoginType.MSISDN, LoginType.RECAPTCHA]])
+                # always let users provide both MSISDN & email
                 flows.extend([
                     [LoginType.MSISDN, LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
                 ])
         else:
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([[LoginType.DUMMY]])
-            if require_email or not require_msisdn:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
+            if not require_msisdn:
                 flows.extend([[LoginType.EMAIL_IDENTITY]])
 
             if show_msisdn:
+                # only support the MSISDN-only flow if we don't require email 3PIDs
                 if not require_email or require_msisdn:
                     flows.extend([[LoginType.MSISDN]])
+                # always let users provide both MSISDN & email
                 flows.extend([
                     [LoginType.MSISDN, LoginType.EMAIL_IDENTITY]
                 ])
@@ -359,30 +359,23 @@ class RegisterRestServlet(RestServlet):
             flows, body, self.hs.get_ip_from_request(request)
         )
 
-        # doublecheck that we're not trying to register an denied 3pid.
-        # the user-facing checks should already have happened when we requested
-        # a 3PID token to validate them in /register/email/requestToken etc
-
-        for constraint in self.hs.config.registrations_require_3pid:
-            if (
-                constraint['medium'] == 'email' and
-                auth_result and LoginType.EMAIL_IDENTITY in auth_result and
-                re.match(
-                    constraint['pattern'],
-                    auth_result[LoginType.EMAIL_IDENTITY].threepid.address
-                )
-            ):
-                raise SynapseError(
-                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
-                )
-            elif (
-                constraint['medium'] == 'msisdn' and
-                auth_result and LoginType.MSISDN in auth_result and
-                re.match(
-                    constraint['pattern'],
-                    auth_result[LoginType.MSISDN].threepid.address
-                )
-            ):
+        # Check that we're not trying to register a denied 3pid.
+        #
+        # the user-facing checks will probably already have happened in
+        # /register/email/requestToken when we requested a 3pid, but that's not
+        # guaranteed.
+
+        if (
+            auth_result and
+            (
+                LoginType.EMAIL_IDENTITY in auth_result or
+                LoginType.EMAIL_MSISDN in auth_result
+            )
+        ):
+            medium = auth_result[LoginType.EMAIL_IDENTITY].threepid['medium']
+            address = auth_result[LoginType.EMAIL_IDENTITY].threepid['address']
+
+            if not check_3pid_allowed(self.hs, medium, address):
                 raise SynapseError(
                     403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
                 )
diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py
new file mode 100644
index 0000000000..e921b97796
--- /dev/null
+++ b/synapse/util/threepids.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def check_3pid_allowed(hs, medium, address):
+    """Checks whether a given format of 3PID is allowed to be used on this HS
+
+    Args:
+        hs (synapse.server.HomeServer): server
+        medium (str): 3pid medium - e.g. email, msisdn
+        address (str): address within that medium (e.g. "wotan@matrix.org")
+            msisdns need to first have been canonicalised
+    """
+
+    if hs.config.allowed_local_3pids:
+        for constraint in hs.config.allowed_local_3pids:
+            logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
+                address, medium, constraint['pattern'], constraint['medium']
+            ))
+            if (
+                medium == constraint['medium'] and
+                re.match(constraint['pattern'], address)
+            ):
+                return True
+    else:
+        return True
+
+    return False
-- 
cgit 1.4.1


From 293380bef761b67479a90d2837bbf6dfa5a70a90 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 15:38:53 +0000
Subject: trailing commas

---
 synapse/rest/client/v2_alpha/account.py  | 8 ++++----
 synapse/rest/client/v2_alpha/register.py | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'synapse')

diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 514bb37da1..30523995af 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -50,7 +50,7 @@ class EmailPasswordRequestTokenRestServlet(RestServlet):
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
@@ -86,7 +86,7 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet):
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
@@ -230,7 +230,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet):
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
@@ -273,7 +273,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet):
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index c3479e29de..bf68e34a58 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -73,7 +73,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet):
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
@@ -113,7 +113,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet):
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
             raise SynapseError(
-                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
             )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
@@ -377,7 +377,7 @@ class RegisterRestServlet(RestServlet):
 
             if not check_3pid_allowed(self.hs, medium, address):
                 raise SynapseError(
-                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
                 )
 
         if registered_user_id is not None:
-- 
cgit 1.4.1


From 8fe253f19b1c61c38111948cce00a7d260d2925a Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 18:23:45 +0000
Subject: fix PR nitpicking

---
 synapse/util/threepids.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'synapse')

diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py
index e921b97796..75efa0117b 100644
--- a/synapse/util/threepids.py
+++ b/synapse/util/threepids.py
@@ -27,13 +27,16 @@ def check_3pid_allowed(hs, medium, address):
         medium (str): 3pid medium - e.g. email, msisdn
         address (str): address within that medium (e.g. "wotan@matrix.org")
             msisdns need to first have been canonicalised
+    Returns:
+        bool: whether the 3PID medium/address is allowed to be added to this HS
     """
 
     if hs.config.allowed_local_3pids:
         for constraint in hs.config.allowed_local_3pids:
-            logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
-                address, medium, constraint['pattern'], constraint['medium']
-            ))
+            logger.debug(
+                "Checking 3PID %s (%s) against %s (%s)",
+                address, medium, constraint['pattern'], constraint['medium'],
+            )
             if (
                 medium == constraint['medium'] and
                 re.match(constraint['pattern'], address)
-- 
cgit 1.4.1


From 62d7d66ae5592175bb35a2a8d2f69b1924d6e1f2 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 18:23:56 +0000
Subject: oops, check all login types

---
 synapse/rest/client/v2_alpha/register.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

(limited to 'synapse')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index bf68e34a58..4e73f9a40a 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -365,20 +365,17 @@ class RegisterRestServlet(RestServlet):
         # /register/email/requestToken when we requested a 3pid, but that's not
         # guaranteed.
 
-        if (
-            auth_result and
-            (
-                LoginType.EMAIL_IDENTITY in auth_result or
-                LoginType.EMAIL_MSISDN in auth_result
-            )
-        ):
-            medium = auth_result[LoginType.EMAIL_IDENTITY].threepid['medium']
-            address = auth_result[LoginType.EMAIL_IDENTITY].threepid['address']
-
-            if not check_3pid_allowed(self.hs, medium, address):
-                raise SynapseError(
-                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED,
-                )
+        if auth_result:
+            for login_type in [LoginType.EMAIL_IDENTITY, LoginType.EMAIL_MSISDN]:
+                if login_type in auth_result:
+                    medium = auth_result[login_type].threepid['medium']
+                    address = auth_result[login_type].threepid['address']
+
+                    if not check_3pid_allowed(self.hs, medium, address):
+                        raise SynapseError(
+                            403, "Third party identifier is not allowed",
+                            Codes.THREEPID_DENIED,
+                        )
 
         if registered_user_id is not None:
             logger.info(
-- 
cgit 1.4.1


From 49fce046243ae3e2c38ac6ac39001172667b8dfa Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 19:55:33 +0000
Subject: fix typo (thanks sytest)

---
 synapse/rest/client/v2_alpha/register.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 4e73f9a40a..3abfe35479 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -366,7 +366,7 @@ class RegisterRestServlet(RestServlet):
         # guaranteed.
 
         if auth_result:
-            for login_type in [LoginType.EMAIL_IDENTITY, LoginType.EMAIL_MSISDN]:
+            for login_type in [LoginType.EMAIL_IDENTITY, LoginType.MSISDN]:
                 if login_type in auth_result:
                     medium = auth_result[login_type].threepid['medium']
                     address = auth_result[login_type].threepid['address']
-- 
cgit 1.4.1


From 87b7d727605c8e122adb768b7487dfcae830593f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 19 Jan 2018 23:51:04 +0000
Subject: Add some comments about the reactor tick time metric

---
 synapse/metrics/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 2265e6e8d6..e0cfb7d08f 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -146,10 +146,15 @@ def runUntilCurrentTimer(func):
             num_pending += 1
 
         num_pending += len(reactor.threadCallQueue)
-
         start = time.time() * 1000
         ret = func(*args, **kwargs)
         end = time.time() * 1000
+
+        # record the amount of wallclock time spent running pending calls.
+        # This is a proxy for the actual amount of time between reactor polls,
+        # since about 25% of time is actually spent running things triggered by
+        # I/O events, but that is harder to capture without rewriting half the
+        # reactor.
         tick_time.inc_by(end - start)
         pending_calls_metric.inc_by(num_pending)
 
-- 
cgit 1.4.1


From 2c8526cac795fa2aa795e1a1aaae2ffb2558824d Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Sat, 20 Jan 2018 00:55:44 +0000
Subject: Use a connection pool for the SimpleHttpClient

In particular I hope this will help the pusher, which makes many requests to
sygnal, and is currently negotiating SSL for each one.
---
 synapse/http/client.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/http/client.py b/synapse/http/client.py
index 4abb479ae3..930d713013 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -30,6 +30,7 @@ from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.web.client import (
     BrowserLikeRedirectAgent, ContentDecoderAgent, GzipDecoder, Agent,
     readBody, PartialDownloadError,
+    HTTPConnectionPool,
 )
 from twisted.web.client import FileBodyProducer as TwistedFileBodyProducer
 from twisted.web.http import PotentialDataLoss
@@ -64,13 +65,19 @@ class SimpleHttpClient(object):
     """
     def __init__(self, hs):
         self.hs = hs
+
+        pool = HTTPConnectionPool(reactor)
+        pool.maxPersistentPerHost = 5
+        pool.cachedConnectionTimeout = 2 * 60
+
         # The default context factory in Twisted 14.0.0 (which we require) is
         # BrowserLikePolicyForHTTPS which will do regular cert validation
         # 'like a browser'
         self.agent = Agent(
             reactor,
             connectTimeout=15,
-            contextFactory=hs.get_http_client_context_factory()
+            contextFactory=hs.get_http_client_context_factory(),
+            pool=pool,
         )
         self.user_agent = hs.version_string
         self.clock = hs.get_clock()
-- 
cgit 1.4.1


From ab9f844aaf3662a64dbc4c56077e9fa37bc7d5d0 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Mon, 22 Jan 2018 19:11:18 +0100
Subject: Add federation_domain_whitelist option (#2820)

Add federation_domain_whitelist

gives a way to restrict which domains your HS is allowed to federate with.
useful mainly for gracefully preventing a private but internet-connected HS from trying to federate to the wider public Matrix network
---
 synapse/api/errors.py                      | 26 ++++++++++++++++++++++++++
 synapse/config/server.py                   | 22 ++++++++++++++++++++++
 synapse/federation/federation_client.py    |  5 ++++-
 synapse/federation/transaction_queue.py    |  4 +++-
 synapse/federation/transport/client.py     |  3 +++
 synapse/federation/transport/server.py     |  9 ++++++++-
 synapse/handlers/device.py                 |  4 ++++
 synapse/handlers/e2e_keys.py               |  8 +++++++-
 synapse/handlers/federation.py             |  4 ++++
 synapse/http/matrixfederationclient.py     | 28 +++++++++++++++++++++++++++-
 synapse/rest/key/v2/remote_key_resource.py |  8 ++++++++
 synapse/rest/media/v1/media_repository.py  | 19 +++++++++++++++++--
 synapse/util/retryutils.py                 | 12 ++++++++++++
 tests/utils.py                             |  1 +
 14 files changed, 146 insertions(+), 7 deletions(-)

(limited to 'synapse')

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 46b0d7b34c..aa15f73f36 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -141,6 +141,32 @@ class RegistrationError(SynapseError):
     pass
 
 
+class FederationDeniedError(SynapseError):
+    """An error raised when the server tries to federate with a server which
+    is not on its federation whitelist.
+
+    Attributes:
+        destination (str): The destination which has been denied
+    """
+
+    def __init__(self, destination):
+        """Raised by federation client or server to indicate that we are
+        are deliberately not attempting to contact a given server because it is
+        not on our federation whitelist.
+
+        Args:
+            destination (str): the domain in question
+        """
+
+        self.destination = destination
+
+        super(FederationDeniedError, self).__init__(
+            code=403,
+            msg="Federation denied with %s." % (self.destination,),
+            errcode=Codes.FORBIDDEN,
+        )
+
+
 class InteractiveAuthIncompleteError(Exception):
     """An error raised when UI auth is not yet complete
 
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 436dd8a6fe..8f0b6d1f28 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -55,6 +55,17 @@ class ServerConfig(Config):
             "block_non_admin_invites", False,
         )
 
+        # FIXME: federation_domain_whitelist needs sytests
+        self.federation_domain_whitelist = None
+        federation_domain_whitelist = config.get(
+            "federation_domain_whitelist", None
+        )
+        # turn the whitelist into a hash for speed of lookup
+        if federation_domain_whitelist is not None:
+            self.federation_domain_whitelist = {}
+            for domain in federation_domain_whitelist:
+                self.federation_domain_whitelist[domain] = True
+
         if self.public_baseurl is not None:
             if self.public_baseurl[-1] != '/':
                 self.public_baseurl += '/'
@@ -210,6 +221,17 @@ class ServerConfig(Config):
         # (except those sent by local server admins). The default is False.
         # block_non_admin_invites: True
 
+        # Restrict federation to the following whitelist of domains.
+        # N.B. we recommend also firewalling your federation listener to limit
+        # inbound federation traffic as early as possible, rather than relying
+        # purely on this application-layer restriction.  If not specified, the
+        # default is to whitelist everything.
+        #
+        # federation_domain_whitelist:
+        #  - lon.example.com
+        #  - nyc.example.com
+        #  - syd.example.com
+
         # List of ports that Synapse should listen on, their purpose and their
         # configuration.
         listeners:
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index b1fe03f702..813907f7f2 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -23,7 +23,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import Membership
 from synapse.api.errors import (
-    CodeMessageException, HttpResponseException, SynapseError,
+    CodeMessageException, HttpResponseException, SynapseError, FederationDeniedError
 )
 from synapse.events import builder
 from synapse.federation.federation_base import (
@@ -266,6 +266,9 @@ class FederationClient(FederationBase):
             except NotRetryingDestination as e:
                 logger.info(e.message)
                 continue
+            except FederationDeniedError as e:
+                logger.info(e.message)
+                continue
             except Exception as e:
                 pdu_attempts[destination] = now
 
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 9d39f46583..a141ec9953 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -19,7 +19,7 @@ from twisted.internet import defer
 from .persistence import TransactionActions
 from .units import Transaction, Edu
 
-from synapse.api.errors import HttpResponseException
+from synapse.api.errors import HttpResponseException, FederationDeniedError
 from synapse.util import logcontext, PreserveLoggingContext
 from synapse.util.async import run_on_reactor
 from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
@@ -490,6 +490,8 @@ class TransactionQueue(object):
                     (e.retry_last_ts + e.retry_interval) / 1000.0
                 ),
             )
+        except FederationDeniedError as e:
+            logger.info(e)
         except Exception as e:
             logger.warn(
                 "TX [%s] Failed to send transaction: %s",
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 1f3ce238f6..5488e82985 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -212,6 +212,9 @@ class TransportLayerClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if the remote destination
+            is not in our federation whitelist
         """
         valid_memberships = {Membership.JOIN, Membership.LEAVE}
         if membership not in valid_memberships:
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 2b02b021ec..06c16ba4fa 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.urls import FEDERATION_PREFIX as PREFIX
-from synapse.api.errors import Codes, SynapseError
+from synapse.api.errors import Codes, SynapseError, FederationDeniedError
 from synapse.http.server import JsonResource
 from synapse.http.servlet import (
     parse_json_object_from_request, parse_integer_from_args, parse_string_from_args,
@@ -81,6 +81,7 @@ class Authenticator(object):
         self.keyring = hs.get_keyring()
         self.server_name = hs.hostname
         self.store = hs.get_datastore()
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
 
     # A method just so we can pass 'self' as the authenticator to the Servlets
     @defer.inlineCallbacks
@@ -92,6 +93,12 @@ class Authenticator(object):
             "signatures": {},
         }
 
+        if (
+            self.federation_domain_whitelist is not None and
+            self.server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(self.server_name)
+
         if content is not None:
             json_request["content"] = content
 
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 2152efc692..0e83453851 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 from synapse.api import errors
 from synapse.api.constants import EventTypes
+from synapse.api.errors import FederationDeniedError
 from synapse.util import stringutils
 from synapse.util.async import Linearizer
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -513,6 +514,9 @@ class DeviceListEduUpdater(object):
                     # This makes it more likely that the device lists will
                     # eventually become consistent.
                     return
+                except FederationDeniedError as e:
+                    logger.info(e)
+                    return
                 except Exception:
                     # TODO: Remember that we are now out of sync and try again
                     # later
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 5af8abf66b..9aa95f89e6 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -19,7 +19,9 @@ import logging
 from canonicaljson import encode_canonical_json
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError, CodeMessageException
+from synapse.api.errors import (
+    SynapseError, CodeMessageException, FederationDeniedError,
+)
 from synapse.types import get_domain_from_id, UserID
 from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
 from synapse.util.retryutils import NotRetryingDestination
@@ -140,6 +142,10 @@ class E2eKeysHandler(object):
                 failures[destination] = {
                     "status": 503, "message": "Not ready for retry",
                 }
+            except FederationDeniedError as e:
+                failures[destination] = {
+                    "status": 403, "message": "Federation Denied",
+                }
             except Exception as e:
                 # include ConnectionRefused and other errors
                 failures[destination] = {
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ac70730885..677532c87b 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -22,6 +22,7 @@ from ._base import BaseHandler
 
 from synapse.api.errors import (
     AuthError, FederationError, StoreError, CodeMessageException, SynapseError,
+    FederationDeniedError,
 )
 from synapse.api.constants import EventTypes, Membership, RejectedReason
 from synapse.events.validator import EventValidator
@@ -782,6 +783,9 @@ class FederationHandler(BaseHandler):
                 except NotRetryingDestination as e:
                     logger.info(e.message)
                     continue
+                except FederationDeniedError as e:
+                    logger.info(e)
+                    continue
                 except Exception as e:
                     logger.exception(
                         "Failed to backfill from %s because %s",
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 833496b72d..9145405cb0 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -27,7 +27,7 @@ import synapse.metrics
 from canonicaljson import encode_canonical_json
 
 from synapse.api.errors import (
-    SynapseError, Codes, HttpResponseException,
+    SynapseError, Codes, HttpResponseException, FederationDeniedError,
 )
 
 from signedjson.sign import sign_json
@@ -123,11 +123,22 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``HTTPRequestException``: if we get an HTTP response
                 code >= 300.
+
             Fails with ``NotRetryingDestination`` if we are not yet ready
                 to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+                is not on our federation whitelist
+
             (May also fail with plenty of other Exceptions for things like DNS
                 failures, connection failures, SSL failures.)
         """
+        if (
+            self.hs.config.federation_domain_whitelist and
+            destination not in self.hs.config.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(destination)
+
         limiter = yield synapse.util.retryutils.get_retry_limiter(
             destination,
             self.clock,
@@ -308,6 +319,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         if not json_data_callback:
@@ -368,6 +382,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         def body_callback(method, url_bytes, headers_dict):
@@ -422,6 +439,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
         logger.debug("get_json args: %s", args)
 
@@ -475,6 +495,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         response = yield self._request(
@@ -518,6 +541,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         encoded_args = {}
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index cc2842aa72..17e6079cba 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -93,6 +93,7 @@ class RemoteKey(Resource):
         self.store = hs.get_datastore()
         self.version_string = hs.version_string
         self.clock = hs.get_clock()
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
 
     def render_GET(self, request):
         self.async_render_GET(request)
@@ -137,6 +138,13 @@ class RemoteKey(Resource):
         logger.info("Handling query for keys %r", query)
         store_queries = []
         for server_name, key_ids in query.items():
+            if (
+                self.federation_domain_whitelist is not None and
+                server_name not in self.federation_domain_whitelist
+            ):
+                logger.debug("Federation denied with %s", server_name)
+                continue
+
             if not key_ids:
                 key_ids = (None,)
             for key_id in key_ids:
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 4f56bcf577..485db8577a 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -32,8 +32,9 @@ from .media_storage import MediaStorage
 
 from synapse.http.matrixfederationclient import MatrixFederationHttpClient
 from synapse.util.stringutils import random_string
-from synapse.api.errors import SynapseError, HttpResponseException, \
-    NotFoundError
+from synapse.api.errors import (
+    SynapseError, HttpResponseException, NotFoundError, FederationDeniedError,
+)
 
 from synapse.util.async import Linearizer
 from synapse.util.stringutils import is_ascii
@@ -75,6 +76,8 @@ class MediaRepository(object):
         self.recently_accessed_remotes = set()
         self.recently_accessed_locals = set()
 
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
+
         # List of StorageProviders where we should search for media and
         # potentially upload to.
         storage_providers = []
@@ -216,6 +219,12 @@ class MediaRepository(object):
             Deferred: Resolves once a response has successfully been written
                 to request
         """
+        if (
+            self.federation_domain_whitelist is not None and
+            server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(server_name)
+
         self.mark_recently_accessed(server_name, media_id)
 
         # We linearize here to ensure that we don't try and download remote
@@ -250,6 +259,12 @@ class MediaRepository(object):
         Returns:
             Deferred[dict]: The media_info of the file
         """
+        if (
+            self.federation_domain_whitelist is not None and
+            server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(server_name)
+
         # We linearize here to ensure that we don't try and download remote
         # media multiple times concurrently
         key = (server_name, media_id)
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 1adedbb361..47b0bb5eb3 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -26,6 +26,18 @@ logger = logging.getLogger(__name__)
 
 class NotRetryingDestination(Exception):
     def __init__(self, retry_last_ts, retry_interval, destination):
+        """Raised by the limiter (and federation client) to indicate that we are
+        are deliberately not attempting to contact a given server.
+
+        Args:
+            retry_last_ts (int): the unix ts in milliseconds of our last attempt
+                to contact the server.  0 indicates that the last attempt was
+                successful or that we've never actually attempted to connect.
+            retry_interval (int): the time in milliseconds to wait until the next
+                attempt.
+            destination (str): the domain in question
+        """
+
         msg = "Not retrying server %s." % (destination,)
         super(NotRetryingDestination, self).__init__(msg)
 
diff --git a/tests/utils.py b/tests/utils.py
index 44e5f75093..3116047892 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -57,6 +57,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.worker_app = None
         config.email_enable_notifs = False
         config.block_non_admin_invites = False
+        config.federation_domain_whitelist = None
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
-- 
cgit 1.4.1


From 93efd7eb04601ee103f176819283e0298c660adc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 22 Jan 2018 18:14:10 +0000
Subject: logging and debug for http pusher

---
 synapse/push/httppusher.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

(limited to 'synapse')

diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index c16f61452c..f2517f39a8 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -13,21 +13,30 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from synapse.push import PusherConfigException
+import logging
 
 from twisted.internet import defer, reactor
 from twisted.internet.error import AlreadyCalled, AlreadyCancelled
 
-import logging
 import push_rule_evaluator
 import push_tools
-
+import synapse
+from synapse.push import PusherConfigException
 from synapse.util.logcontext import LoggingContext
 from synapse.util.metrics import Measure
 
 logger = logging.getLogger(__name__)
 
+metrics = synapse.metrics.get_metrics_for(__name__)
+
+http_push_processed_counter = metrics.register_counter(
+    "http_pushes_processed",
+)
+
+http_push_failed_counter = metrics.register_counter(
+    "http_pushes_failed",
+)
+
 
 class HttpPusher(object):
     INITIAL_BACKOFF_SEC = 1  # in seconds because that's what Twisted takes
@@ -152,9 +161,15 @@ class HttpPusher(object):
             self.user_id, self.last_stream_ordering, self.max_stream_ordering
         )
 
+        logger.info(
+            "Processing %i unprocessed push actions starting at stream_ordering %i",
+            len(unprocessed), self.last_stream_ordering,
+        )
+
         for push_action in unprocessed:
             processed = yield self._process_one(push_action)
             if processed:
+                http_push_processed_counter.inc()
                 self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC
                 self.last_stream_ordering = push_action['stream_ordering']
                 yield self.store.update_pusher_last_stream_ordering_and_success(
@@ -169,6 +184,7 @@ class HttpPusher(object):
                         self.failing_since
                     )
             else:
+                http_push_failed_counter.inc()
                 if not self.failing_since:
                     self.failing_since = self.clock.time_msec()
                     yield self.store.update_pusher_failing_since(
-- 
cgit 1.4.1


From 4528dd2443f4dc9e737bf4eeccedfb8807a1ea2c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 22 Jan 2018 20:15:42 +0000
Subject: Fix logging and add user_id

---
 synapse/push/httppusher.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'synapse')

diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index f2517f39a8..4a03af5b21 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -162,8 +162,9 @@ class HttpPusher(object):
         )
 
         logger.info(
-            "Processing %i unprocessed push actions starting at stream_ordering %i",
-            len(unprocessed), self.last_stream_ordering,
+            "Processing %i unprocessed push actions for %s starting at "
+            "stream_ordering %s",
+            len(unprocessed), self.user_id, self.last_stream_ordering,
         )
 
         for push_action in unprocessed:
-- 
cgit 1.4.1


From d32385336f2edf2c98ae9eb560bad9402860d7cc Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Tue, 23 Jan 2018 09:59:06 +0100
Subject: add ?ts massaging for ASes (#2754)

blindly implement ?ts for AS. untested
---
 synapse/rest/client/v1/room.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

(limited to 'synapse')

diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 682a0af9fc..867ec8602c 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -195,15 +195,20 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
         requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         content = parse_json_object_from_request(request)
 
+        event_dict = {
+            "type": event_type,
+            "content": content,
+            "room_id": room_id,
+            "sender": requester.user.to_string(),
+        }
+
+        if 'ts' in request.args and requester.app_service:
+            event_dict['origin_server_ts'] = parse_integer(request, "ts", 0)
+
         msg_handler = self.handlers.message_handler
         event = yield msg_handler.create_and_send_nonmember_event(
             requester,
-            {
-                "type": event_type,
-                "content": content,
-                "room_id": room_id,
-                "sender": requester.user.to_string(),
-            },
+            event_dict,
             txn_id=txn_id,
         )
 
-- 
cgit 1.4.1


From 9a72b70630e111639243c6ab7867c4d4b970e2df Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 24 Jan 2018 11:07:24 +0100
Subject: fix thinko on 3pid whitelisting

---
 synapse/rest/client/v2_alpha/register.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 3abfe35479..c6f4680a76 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -368,8 +368,8 @@ class RegisterRestServlet(RestServlet):
         if auth_result:
             for login_type in [LoginType.EMAIL_IDENTITY, LoginType.MSISDN]:
                 if login_type in auth_result:
-                    medium = auth_result[login_type].threepid['medium']
-                    address = auth_result[login_type].threepid['address']
+                    medium = auth_result[login_type]['medium']
+                    address = auth_result[login_type]['address']
 
                     if not check_3pid_allowed(self.hs, medium, address):
                         raise SynapseError(
-- 
cgit 1.4.1


From 349c7399663b5fce856995a3a901019f5d210cc4 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:28:44 +0000
Subject: synapse 500s on a call to publicRooms in the case where the number of
 public rooms is zero, the specific cause is due to xrange trying to use a
 step value of zero, but if the total room number really is zero then it makes
 sense to just bail and save the extra processing

---
 synapse/handlers/room_list.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'synapse')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index bb40075387..ae5db4d2c5 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -186,6 +186,11 @@ class RoomListHandler(BaseHandler):
         logger.info("After sorting and filtering, %i rooms remain",
                     len(rooms_to_scan))
 
+        #bail if no rooms to work on
+        if len(rooms_to_scan) == 0:
+            defer.returnValue([])
+
+
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
         #
-- 
cgit 1.4.1


From d02e43b15f6b9b24ffe5e0c0d696f8fd71fc8af3 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:29:46 +0000
Subject: remove white space

---
 synapse/handlers/room_list.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index ae5db4d2c5..9f8173644a 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -190,7 +190,6 @@ class RoomListHandler(BaseHandler):
         if len(rooms_to_scan) == 0:
             defer.returnValue([])
 
-
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
         #
-- 
cgit 1.4.1


From 6c6e197b0a100f14fa69d8decba59e58c7c25b6c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:47:46 +0000
Subject: fix PEP8 violation

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 9f8173644a..f466a64ed9 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -186,7 +186,7 @@ class RoomListHandler(BaseHandler):
         logger.info("After sorting and filtering, %i rooms remain",
                     len(rooms_to_scan))
 
-        #bail if no rooms to work on
+        # bail if no rooms to work on
         if len(rooms_to_scan) == 0:
             defer.returnValue([])
 
-- 
cgit 1.4.1


From f6320835764dbb6cac058763737d67ca6359e3a9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:52:17 +0000
Subject: fix return type, should be a dict

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index f466a64ed9..2ee63548ca 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -188,7 +188,7 @@ class RoomListHandler(BaseHandler):
 
         # bail if no rooms to work on
         if len(rooms_to_scan) == 0:
-            defer.returnValue([])
+            defer.returnValue({})
 
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
-- 
cgit 1.4.1


From 86c4f49a31fe044a727c64e40009596050cdab95 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 26 Jan 2018 00:12:02 +0000
Subject: rather than try reconstruct the results object, better to guard
 against the xrange step argument being 0

---
 synapse/handlers/room_list.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

(limited to 'synapse')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 2ee63548ca..cf62ead816 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -186,10 +186,6 @@ class RoomListHandler(BaseHandler):
         logger.info("After sorting and filtering, %i rooms remain",
                     len(rooms_to_scan))
 
-        # bail if no rooms to work on
-        if len(rooms_to_scan) == 0:
-            defer.returnValue({})
-
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
         #
@@ -207,8 +203,8 @@ class RoomListHandler(BaseHandler):
         if limit:
             step = limit + 1
         else:
-            step = len(rooms_to_scan)
-
+            # step cannot be zero
+            step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
         chunk = []
         for i in xrange(0, len(rooms_to_scan), step):
             batch = rooms_to_scan[i:i + step]
-- 
cgit 1.4.1


From 73560237d646835197e07e9e6c50674786a79a28 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 26 Jan 2018 00:15:10 +0000
Subject: add white space line

---
 synapse/handlers/room_list.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index cf62ead816..dfa09141ed 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -205,6 +205,7 @@ class RoomListHandler(BaseHandler):
         else:
             # step cannot be zero
             step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
+
         chunk = []
         for i in xrange(0, len(rooms_to_scan), step):
             batch = rooms_to_scan[i:i + step]
-- 
cgit 1.4.1


From 25adde9a04b82a2261404f9b6f9a25cec086f62f Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 25 Jan 2018 22:01:02 +0000
Subject: Factor out get_db_conn to HomeServer base class

This function is identical to all subclasses, so we may as well push it up to
the base class to reduce duplication (and make use of it in the tests)
---
 synapse/app/appservice.py        | 13 -------------
 synapse/app/client_reader.py     | 13 -------------
 synapse/app/federation_reader.py | 13 -------------
 synapse/app/federation_sender.py | 13 -------------
 synapse/app/frontend_proxy.py    | 13 -------------
 synapse/app/homeserver.py        | 13 -------------
 synapse/app/media_repository.py  | 13 -------------
 synapse/app/pusher.py            | 13 -------------
 synapse/app/synchrotron.py       | 13 -------------
 synapse/app/user_dir.py          | 13 -------------
 synapse/server.py                | 17 +++++++++++++++++
 11 files changed, 17 insertions(+), 130 deletions(-)

(limited to 'synapse')

diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py
index 7d0c2879ae..c6fe4516d1 100644
--- a/synapse/app/appservice.py
+++ b/synapse/app/appservice.py
@@ -49,19 +49,6 @@ class AppserviceSlaveStore(
 
 
 class AppserviceServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = AppserviceSlaveStore(self.get_db_conn(), self)
diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py
index dc3f6efd43..3b3352798d 100644
--- a/synapse/app/client_reader.py
+++ b/synapse/app/client_reader.py
@@ -64,19 +64,6 @@ class ClientReaderSlavedStore(
 
 
 class ClientReaderServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = ClientReaderSlavedStore(self.get_db_conn(), self)
diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py
index a072291e1f..4de43c41f0 100644
--- a/synapse/app/federation_reader.py
+++ b/synapse/app/federation_reader.py
@@ -58,19 +58,6 @@ class FederationReaderSlavedStore(
 
 
 class FederationReaderServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = FederationReaderSlavedStore(self.get_db_conn(), self)
diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py
index 09e9488f06..f760826d27 100644
--- a/synapse/app/federation_sender.py
+++ b/synapse/app/federation_sender.py
@@ -76,19 +76,6 @@ class FederationSenderSlaveStore(
 
 
 class FederationSenderServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = FederationSenderSlaveStore(self.get_db_conn(), self)
diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py
index ae531c0aa4..e32ee8fe93 100644
--- a/synapse/app/frontend_proxy.py
+++ b/synapse/app/frontend_proxy.py
@@ -118,19 +118,6 @@ class FrontendProxySlavedStore(
 
 
 class FrontendProxyServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = FrontendProxySlavedStore(self.get_db_conn(), self)
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 92ab3b311b..cb82a415a6 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -266,19 +266,6 @@ class SynapseHomeServer(HomeServer):
         except IncorrectDatabaseSetup as e:
             quit_with_error(e.message)
 
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
 
 def setup(config_options):
     """
diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py
index eab1597aaa..1ed1ca8772 100644
--- a/synapse/app/media_repository.py
+++ b/synapse/app/media_repository.py
@@ -60,19 +60,6 @@ class MediaRepositorySlavedStore(
 
 
 class MediaRepositoryServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = MediaRepositorySlavedStore(self.get_db_conn(), self)
diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py
index 7fbbb0b0e1..32ccea3f13 100644
--- a/synapse/app/pusher.py
+++ b/synapse/app/pusher.py
@@ -81,19 +81,6 @@ class PusherSlaveStore(
 
 
 class PusherServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = PusherSlaveStore(self.get_db_conn(), self)
diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py
index 0abba3016e..f87531f1b6 100644
--- a/synapse/app/synchrotron.py
+++ b/synapse/app/synchrotron.py
@@ -246,19 +246,6 @@ class SynchrotronApplicationService(object):
 
 
 class SynchrotronServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = SynchrotronSlavedStore(self.get_db_conn(), self)
diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py
index a48c4a2ae6..494ccb702c 100644
--- a/synapse/app/user_dir.py
+++ b/synapse/app/user_dir.py
@@ -92,19 +92,6 @@ class UserDirectorySlaveStore(
 
 
 class UserDirectoryServer(HomeServer):
-    def get_db_conn(self, run_new_connection=True):
-        # Any param beginning with cp_ is a parameter for adbapi, and should
-        # not be passed to the database engine.
-        db_params = {
-            k: v for k, v in self.db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-        db_conn = self.database_engine.module.connect(**db_params)
-
-        if run_new_connection:
-            self.database_engine.on_new_connection(db_conn)
-        return db_conn
-
     def setup(self):
         logger.info("Setting up.")
         self.datastore = UserDirectorySlaveStore(self.get_db_conn(), self)
diff --git a/synapse/server.py b/synapse/server.py
index 99693071b6..ff8a8fbc46 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -307,6 +307,23 @@ class HomeServer(object):
             **self.db_config.get("args", {})
         )
 
+    def get_db_conn(self, run_new_connection=True):
+        """Makes a new connection to the database, skipping the db pool
+
+        Returns:
+            Connection: a connection object implementing the PEP-249 spec
+        """
+        # Any param beginning with cp_ is a parameter for adbapi, and should
+        # not be passed to the database engine.
+        db_params = {
+            k: v for k, v in self.db_config.get("args", {}).items()
+            if not k.startswith("cp_")
+        }
+        db_conn = self.database_engine.module.connect(**db_params)
+        if run_new_connection:
+            self.database_engine.on_new_connection(db_conn)
+        return db_conn
+
     def build_media_repository_resource(self):
         # build the media repo resource. This indirects through the HomeServer
         # to ensure that we only have a single instance of
-- 
cgit 1.4.1


From 46022025ea35895af3cf8d15973fb94a3a6b4f38 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 25 Jan 2018 21:20:28 +0000
Subject: Fix SQL for user search

fix some syntax errors for user search when search_all_users is enabled

fixes #2801, hopefully
---
 synapse/storage/user_directory.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'synapse')

diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py
index f150ef0103..dfdcbb3181 100644
--- a/synapse/storage/user_directory.py
+++ b/synapse/storage/user_directory.py
@@ -641,13 +641,12 @@ class UserDirectoryStore(SQLBaseStore):
         """
 
         if self.hs.config.user_directory_search_all_users:
-            # dummy to keep the number of binds & aliases the same
+            # make s.user_id null to keep the ordering algorithm happy
             join_clause = """
-                LEFT JOIN (
-                    SELECT NULL as user_id WHERE NULL = ?
-                ) AS s USING (user_id)"
+                CROSS JOIN (SELECT NULL as user_id) AS s
             """
-            where_clause = ""
+            join_args = ()
+            where_clause = "1=1"
         else:
             join_clause = """
                 LEFT JOIN users_in_public_rooms AS p USING (user_id)
@@ -656,6 +655,7 @@ class UserDirectoryStore(SQLBaseStore):
                     WHERE user_id = ? AND share_private
                 ) AS s USING (user_id)
             """
+            join_args = (user_id,)
             where_clause = "(s.user_id IS NOT NULL OR p.user_id IS NOT NULL)"
 
         if isinstance(self.database_engine, PostgresEngine):
@@ -697,7 +697,7 @@ class UserDirectoryStore(SQLBaseStore):
                     avatar_url IS NULL
                 LIMIT ?
             """ % (join_clause, where_clause)
-            args = (user_id, full_query, exact_query, prefix_query, limit + 1,)
+            args = join_args + (full_query, exact_query, prefix_query, limit + 1,)
         elif isinstance(self.database_engine, Sqlite3Engine):
             search_query = _parse_query_sqlite(search_term)
 
@@ -715,7 +715,7 @@ class UserDirectoryStore(SQLBaseStore):
                     avatar_url IS NULL
                 LIMIT ?
             """ % (join_clause, where_clause)
-            args = (user_id, search_query, limit + 1)
+            args = join_args + (search_query, limit + 1)
         else:
             # This should be unreachable.
             raise Exception("Unrecognized database engine")
-- 
cgit 1.4.1


From 02ba118f81009b1c3ae290a17c35e1b9d75e802b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 29 Jan 2018 14:30:15 +0000
Subject: Increase http conn pool size

---
 synapse/http/client.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/http/client.py b/synapse/http/client.py
index 930d713013..f3e4973c2e 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -18,6 +18,7 @@ from OpenSSL.SSL import VERIFY_NONE
 from synapse.api.errors import (
     CodeMessageException, MatrixCodeMessageException, SynapseError, Codes,
 )
+from synapse.util.caches import CACHE_SIZE_FACTOR
 from synapse.util.logcontext import make_deferred_yieldable
 from synapse.util import logcontext
 import synapse.metrics
@@ -67,7 +68,11 @@ class SimpleHttpClient(object):
         self.hs = hs
 
         pool = HTTPConnectionPool(reactor)
-        pool.maxPersistentPerHost = 5
+
+        # the pusher makes lots of concurrent SSL connections to sygnal, and
+        # tends to do so in batches, so we need to allow the pool to keep lots
+        # of idle connections around.
+        pool.maxPersistentPerHost = max((100 * CACHE_SIZE_FACTOR, 5))
         pool.cachedConnectionTimeout = 2 * 60
 
         # The default context factory in Twisted 14.0.0 (which we require) is
-- 
cgit 1.4.1


From e051abd20b1978ddc53723c8233bc54742243045 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 24 Jan 2018 21:06:54 +0000
Subject: add appid/device_display_name to to pusher logging

---
 synapse/push/httppusher.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse')

diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 4a03af5b21..02bd013ca3 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -164,7 +164,7 @@ class HttpPusher(object):
         logger.info(
             "Processing %i unprocessed push actions for %s starting at "
             "stream_ordering %s",
-            len(unprocessed), self.user_id, self.last_stream_ordering,
+            len(unprocessed), self.name, self.last_stream_ordering,
         )
 
         for push_action in unprocessed:
@@ -342,7 +342,7 @@ class HttpPusher(object):
 
     @defer.inlineCallbacks
     def _send_badge(self, badge):
-        logger.info("Sending updated badge count %d to %r", badge, self.user_id)
+        logger.info("Sending updated badge count %d to %s", badge, self.name)
         d = {
             'notification': {
                 'id': '',
-- 
cgit 1.4.1


From 03dd745fe28a00c8788a2147d4f5c2a852182429 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 29 Jan 2018 15:49:06 +0000
Subject: Better logging when pushes fail

---
 synapse/push/httppusher.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'synapse')

diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index 02bd013ca3..2cbac571b8 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -333,7 +333,10 @@ class HttpPusher(object):
         try:
             resp = yield self.http_client.post_json_get_json(self.url, notification_dict)
         except Exception:
-            logger.warn("Failed to push %s ", self.url)
+            logger.warn(
+                "Failed to push event %s to %s",
+                event.event_id, self.name, exc_info=True,
+            )
             defer.returnValue(False)
         rejected = []
         if 'rejected' in resp:
@@ -364,7 +367,10 @@ class HttpPusher(object):
         try:
             resp = yield self.http_client.post_json_get_json(self.url, d)
         except Exception:
-            logger.exception("Failed to push %s ", self.url)
+            logger.warn(
+                "Failed to send badge count to %s",
+                self.name, exc_info=True,
+            )
             defer.returnValue(False)
         rejected = []
         if 'rejected' in resp:
-- 
cgit 1.4.1


From b387ee17b68e4398a8fa26fdf122b773a046e429 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Sat, 27 Jan 2018 14:00:11 +0000
Subject: Improve exception handling in persist_event

1. use `deferred.errback()` instead of `deferred.errback(e)`, which means that
a Failure object will be constructed using the current exception state,
*including* its stack trace - so the stack trace is saved in the Failure,
leading to better exception reports.

2. Set `consumeErrors=True` on the ObservableDeferred, because we know that
there will always be at least one observer - which avoids a spurious "CRITICAL:
unhandled exception in Deferred" error in the logs
---
 synapse/storage/events.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 7a9cd3ec90..33fccfa7a8 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -110,7 +110,7 @@ class _EventPeristenceQueue(object):
                 end_item.events_and_contexts.extend(events_and_contexts)
                 return end_item.deferred.observe()
 
-        deferred = ObservableDeferred(defer.Deferred())
+        deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True)
 
         queue.append(self._EventPersistQueueItem(
             events_and_contexts=events_and_contexts,
@@ -152,8 +152,8 @@ class _EventPeristenceQueue(object):
                     try:
                         ret = yield per_item_callback(item)
                         item.deferred.callback(ret)
-                    except Exception as e:
-                        item.deferred.errback(e)
+                    except Exception:
+                        item.deferred.errback()
             finally:
                 queue = self._event_persist_queues.pop(room_id, None)
                 if queue:
-- 
cgit 1.4.1


From d413a2ba981b3e8d443d2ccdd0c8de7d43b62e73 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Fri, 19 Jan 2018 17:10:26 +0000
Subject: Remove unused "event_type" param on state.get_current_state_ids

this param doesn't seem to be used, and is a bit pointless anyway because it
can easily be replicated by the caller. It is also horrible, because it changes
the return type of the method.
---
 synapse/state.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

(limited to 'synapse')

diff --git a/synapse/state.py b/synapse/state.py
index 1f9abf9d3d..18f1edef42 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -146,8 +146,7 @@ class StateHandler(object):
         defer.returnValue(state)
 
     @defer.inlineCallbacks
-    def get_current_state_ids(self, room_id, event_type=None, state_key="",
-                              latest_event_ids=None):
+    def get_current_state_ids(self, room_id, latest_event_ids=None):
         if not latest_event_ids:
             latest_event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
 
@@ -155,10 +154,6 @@ class StateHandler(object):
         ret = yield self.resolve_state_groups(room_id, latest_event_ids)
         state = ret.state
 
-        if event_type:
-            defer.returnValue(state.get((event_type, state_key)))
-            return
-
         defer.returnValue(state)
 
     @defer.inlineCallbacks
-- 
cgit 1.4.1


From adfc0c95391bf43d48f7069823fc0f4902803402 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 29 Jan 2018 17:39:55 +0000
Subject: docstring for get_current_state_ids

---
 synapse/state.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'synapse')

diff --git a/synapse/state.py b/synapse/state.py
index 18f1edef42..4c8247e7c2 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -147,6 +147,19 @@ class StateHandler(object):
 
     @defer.inlineCallbacks
     def get_current_state_ids(self, room_id, latest_event_ids=None):
+        """Get the current state, or the state at a set of events, for a room
+
+        Args:
+            room_id (str):
+
+            latest_event_ids (iterable[str]|None): if given, the forward
+                extremities to resolve. If None, we look them up from the
+                database (via a cache)
+
+        Returns:
+            Deferred[dict[(str, str), str)]]: the state dict, mapping from
+                (event_type, state_key) -> event_id
+        """
         if not latest_event_ids:
             latest_event_ids = yield self.store.get_latest_event_ids_in_room(room_id)
 
-- 
cgit 1.4.1


From 773f0eed1efa114bb32f6e54e8edc038a04d3526 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 30 Jan 2018 15:02:51 +0000
Subject: Fix sql error in quarantine_media

---
 synapse/storage/room.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse')

diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 23688430b7..d91c853070 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -589,7 +589,7 @@ class RoomStore(SQLBaseStore):
                     """
                         UPDATE remote_media_cache
                         SET quarantined_by = ?
-                        WHERE media_origin AND media_id = ?
+                        WHERE media_origin = ? AND media_id = ?
                     """,
                     (
                         (quarantined_by, origin, media_id)
-- 
cgit 1.4.1


From af19f5e9aa47dd9875df3937d514ae7708eed539 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 30 Jan 2018 17:52:03 +0000
Subject: Remove spurious log argument

... which would cause scary-looking and unhelpful errors in the log on dns fail
---
 synapse/http/endpoint.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'synapse')

diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index e2b99ef3bd..87639b9151 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -357,8 +357,7 @@ def _get_hosts_for_srv_record(dns_client, host):
     def eb(res, record_type):
         if res.check(DNSNameError):
             return []
-        logger.warn("Error looking up %s for %s: %s",
-                    record_type, host, res, res.value)
+        logger.warn("Error looking up %s for %s: %s", record_type, host, res)
         return res
 
     # no logcontexts here, so we can safely fire these off and gatherResults
-- 
cgit 1.4.1