From 8c9d5b4873dc5e54c1383e3d31060fd32b08fae1 Mon Sep 17 00:00:00 2001
From: Michael Telatynski <7t3chguy@gmail.com>
Date: Wed, 10 Jan 2018 15:04:02 +0000
Subject: Fix publicised groups API (singular) over federation which was
 missing its fed client API, since there is no other API it might as well
 reuse the bulk one and unwrap it

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
---
 synapse/handlers/groups_local.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 7e5d3f148d..0d3feacd47 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -383,11 +383,11 @@ class GroupsLocalHandler(object):
 
             defer.returnValue({"groups": result})
         else:
-            result = yield self.transport_client.get_publicised_groups_for_user(
-                get_domain_from_id(user_id), user_id
-            )
+            result = yield self.transport_client.bulk_get_publicised_groups(
+                get_domain_from_id(user_id), [user_id],
+            )["users"][user_id]
             # TODO: Verify attestations
-            defer.returnValue(result)
+            defer.returnValue({"groups": result})
 
     @defer.inlineCallbacks
     def bulk_get_publicised_groups(self, user_ids, proxy=True):
-- 
cgit 1.4.1


From 5f07f5694c9a0535e64262c3726007e966ba72ea Mon Sep 17 00:00:00 2001
From: Michael Telatynski <7t3chguy@gmail.com>
Date: Wed, 10 Jan 2018 15:11:35 +0000
Subject: fix order of operations derp and also use `.get` to default to {}

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
---
 synapse/handlers/groups_local.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py
index 0d3feacd47..e4d0cc8b02 100644
--- a/synapse/handlers/groups_local.py
+++ b/synapse/handlers/groups_local.py
@@ -383,9 +383,10 @@ class GroupsLocalHandler(object):
 
             defer.returnValue({"groups": result})
         else:
-            result = yield self.transport_client.bulk_get_publicised_groups(
+            bulk_result = yield self.transport_client.bulk_get_publicised_groups(
                 get_domain_from_id(user_id), [user_id],
-            )["users"][user_id]
+            )
+            result = bulk_result.get("users", {}).get(user_id)
             # TODO: Verify attestations
             defer.returnValue({"groups": result})
 
-- 
cgit 1.4.1


From f8e1ab5fee708bea93c54b39dc802a08c7ed7e21 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 10 Jan 2018 18:01:28 +0000
Subject: Do bcrypt hashing in a background thread

---
 synapse/handlers/auth.py         | 24 ++++++++++++++++--------
 synapse/handlers/register.py     |  2 +-
 synapse/handlers/set_password.py |  2 +-
 3 files changed, 18 insertions(+), 10 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 2f30f183ce..af9060fdbf 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from twisted.internet import defer
+from twisted.internet import defer, threads
 
 from ._base import BaseHandler
 from synapse.api.constants import LoginType
@@ -22,6 +22,7 @@ from synapse.module_api import ModuleApi
 from synapse.types import UserID
 from synapse.util.async import run_on_reactor
 from synapse.util.caches.expiringcache import ExpiringCache
+from synapse.util.logcontext import make_deferred_yieldable
 
 from twisted.web.client import PartialDownloadError
 
@@ -626,7 +627,7 @@ class AuthHandler(BaseHandler):
         if not lookupres:
             defer.returnValue(None)
         (user_id, password_hash) = lookupres
-        result = self.validate_hash(password, password_hash)
+        result = yield self.validate_hash(password, password_hash)
         if not result:
             logger.warn("Failed password login for user %s", user_id)
             defer.returnValue(None)
@@ -754,10 +755,13 @@ class AuthHandler(BaseHandler):
             password (str): Password to hash.
 
         Returns:
-            Hashed password (str).
+            Deferred(str): Hashed password.
         """
-        return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
-                             bcrypt.gensalt(self.bcrypt_rounds))
+        def _do_hash():
+            return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
+                                 bcrypt.gensalt(self.bcrypt_rounds))
+
+        return make_deferred_yieldable(threads.deferToThread(_do_hash))
 
     def validate_hash(self, password, stored_hash):
         """Validates that self.hash(password) == stored_hash.
@@ -767,13 +771,17 @@ class AuthHandler(BaseHandler):
             stored_hash (str): Expected hash value.
 
         Returns:
-            Whether self.hash(password) == stored_hash (bool).
+            Deferred(bool): Whether self.hash(password) == stored_hash.
         """
-        if stored_hash:
+
+        def _do_validate_hash():
             return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
                                  stored_hash.encode('utf8')) == stored_hash
+
+        if stored_hash:
+            return make_deferred_yieldable(threads.deferToThread(_do_validate_hash))
         else:
-            return False
+            return defer.succeed(False)
 
 
 class MacaroonGeneartor(object):
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index f6e7e58563..132e2052c0 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -130,7 +130,7 @@ class RegistrationHandler(BaseHandler):
         yield run_on_reactor()
         password_hash = None
         if password:
-            password_hash = self.auth_handler().hash(password)
+            password_hash = yield self.auth_handler().hash(password)
 
         if localpart:
             yield self.check_username(localpart, guest_access_token=guest_access_token)
diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py
index 44414e1dc1..e057ae54c9 100644
--- a/synapse/handlers/set_password.py
+++ b/synapse/handlers/set_password.py
@@ -31,7 +31,7 @@ class SetPasswordHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def set_password(self, user_id, newpassword, requester=None):
-        password_hash = self._auth_handler.hash(newpassword)
+        password_hash = yield self._auth_handler.hash(newpassword)
 
         except_device_id = requester.device_id if requester else None
         except_access_token_id = requester.access_token_id if requester else None
-- 
cgit 1.4.1


From a027c2af8d348554cad4855094a6f46ef21bfad7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 15 Jan 2018 18:20:30 +0000
Subject: Metrics for events processed in appservice and fed sender

More metrics I wished I'd had
---
 synapse/federation/transaction_queue.py | 4 ++++
 synapse/handlers/appservice.py          | 7 +++++++
 2 files changed, 11 insertions(+)

(limited to 'synapse/handlers')

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 3e7809b04f..9d39f46583 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -42,6 +42,8 @@ sent_edus_counter = client_metrics.register_counter("sent_edus")
 
 sent_transactions_counter = client_metrics.register_counter("sent_transactions")
 
+events_processed_counter = client_metrics.register_counter("events_processed")
+
 
 class TransactionQueue(object):
     """This class makes sure we only have one transaction in flight at
@@ -205,6 +207,8 @@ class TransactionQueue(object):
 
                     self._send_pdu(event, destinations)
 
+                events_processed_counter.inc_by(len(events))
+
                 yield self.store.update_federation_out_pos(
                     "events", next_token
                 )
diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py
index feca3e4c10..3dd3fa2a27 100644
--- a/synapse/handlers/appservice.py
+++ b/synapse/handlers/appservice.py
@@ -15,6 +15,7 @@
 
 from twisted.internet import defer
 
+import synapse
 from synapse.api.constants import EventTypes
 from synapse.util.metrics import Measure
 from synapse.util.logcontext import make_deferred_yieldable, preserve_fn
@@ -23,6 +24,10 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+metrics = synapse.metrics.get_metrics_for(__name__)
+
+events_processed_counter = metrics.register_counter("events_processed")
+
 
 def log_failure(failure):
     logger.error(
@@ -103,6 +108,8 @@ class ApplicationServicesHandler(object):
                                 service, event
                             )
 
+                    events_processed_counter.inc_by(len(events))
+
                     yield self.store.set_appservice_last_pos(upper_bound)
             finally:
                 self.is_processing = False
-- 
cgit 1.4.1


From a5213df1f7448c1810e6ae842448c81abe57ba69 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 16 Jan 2018 18:25:28 +0000
Subject: Sanity checking for user ids

Check the user_id passed to a couple of APIs for validity, to avoid
"IndexError: list index out of range" exception which looks scary and results
in a 500 rather than a more useful error.

Fixes #1432, among other things
---
 synapse/handlers/devicemessage.py | 14 +++++++++++---
 synapse/handlers/e2e_keys.py      | 13 ++++++++-----
 2 files changed, 19 insertions(+), 8 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py
index f7fad15c62..d996aa90bb 100644
--- a/synapse/handlers/devicemessage.py
+++ b/synapse/handlers/devicemessage.py
@@ -17,7 +17,8 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.types import get_domain_from_id
+from synapse.api.errors import SynapseError
+from synapse.types import get_domain_from_id, UserID
 from synapse.util.stringutils import random_string
 
 
@@ -33,7 +34,7 @@ class DeviceMessageHandler(object):
         """
         self.store = hs.get_datastore()
         self.notifier = hs.get_notifier()
-        self.is_mine_id = hs.is_mine_id
+        self.is_mine = hs.is_mine
         self.federation = hs.get_federation_sender()
 
         hs.get_replication_layer().register_edu_handler(
@@ -52,6 +53,12 @@ class DeviceMessageHandler(object):
         message_type = content["type"]
         message_id = content["message_id"]
         for user_id, by_device in content["messages"].items():
+            # we use UserID.from_string to catch invalid user ids
+            if not self.is_mine(UserID.from_string(user_id)):
+                logger.warning("Request for keys for non-local user %s",
+                               user_id)
+                raise SynapseError(400, "Not a user here")
+
             messages_by_device = {
                 device_id: {
                     "content": message_content,
@@ -77,7 +84,8 @@ class DeviceMessageHandler(object):
         local_messages = {}
         remote_messages = {}
         for user_id, by_device in messages.items():
-            if self.is_mine_id(user_id):
+            # we use UserID.from_string to catch invalid user ids
+            if self.is_mine(UserID.from_string(user_id)):
                 messages_by_device = {
                     device_id: {
                         "content": message_content,
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 668a90e495..5af8abf66b 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -20,7 +20,7 @@ from canonicaljson import encode_canonical_json
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError, CodeMessageException
-from synapse.types import get_domain_from_id
+from synapse.types import get_domain_from_id, UserID
 from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
 from synapse.util.retryutils import NotRetryingDestination
 
@@ -32,7 +32,7 @@ class E2eKeysHandler(object):
         self.store = hs.get_datastore()
         self.federation = hs.get_replication_layer()
         self.device_handler = hs.get_device_handler()
-        self.is_mine_id = hs.is_mine_id
+        self.is_mine = hs.is_mine
         self.clock = hs.get_clock()
 
         # doesn't really work as part of the generic query API, because the
@@ -70,7 +70,8 @@ class E2eKeysHandler(object):
         remote_queries = {}
 
         for user_id, device_ids in device_keys_query.items():
-            if self.is_mine_id(user_id):
+            # we use UserID.from_string to catch invalid user ids
+            if self.is_mine(UserID.from_string(user_id)):
                 local_query[user_id] = device_ids
             else:
                 remote_queries[user_id] = device_ids
@@ -170,7 +171,8 @@ class E2eKeysHandler(object):
 
         result_dict = {}
         for user_id, device_ids in query.items():
-            if not self.is_mine_id(user_id):
+            # we use UserID.from_string to catch invalid user ids
+            if not self.is_mine(UserID.from_string(user_id)):
                 logger.warning("Request for keys for non-local user %s",
                                user_id)
                 raise SynapseError(400, "Not a user here")
@@ -213,7 +215,8 @@ class E2eKeysHandler(object):
         remote_queries = {}
 
         for user_id, device_keys in query.get("one_time_keys", {}).items():
-            if self.is_mine_id(user_id):
+            # we use UserID.from_string to catch invalid user ids
+            if self.is_mine(UserID.from_string(user_id)):
                 for device_id, algorithm in device_keys.items():
                     local_query.append((user_id, device_id, algorithm))
             else:
-- 
cgit 1.4.1


From 9d332e0f797e4f302a08b3708df4ac8b42b08216 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 00:53:58 +0000
Subject: fix up v1, and improve errors

---
 synapse/handlers/register.py             | 13 ++++++++++-
 synapse/rest/client/v1/register.py       | 40 +++++++++++++++++++++++---------
 synapse/rest/client/v2_alpha/account.py  | 16 +++++++++----
 synapse/rest/client/v2_alpha/register.py | 16 +++++++++----
 4 files changed, 65 insertions(+), 20 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 5b808beac1..157ebaf251 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -15,6 +15,7 @@
 
 """Contains functions for registering clients."""
 import logging
+import re
 
 from twisted.internet import defer
 
@@ -293,7 +294,7 @@ class RegistrationHandler(BaseHandler):
         """
 
         for c in threepidCreds:
-            logger.info("validating theeepidcred sid %s on id server %s",
+            logger.info("validating threepidcred sid %s on id server %s",
                         c['sid'], c['idServer'])
             try:
                 identity_handler = self.hs.get_handlers().identity_handler
@@ -307,6 +308,16 @@ class RegistrationHandler(BaseHandler):
             logger.info("got threepid with medium '%s' and address '%s'",
                         threepid['medium'], threepid['address'])
 
+            for constraint in self.hs.config.registrations_require_3pid:
+                if (
+                    constraint['medium'] == 'email' and
+                    threepid['medium'] == 'email' and
+                    re.match(constraint['pattern'], threepid['address'])
+                ):
+                    raise RegistrationError(
+                        403, "Third party identifier is not allowed"
+                    )
+
     @defer.inlineCallbacks
     def bind_emails(self, user_id, threepidCreds):
         """Links emails with a user ID and informs an identity server.
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index 32ed1d3ab2..f793542ad6 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -70,10 +70,24 @@ class RegisterRestServlet(ClientV1RestServlet):
         self.handlers = hs.get_handlers()
 
     def on_GET(self, request):
+
+        require_email = False
+        require_msisdn = False
+        for constraint in self.hs.config.registrations_require_3pid:
+            if constraint['medium'] == 'email':
+                require_email = True
+            elif constraint['medium'] == 'msisdn':
+                require_msisdn = True
+            else:
+                logger.warn(
+                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
+                    constraint['medium']
+                )
+
+        flows = []
         if self.hs.config.enable_registration_captcha:
-            return (
-                200,
-                {"flows": [
+            if require_email or not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.RECAPTCHA,
                         "stages": [
@@ -82,27 +96,31 @@ class RegisterRestServlet(ClientV1RestServlet):
                             LoginType.PASSWORD
                         ]
                     },
+                ])
+            if not require_email and not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.RECAPTCHA,
                         "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD]
                     }
-                ]}
-            )
+                ])
         else:
-            return (
-                200,
-                {"flows": [
+            if require_email or not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.EMAIL_IDENTITY,
                         "stages": [
                             LoginType.EMAIL_IDENTITY, LoginType.PASSWORD
                         ]
-                    },
+                    }
+                ])
+            if not require_email and not require_msisdn:
+                flows.extend([
                     {
                         "type": LoginType.PASSWORD
                     }
-                ]}
-            )
+                ])
+        return (200, {"flows": flows})
 
     @defer.inlineCallbacks
     def on_POST(self, request):
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 66221e8f00..2977ad439f 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -48,7 +48,9 @@ class EmailPasswordRequestTokenRestServlet(RestServlet):
         ])
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'email', body['email']
@@ -82,7 +84,9 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet):
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'msisdn', msisdn
@@ -224,7 +228,9 @@ class EmailThreepidRequestTokenRestServlet(RestServlet):
             raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'email', body['email']
@@ -265,7 +271,9 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet):
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.datastore.get_user_id_by_threepid(
             'msisdn', msisdn
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 762782c1f0..898d8b133a 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -72,7 +72,9 @@ class EmailRegisterRequestTokenRestServlet(RestServlet):
         ])
 
         if not check_3pid_allowed(self.hs, "email", body['email']):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'email', body['email']
@@ -110,7 +112,9 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet):
         msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
 
         if not check_3pid_allowed(self.hs, "msisdn", msisdn):
-            raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+            raise SynapseError(
+                403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+            )
 
         existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
             'msisdn', msisdn
@@ -368,7 +372,9 @@ class RegisterRestServlet(RestServlet):
                     auth_result[LoginType.EMAIL_IDENTITY].threepid.address
                 )
             ):
-                raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+                raise SynapseError(
+                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                )
             elif (
                 constraint['medium'] == 'msisdn' and
                 auth_result and LoginType.MSISDN in auth_result and
@@ -377,7 +383,9 @@ class RegisterRestServlet(RestServlet):
                     auth_result[LoginType.MSISDN].threepid.address
                 )
             ):
-                raise SynapseError(403, "3PID denied", Codes.THREEPID_DENIED)
+                raise SynapseError(
+                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
+                )
 
         if registered_user_id is not None:
             logger.info(
-- 
cgit 1.4.1


From 447f4f0d5f136dcadd5fdc286ded2d6e24a3f686 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 19 Jan 2018 15:33:55 +0000
Subject: rewrite based on PR feedback:

 * [ ] split config options into allowed_local_3pids and registrations_require_3pid
 * [ ] simplify and comment logic for picking registration flows
 * [ ] fix docstring and move check_3pid_allowed into a new util module
 * [ ] use check_3pid_allowed everywhere

@erikjohnston PTAL
---
 synapse/config/registration.py           | 12 +++--
 synapse/handlers/register.py             | 15 +++----
 synapse/rest/client/v1/register.py       | 20 +++------
 synapse/rest/client/v2_alpha/_base.py    | 21 ---------
 synapse/rest/client/v2_alpha/account.py  |  3 +-
 synapse/rest/client/v2_alpha/register.py | 75 +++++++++++++++-----------------
 synapse/util/threepids.py                | 45 +++++++++++++++++++
 7 files changed, 102 insertions(+), 89 deletions(-)
 create mode 100644 synapse/util/threepids.py

(limited to 'synapse/handlers')

diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index e5e4f77872..336959094b 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -32,6 +32,7 @@ class RegistrationConfig(Config):
             )
 
         self.registrations_require_3pid = config.get("registrations_require_3pid", [])
+        self.allowed_local_3pids = config.get("allowed_local_3pids", [])
         self.registration_shared_secret = config.get("registration_shared_secret")
 
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
@@ -53,11 +54,16 @@ class RegistrationConfig(Config):
         # Enable registration for new users.
         enable_registration: False
 
-        # Mandate that registrations require a 3PID which matches one or more
-        # of these 3PIDs.  N.B. regexp escape backslashes are doubled (once for
-        # YAML and once for the regexp itself)
+        # The user must provide all of the below types of 3PID when registering.
         #
         # registrations_require_3pid:
+        #     - email
+        #     - msisdn
+
+        # Mandate that users are only allowed to associate certain formats of
+        # 3PIDs with accounts on this server.
+        #
+        # allowed_local_3pids:
         #     - medium: email
         #       pattern: ".*@matrix\\.org"
         #     - medium: email
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 157ebaf251..9021d4d57f 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -15,7 +15,6 @@
 
 """Contains functions for registering clients."""
 import logging
-import re
 
 from twisted.internet import defer
 
@@ -26,6 +25,7 @@ from synapse.http.client import CaptchaServerHttpClient
 from synapse import types
 from synapse.types import UserID
 from synapse.util.async import run_on_reactor
+from synapse.util.threepids import check_3pid_allowed
 from ._base import BaseHandler
 
 logger = logging.getLogger(__name__)
@@ -308,15 +308,10 @@ class RegistrationHandler(BaseHandler):
             logger.info("got threepid with medium '%s' and address '%s'",
                         threepid['medium'], threepid['address'])
 
-            for constraint in self.hs.config.registrations_require_3pid:
-                if (
-                    constraint['medium'] == 'email' and
-                    threepid['medium'] == 'email' and
-                    re.match(constraint['pattern'], threepid['address'])
-                ):
-                    raise RegistrationError(
-                        403, "Third party identifier is not allowed"
-                    )
+            if not check_3pid_allowed(self.hs, threepid['medium'], threepid['address']):
+                raise RegistrationError(
+                    403, "Third party identifier is not allowed"
+                )
 
     @defer.inlineCallbacks
     def bind_emails(self, user_id, threepidCreds):
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index f793542ad6..5c5fa8f7ab 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -71,22 +71,13 @@ class RegisterRestServlet(ClientV1RestServlet):
 
     def on_GET(self, request):
 
-        require_email = False
-        require_msisdn = False
-        for constraint in self.hs.config.registrations_require_3pid:
-            if constraint['medium'] == 'email':
-                require_email = True
-            elif constraint['medium'] == 'msisdn':
-                require_msisdn = True
-            else:
-                logger.warn(
-                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
-                    constraint['medium']
-                )
+        require_email = 'email' in self.hs.config.registrations_require_3pid
+        require_msisdn = 'msisdn' in self.hs.config.registrations_require_3pid
 
         flows = []
         if self.hs.config.enable_registration_captcha:
-            if require_email or not require_msisdn:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
+            if not require_msisdn:
                 flows.extend([
                     {
                         "type": LoginType.RECAPTCHA,
@@ -97,6 +88,7 @@ class RegisterRestServlet(ClientV1RestServlet):
                         ]
                     },
                 ])
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([
                     {
@@ -105,6 +97,7 @@ class RegisterRestServlet(ClientV1RestServlet):
                     }
                 ])
         else:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
             if require_email or not require_msisdn:
                 flows.extend([
                     {
@@ -114,6 +107,7 @@ class RegisterRestServlet(ClientV1RestServlet):
                         ]
                     }
                 ])
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([
                     {
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index b286ff0d95..77434937ff 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -60,27 +60,6 @@ def set_timeline_upper_limit(filter_json, filter_timeline_limit):
             filter_timeline_limit)
 
 
-def check_3pid_allowed(hs, medium, address):
-    # check whether the HS has whitelisted the given 3PID
-
-    allow = False
-    if hs.config.registrations_require_3pid:
-        for constraint in hs.config.registrations_require_3pid:
-            logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
-                address, medium, constraint['pattern'], constraint['medium']
-            ))
-            if (
-                medium == constraint['medium'] and
-                re.match(constraint['pattern'], address)
-            ):
-                allow = True
-                break
-    else:
-        allow = True
-
-    return allow
-
-
 def interactive_auth_handler(orig):
     """Wraps an on_POST method to handle InteractiveAuthIncompleteErrors
 
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 2977ad439f..514bb37da1 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -26,7 +26,8 @@ from synapse.http.servlet import (
 )
 from synapse.util.async import run_on_reactor
 from synapse.util.msisdn import phone_number_to_msisdn
-from ._base import client_v2_patterns, interactive_auth_handler, check_3pid_allowed
+from synapse.util.threepids import check_3pid_allowed
+from ._base import client_v2_patterns, interactive_auth_handler
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 898d8b133a..c3479e29de 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -26,11 +26,11 @@ from synapse.http.servlet import (
     RestServlet, parse_json_object_from_request, assert_params_in_request, parse_string
 )
 from synapse.util.msisdn import phone_number_to_msisdn
+from synapse.util.threepids import check_3pid_allowed
 
-from ._base import client_v2_patterns, interactive_auth_handler, check_3pid_allowed
+from ._base import client_v2_patterns, interactive_auth_handler
 
 import logging
-import re
 import hmac
 from hashlib import sha1
 from synapse.util.async import run_on_reactor
@@ -316,41 +316,41 @@ class RegisterRestServlet(RestServlet):
         if 'x_show_msisdn' in body and body['x_show_msisdn']:
             show_msisdn = True
 
-        require_email = False
-        require_msisdn = False
-        for constraint in self.hs.config.registrations_require_3pid:
-            if constraint['medium'] == 'email':
-                require_email = True
-            elif constraint['medium'] == 'msisdn':
-                require_msisdn = True
-            else:
-                logger.warn(
-                    "Unrecognised 3PID medium %s in registrations_require_3pid" %
-                    constraint['medium']
-                )
+        # FIXME: need a better error than "no auth flow found" for scenarios
+        # where we required 3PID for registration but the user didn't give one
+        require_email = 'email' in self.hs.config.registrations_require_3pid
+        require_msisdn = 'msisdn' in self.hs.config.registrations_require_3pid
 
         flows = []
         if self.hs.config.enable_registration_captcha:
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([[LoginType.RECAPTCHA]])
-            if require_email or not require_msisdn:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
+            if not require_msisdn:
                 flows.extend([[LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]])
 
             if show_msisdn:
-                if not require_email or require_msisdn:
+                # only support the MSISDN-only flow if we don't require email 3PIDs
+                if not require_email:
                     flows.extend([[LoginType.MSISDN, LoginType.RECAPTCHA]])
+                # always let users provide both MSISDN & email
                 flows.extend([
                     [LoginType.MSISDN, LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
                 ])
         else:
+            # only support 3PIDless registration if no 3PIDs are required
             if not require_email and not require_msisdn:
                 flows.extend([[LoginType.DUMMY]])
-            if require_email or not require_msisdn:
+            # only support the email-only flow if we don't require MSISDN 3PIDs
+            if not require_msisdn:
                 flows.extend([[LoginType.EMAIL_IDENTITY]])
 
             if show_msisdn:
+                # only support the MSISDN-only flow if we don't require email 3PIDs
                 if not require_email or require_msisdn:
                     flows.extend([[LoginType.MSISDN]])
+                # always let users provide both MSISDN & email
                 flows.extend([
                     [LoginType.MSISDN, LoginType.EMAIL_IDENTITY]
                 ])
@@ -359,30 +359,23 @@ class RegisterRestServlet(RestServlet):
             flows, body, self.hs.get_ip_from_request(request)
         )
 
-        # doublecheck that we're not trying to register an denied 3pid.
-        # the user-facing checks should already have happened when we requested
-        # a 3PID token to validate them in /register/email/requestToken etc
-
-        for constraint in self.hs.config.registrations_require_3pid:
-            if (
-                constraint['medium'] == 'email' and
-                auth_result and LoginType.EMAIL_IDENTITY in auth_result and
-                re.match(
-                    constraint['pattern'],
-                    auth_result[LoginType.EMAIL_IDENTITY].threepid.address
-                )
-            ):
-                raise SynapseError(
-                    403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
-                )
-            elif (
-                constraint['medium'] == 'msisdn' and
-                auth_result and LoginType.MSISDN in auth_result and
-                re.match(
-                    constraint['pattern'],
-                    auth_result[LoginType.MSISDN].threepid.address
-                )
-            ):
+        # Check that we're not trying to register a denied 3pid.
+        #
+        # the user-facing checks will probably already have happened in
+        # /register/email/requestToken when we requested a 3pid, but that's not
+        # guaranteed.
+
+        if (
+            auth_result and
+            (
+                LoginType.EMAIL_IDENTITY in auth_result or
+                LoginType.EMAIL_MSISDN in auth_result
+            )
+        ):
+            medium = auth_result[LoginType.EMAIL_IDENTITY].threepid['medium']
+            address = auth_result[LoginType.EMAIL_IDENTITY].threepid['address']
+
+            if not check_3pid_allowed(self.hs, medium, address):
                 raise SynapseError(
                     403, "Third party identifier is not allowed", Codes.THREEPID_DENIED
                 )
diff --git a/synapse/util/threepids.py b/synapse/util/threepids.py
new file mode 100644
index 0000000000..e921b97796
--- /dev/null
+++ b/synapse/util/threepids.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def check_3pid_allowed(hs, medium, address):
+    """Checks whether a given format of 3PID is allowed to be used on this HS
+
+    Args:
+        hs (synapse.server.HomeServer): server
+        medium (str): 3pid medium - e.g. email, msisdn
+        address (str): address within that medium (e.g. "wotan@matrix.org")
+            msisdns need to first have been canonicalised
+    """
+
+    if hs.config.allowed_local_3pids:
+        for constraint in hs.config.allowed_local_3pids:
+            logger.debug("Checking 3PID %s (%s) against %s (%s)" % (
+                address, medium, constraint['pattern'], constraint['medium']
+            ))
+            if (
+                medium == constraint['medium'] and
+                re.match(constraint['pattern'], address)
+            ):
+                return True
+    else:
+        return True
+
+    return False
-- 
cgit 1.4.1


From ab9f844aaf3662a64dbc4c56077e9fa37bc7d5d0 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@arasphere.net>
Date: Mon, 22 Jan 2018 19:11:18 +0100
Subject: Add federation_domain_whitelist option (#2820)

Add federation_domain_whitelist

gives a way to restrict which domains your HS is allowed to federate with.
useful mainly for gracefully preventing a private but internet-connected HS from trying to federate to the wider public Matrix network
---
 synapse/api/errors.py                      | 26 ++++++++++++++++++++++++++
 synapse/config/server.py                   | 22 ++++++++++++++++++++++
 synapse/federation/federation_client.py    |  5 ++++-
 synapse/federation/transaction_queue.py    |  4 +++-
 synapse/federation/transport/client.py     |  3 +++
 synapse/federation/transport/server.py     |  9 ++++++++-
 synapse/handlers/device.py                 |  4 ++++
 synapse/handlers/e2e_keys.py               |  8 +++++++-
 synapse/handlers/federation.py             |  4 ++++
 synapse/http/matrixfederationclient.py     | 28 +++++++++++++++++++++++++++-
 synapse/rest/key/v2/remote_key_resource.py |  8 ++++++++
 synapse/rest/media/v1/media_repository.py  | 19 +++++++++++++++++--
 synapse/util/retryutils.py                 | 12 ++++++++++++
 tests/utils.py                             |  1 +
 14 files changed, 146 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 46b0d7b34c..aa15f73f36 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -141,6 +141,32 @@ class RegistrationError(SynapseError):
     pass
 
 
+class FederationDeniedError(SynapseError):
+    """An error raised when the server tries to federate with a server which
+    is not on its federation whitelist.
+
+    Attributes:
+        destination (str): The destination which has been denied
+    """
+
+    def __init__(self, destination):
+        """Raised by federation client or server to indicate that we are
+        are deliberately not attempting to contact a given server because it is
+        not on our federation whitelist.
+
+        Args:
+            destination (str): the domain in question
+        """
+
+        self.destination = destination
+
+        super(FederationDeniedError, self).__init__(
+            code=403,
+            msg="Federation denied with %s." % (self.destination,),
+            errcode=Codes.FORBIDDEN,
+        )
+
+
 class InteractiveAuthIncompleteError(Exception):
     """An error raised when UI auth is not yet complete
 
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 436dd8a6fe..8f0b6d1f28 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -55,6 +55,17 @@ class ServerConfig(Config):
             "block_non_admin_invites", False,
         )
 
+        # FIXME: federation_domain_whitelist needs sytests
+        self.federation_domain_whitelist = None
+        federation_domain_whitelist = config.get(
+            "federation_domain_whitelist", None
+        )
+        # turn the whitelist into a hash for speed of lookup
+        if federation_domain_whitelist is not None:
+            self.federation_domain_whitelist = {}
+            for domain in federation_domain_whitelist:
+                self.federation_domain_whitelist[domain] = True
+
         if self.public_baseurl is not None:
             if self.public_baseurl[-1] != '/':
                 self.public_baseurl += '/'
@@ -210,6 +221,17 @@ class ServerConfig(Config):
         # (except those sent by local server admins). The default is False.
         # block_non_admin_invites: True
 
+        # Restrict federation to the following whitelist of domains.
+        # N.B. we recommend also firewalling your federation listener to limit
+        # inbound federation traffic as early as possible, rather than relying
+        # purely on this application-layer restriction.  If not specified, the
+        # default is to whitelist everything.
+        #
+        # federation_domain_whitelist:
+        #  - lon.example.com
+        #  - nyc.example.com
+        #  - syd.example.com
+
         # List of ports that Synapse should listen on, their purpose and their
         # configuration.
         listeners:
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index b1fe03f702..813907f7f2 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -23,7 +23,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import Membership
 from synapse.api.errors import (
-    CodeMessageException, HttpResponseException, SynapseError,
+    CodeMessageException, HttpResponseException, SynapseError, FederationDeniedError
 )
 from synapse.events import builder
 from synapse.federation.federation_base import (
@@ -266,6 +266,9 @@ class FederationClient(FederationBase):
             except NotRetryingDestination as e:
                 logger.info(e.message)
                 continue
+            except FederationDeniedError as e:
+                logger.info(e.message)
+                continue
             except Exception as e:
                 pdu_attempts[destination] = now
 
diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 9d39f46583..a141ec9953 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -19,7 +19,7 @@ from twisted.internet import defer
 from .persistence import TransactionActions
 from .units import Transaction, Edu
 
-from synapse.api.errors import HttpResponseException
+from synapse.api.errors import HttpResponseException, FederationDeniedError
 from synapse.util import logcontext, PreserveLoggingContext
 from synapse.util.async import run_on_reactor
 from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
@@ -490,6 +490,8 @@ class TransactionQueue(object):
                     (e.retry_last_ts + e.retry_interval) / 1000.0
                 ),
             )
+        except FederationDeniedError as e:
+            logger.info(e)
         except Exception as e:
             logger.warn(
                 "TX [%s] Failed to send transaction: %s",
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 1f3ce238f6..5488e82985 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -212,6 +212,9 @@ class TransportLayerClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if the remote destination
+            is not in our federation whitelist
         """
         valid_memberships = {Membership.JOIN, Membership.LEAVE}
         if membership not in valid_memberships:
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 2b02b021ec..06c16ba4fa 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.urls import FEDERATION_PREFIX as PREFIX
-from synapse.api.errors import Codes, SynapseError
+from synapse.api.errors import Codes, SynapseError, FederationDeniedError
 from synapse.http.server import JsonResource
 from synapse.http.servlet import (
     parse_json_object_from_request, parse_integer_from_args, parse_string_from_args,
@@ -81,6 +81,7 @@ class Authenticator(object):
         self.keyring = hs.get_keyring()
         self.server_name = hs.hostname
         self.store = hs.get_datastore()
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
 
     # A method just so we can pass 'self' as the authenticator to the Servlets
     @defer.inlineCallbacks
@@ -92,6 +93,12 @@ class Authenticator(object):
             "signatures": {},
         }
 
+        if (
+            self.federation_domain_whitelist is not None and
+            self.server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(self.server_name)
+
         if content is not None:
             json_request["content"] = content
 
diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 2152efc692..0e83453851 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 from synapse.api import errors
 from synapse.api.constants import EventTypes
+from synapse.api.errors import FederationDeniedError
 from synapse.util import stringutils
 from synapse.util.async import Linearizer
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -513,6 +514,9 @@ class DeviceListEduUpdater(object):
                     # This makes it more likely that the device lists will
                     # eventually become consistent.
                     return
+                except FederationDeniedError as e:
+                    logger.info(e)
+                    return
                 except Exception:
                     # TODO: Remember that we are now out of sync and try again
                     # later
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 5af8abf66b..9aa95f89e6 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -19,7 +19,9 @@ import logging
 from canonicaljson import encode_canonical_json
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError, CodeMessageException
+from synapse.api.errors import (
+    SynapseError, CodeMessageException, FederationDeniedError,
+)
 from synapse.types import get_domain_from_id, UserID
 from synapse.util.logcontext import preserve_fn, make_deferred_yieldable
 from synapse.util.retryutils import NotRetryingDestination
@@ -140,6 +142,10 @@ class E2eKeysHandler(object):
                 failures[destination] = {
                     "status": 503, "message": "Not ready for retry",
                 }
+            except FederationDeniedError as e:
+                failures[destination] = {
+                    "status": 403, "message": "Federation Denied",
+                }
             except Exception as e:
                 # include ConnectionRefused and other errors
                 failures[destination] = {
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index ac70730885..677532c87b 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -22,6 +22,7 @@ from ._base import BaseHandler
 
 from synapse.api.errors import (
     AuthError, FederationError, StoreError, CodeMessageException, SynapseError,
+    FederationDeniedError,
 )
 from synapse.api.constants import EventTypes, Membership, RejectedReason
 from synapse.events.validator import EventValidator
@@ -782,6 +783,9 @@ class FederationHandler(BaseHandler):
                 except NotRetryingDestination as e:
                     logger.info(e.message)
                     continue
+                except FederationDeniedError as e:
+                    logger.info(e)
+                    continue
                 except Exception as e:
                     logger.exception(
                         "Failed to backfill from %s because %s",
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 833496b72d..9145405cb0 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -27,7 +27,7 @@ import synapse.metrics
 from canonicaljson import encode_canonical_json
 
 from synapse.api.errors import (
-    SynapseError, Codes, HttpResponseException,
+    SynapseError, Codes, HttpResponseException, FederationDeniedError,
 )
 
 from signedjson.sign import sign_json
@@ -123,11 +123,22 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``HTTPRequestException``: if we get an HTTP response
                 code >= 300.
+
             Fails with ``NotRetryingDestination`` if we are not yet ready
                 to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+                is not on our federation whitelist
+
             (May also fail with plenty of other Exceptions for things like DNS
                 failures, connection failures, SSL failures.)
         """
+        if (
+            self.hs.config.federation_domain_whitelist and
+            destination not in self.hs.config.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(destination)
+
         limiter = yield synapse.util.retryutils.get_retry_limiter(
             destination,
             self.clock,
@@ -308,6 +319,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         if not json_data_callback:
@@ -368,6 +382,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         def body_callback(method, url_bytes, headers_dict):
@@ -422,6 +439,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
         logger.debug("get_json args: %s", args)
 
@@ -475,6 +495,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         response = yield self._request(
@@ -518,6 +541,9 @@ class MatrixFederationHttpClient(object):
 
             Fails with ``NotRetryingDestination`` if we are not yet ready
             to retry this server.
+
+            Fails with ``FederationDeniedError`` if this destination
+            is not on our federation whitelist
         """
 
         encoded_args = {}
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index cc2842aa72..17e6079cba 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -93,6 +93,7 @@ class RemoteKey(Resource):
         self.store = hs.get_datastore()
         self.version_string = hs.version_string
         self.clock = hs.get_clock()
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
 
     def render_GET(self, request):
         self.async_render_GET(request)
@@ -137,6 +138,13 @@ class RemoteKey(Resource):
         logger.info("Handling query for keys %r", query)
         store_queries = []
         for server_name, key_ids in query.items():
+            if (
+                self.federation_domain_whitelist is not None and
+                server_name not in self.federation_domain_whitelist
+            ):
+                logger.debug("Federation denied with %s", server_name)
+                continue
+
             if not key_ids:
                 key_ids = (None,)
             for key_id in key_ids:
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index 4f56bcf577..485db8577a 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -32,8 +32,9 @@ from .media_storage import MediaStorage
 
 from synapse.http.matrixfederationclient import MatrixFederationHttpClient
 from synapse.util.stringutils import random_string
-from synapse.api.errors import SynapseError, HttpResponseException, \
-    NotFoundError
+from synapse.api.errors import (
+    SynapseError, HttpResponseException, NotFoundError, FederationDeniedError,
+)
 
 from synapse.util.async import Linearizer
 from synapse.util.stringutils import is_ascii
@@ -75,6 +76,8 @@ class MediaRepository(object):
         self.recently_accessed_remotes = set()
         self.recently_accessed_locals = set()
 
+        self.federation_domain_whitelist = hs.config.federation_domain_whitelist
+
         # List of StorageProviders where we should search for media and
         # potentially upload to.
         storage_providers = []
@@ -216,6 +219,12 @@ class MediaRepository(object):
             Deferred: Resolves once a response has successfully been written
                 to request
         """
+        if (
+            self.federation_domain_whitelist is not None and
+            server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(server_name)
+
         self.mark_recently_accessed(server_name, media_id)
 
         # We linearize here to ensure that we don't try and download remote
@@ -250,6 +259,12 @@ class MediaRepository(object):
         Returns:
             Deferred[dict]: The media_info of the file
         """
+        if (
+            self.federation_domain_whitelist is not None and
+            server_name not in self.federation_domain_whitelist
+        ):
+            raise FederationDeniedError(server_name)
+
         # We linearize here to ensure that we don't try and download remote
         # media multiple times concurrently
         key = (server_name, media_id)
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 1adedbb361..47b0bb5eb3 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -26,6 +26,18 @@ logger = logging.getLogger(__name__)
 
 class NotRetryingDestination(Exception):
     def __init__(self, retry_last_ts, retry_interval, destination):
+        """Raised by the limiter (and federation client) to indicate that we are
+        are deliberately not attempting to contact a given server.
+
+        Args:
+            retry_last_ts (int): the unix ts in milliseconds of our last attempt
+                to contact the server.  0 indicates that the last attempt was
+                successful or that we've never actually attempted to connect.
+            retry_interval (int): the time in milliseconds to wait until the next
+                attempt.
+            destination (str): the domain in question
+        """
+
         msg = "Not retrying server %s." % (destination,)
         super(NotRetryingDestination, self).__init__(msg)
 
diff --git a/tests/utils.py b/tests/utils.py
index 44e5f75093..3116047892 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -57,6 +57,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.worker_app = None
         config.email_enable_notifs = False
         config.block_non_admin_invites = False
+        config.federation_domain_whitelist = None
 
         # disable user directory updates, because they get done in the
         # background, which upsets the test runner.
-- 
cgit 1.4.1


From 349c7399663b5fce856995a3a901019f5d210cc4 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:28:44 +0000
Subject: synapse 500s on a call to publicRooms in the case where the number of
 public rooms is zero, the specific cause is due to xrange trying to use a
 step value of zero, but if the total room number really is zero then it makes
 sense to just bail and save the extra processing

---
 synapse/handlers/room_list.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index bb40075387..ae5db4d2c5 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -186,6 +186,11 @@ class RoomListHandler(BaseHandler):
         logger.info("After sorting and filtering, %i rooms remain",
                     len(rooms_to_scan))
 
+        #bail if no rooms to work on
+        if len(rooms_to_scan) == 0:
+            defer.returnValue([])
+
+
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
         #
-- 
cgit 1.4.1


From d02e43b15f6b9b24ffe5e0c0d696f8fd71fc8af3 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:29:46 +0000
Subject: remove white space

---
 synapse/handlers/room_list.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index ae5db4d2c5..9f8173644a 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -190,7 +190,6 @@ class RoomListHandler(BaseHandler):
         if len(rooms_to_scan) == 0:
             defer.returnValue([])
 
-
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
         #
-- 
cgit 1.4.1


From 6c6e197b0a100f14fa69d8decba59e58c7c25b6c Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:47:46 +0000
Subject: fix PEP8 violation

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 9f8173644a..f466a64ed9 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -186,7 +186,7 @@ class RoomListHandler(BaseHandler):
         logger.info("After sorting and filtering, %i rooms remain",
                     len(rooms_to_scan))
 
-        #bail if no rooms to work on
+        # bail if no rooms to work on
         if len(rooms_to_scan) == 0:
             defer.returnValue([])
 
-- 
cgit 1.4.1


From f6320835764dbb6cac058763737d67ca6359e3a9 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Thu, 25 Jan 2018 23:52:17 +0000
Subject: fix return type, should be a dict

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index f466a64ed9..2ee63548ca 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -188,7 +188,7 @@ class RoomListHandler(BaseHandler):
 
         # bail if no rooms to work on
         if len(rooms_to_scan) == 0:
-            defer.returnValue([])
+            defer.returnValue({})
 
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
-- 
cgit 1.4.1


From 86c4f49a31fe044a727c64e40009596050cdab95 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 26 Jan 2018 00:12:02 +0000
Subject: rather than try reconstruct the results object, better to guard
 against the xrange step argument being 0

---
 synapse/handlers/room_list.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 2ee63548ca..cf62ead816 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -186,10 +186,6 @@ class RoomListHandler(BaseHandler):
         logger.info("After sorting and filtering, %i rooms remain",
                     len(rooms_to_scan))
 
-        # bail if no rooms to work on
-        if len(rooms_to_scan) == 0:
-            defer.returnValue({})
-
         # _append_room_entry_to_chunk will append to chunk but will stop if
         # len(chunk) > limit
         #
@@ -207,8 +203,8 @@ class RoomListHandler(BaseHandler):
         if limit:
             step = limit + 1
         else:
-            step = len(rooms_to_scan)
-
+            # step cannot be zero
+            step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
         chunk = []
         for i in xrange(0, len(rooms_to_scan), step):
             batch = rooms_to_scan[i:i + step]
-- 
cgit 1.4.1


From 73560237d646835197e07e9e6c50674786a79a28 Mon Sep 17 00:00:00 2001
From: Neil Johnson <neil@fragile.org.uk>
Date: Fri, 26 Jan 2018 00:15:10 +0000
Subject: add white space line

---
 synapse/handlers/room_list.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index cf62ead816..dfa09141ed 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -205,6 +205,7 @@ class RoomListHandler(BaseHandler):
         else:
             # step cannot be zero
             step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
+
         chunk = []
         for i in xrange(0, len(rooms_to_scan), step):
             batch = rooms_to_scan[i:i + step]
-- 
cgit 1.4.1