From a61dd408ed78db974a4d78f0708cc9405e36df64 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Sun, 12 Mar 2017 19:30:45 +0000
Subject: enable guest access for the 3pl/3pid APIs

---
 synapse/rest/client/v2_alpha/thirdparty.py | 8 --------
 1 file changed, 8 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py
index 31f94bc6e9..ee2f158d03 100644
--- a/synapse/rest/client/v2_alpha/thirdparty.py
+++ b/synapse/rest/client/v2_alpha/thirdparty.py
@@ -36,8 +36,6 @@ class ThirdPartyProtocolsServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        yield self.auth.get_user_by_req(request)
-
         protocols = yield self.appservice_handler.get_3pe_protocols()
         defer.returnValue((200, protocols))
 
@@ -54,8 +52,6 @@ class ThirdPartyProtocolServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, protocol):
-        yield self.auth.get_user_by_req(request)
-
         protocols = yield self.appservice_handler.get_3pe_protocols(
             only_protocol=protocol,
         )
@@ -77,8 +73,6 @@ class ThirdPartyUserServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, protocol):
-        yield self.auth.get_user_by_req(request)
-
         fields = request.args
         fields.pop("access_token", None)
 
@@ -101,8 +95,6 @@ class ThirdPartyLocationServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, protocol):
-        yield self.auth.get_user_by_req(request)
-
         fields = request.args
         fields.pop("access_token", None)
 
-- 
cgit 1.4.1


From e0ff66251f71cb46aea30a187edc1dc027760b9e Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 15 Mar 2017 12:22:18 +0000
Subject: add setting (on by default) to support TURN for guests

---
 docs/turn-howto.rst            | 38 ++++++++++++++++++++++++++++++++++----
 synapse/config/voip.py         |  8 ++++++++
 synapse/rest/client/v1/voip.py |  5 ++++-
 3 files changed, 46 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/docs/turn-howto.rst b/docs/turn-howto.rst
index 04c0100715..e48628ce6e 100644
--- a/docs/turn-howto.rst
+++ b/docs/turn-howto.rst
@@ -50,14 +50,37 @@ You may be able to setup coturn via your package manager,  or set it up manually
 
        pwgen -s 64 1
 
- 5. Ensure youe firewall allows traffic into the TURN server on
+ 5. Consider your security settings.  TURN lets users request a relay
+    which will connect to arbitrary IP addresses and ports.  At the least
+    we recommend:
+
+       # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay.
+       no-tcp-relay
+
+       # don't let the relay ever try to connect to private IP address ranges within your network (if any)
+       # given the turn server is likely behind your firewall, remember to include any privileged public IPs too.
+       denied-peer-ip=10.0.0.0-10.255.255.255
+       denied-peer-ip=192.168.0.0-192.168.255.255
+       denied-peer-ip=172.16.0.0-172.31.255.255
+
+       # special case the turn server itself so that client->TURN->TURN->client flows work
+       allowed-peer-ip=10.0.0.1
+
+       # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS.
+       user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user.
+       total-quota=1200
+
+    Ideally coturn should refuse to relay traffic which isn't SRTP;
+    see https://github.com/matrix-org/synapse/issues/2009
+
+ 6. Ensure your firewall allows traffic into the TURN server on
     the ports you've configured it to listen on (remember to allow
-    both TCP and UDP if you've enabled both).
+    both TCP and UDP TURN traffic)
 
- 6. If you've configured coturn to support TLS/DTLS, generate or
+ 7. If you've configured coturn to support TLS/DTLS, generate or
     import your private key and certificate.
 
- 7. Start the turn server::
+ 8. Start the turn server::
  
        bin/turnserver -o
 
@@ -83,12 +106,19 @@ Your home server configuration file needs the following extra keys:
     to refresh credentials. The TURN REST API specification recommends
     one day (86400000).
 
+  4. "turn_allow_guests": Whether to allow guest users to use the TURN
+    server.  This is enabled by default, as otherwise VoIP will not
+    work reliably for guests.  However, it does introduce a security risk
+    as it lets guests connect to arbitrary endpoints without having gone
+    through a CAPTCHA or similar to register a real account.
+
 As an example, here is the relevant section of the config file for
 matrix.org::
 
     turn_uris: [ "turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp" ]
     turn_shared_secret: n0t4ctuAllymatr1Xd0TorgSshar3d5ecret4obvIousreAsons
     turn_user_lifetime: 86400000
+    turn_allow_guests: True
 
 Now, restart synapse::
 
diff --git a/synapse/config/voip.py b/synapse/config/voip.py
index eeb693027b..c93c92d177 100644
--- a/synapse/config/voip.py
+++ b/synapse/config/voip.py
@@ -23,6 +23,7 @@ class VoipConfig(Config):
         self.turn_username = config.get("turn_username")
         self.turn_password = config.get("turn_password")
         self.turn_user_lifetime = self.parse_duration(config["turn_user_lifetime"])
+        self.turn_allow_guests = config.get("turn_allow_guests") or True
 
     def default_config(self, **kwargs):
         return """\
@@ -41,4 +42,11 @@ class VoipConfig(Config):
 
         # How long generated TURN credentials last
         turn_user_lifetime: "1h"
+
+        # Whether guests should be allowed to use the TURN server.
+        # This is defaults to True, otherwise VoIP will be unreliable for guests.
+        # However, it does introduce a slight security risk as it allows users to
+        # connect to arbitrary endpoints without having first signed up for a
+        # valid account (e.g. by passing a CAPTCHA).
+        turn_allow_guests: True
         """
diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py
index 03141c623c..c43b30b73a 100644
--- a/synapse/rest/client/v1/voip.py
+++ b/synapse/rest/client/v1/voip.py
@@ -28,7 +28,10 @@ class VoipRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        requester = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(
+            request,
+            self.hs.config.turn_allow_guests
+        )
 
         turnUris = self.hs.config.turn_uris
         turnSecret = self.hs.config.turn_shared_secret
-- 
cgit 1.4.1


From 9ee397b440c01f2dd170c7f7341cb47b90cf2762 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Fri, 31 Mar 2017 13:54:26 +0100
Subject: switch to allow_guest=True for authing 3Ps as per PR feedback

---
 synapse/rest/client/v2_alpha/thirdparty.py | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py
index ee2f158d03..6fceb23e26 100644
--- a/synapse/rest/client/v2_alpha/thirdparty.py
+++ b/synapse/rest/client/v2_alpha/thirdparty.py
@@ -36,6 +36,8 @@ class ThirdPartyProtocolsServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
+        yield self.auth.get_user_by_req(request, allow_guest=True)
+
         protocols = yield self.appservice_handler.get_3pe_protocols()
         defer.returnValue((200, protocols))
 
@@ -52,6 +54,8 @@ class ThirdPartyProtocolServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, protocol):
+        yield self.auth.get_user_by_req(request, allow_guest=True)
+
         protocols = yield self.appservice_handler.get_3pe_protocols(
             only_protocol=protocol,
         )
@@ -73,6 +77,8 @@ class ThirdPartyUserServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, protocol):
+        yield self.auth.get_user_by_req(request, allow_guest=True)
+
         fields = request.args
         fields.pop("access_token", None)
 
@@ -95,6 +101,8 @@ class ThirdPartyLocationServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, protocol):
+        yield self.auth.get_user_by_req(request, allow_guest=True)
+
         fields = request.args
         fields.pop("access_token", None)
 
-- 
cgit 1.4.1


From e263c26690685b713ab8a5f4fdb7b9e92a7b8d99 Mon Sep 17 00:00:00 2001
From: lukebarnard <luke.barnard99@gmail.com>
Date: Tue, 11 Apr 2017 11:55:30 +0100
Subject: Initial commit of RM server-side impl

(See https://docs.google.com/document/d/1UWqdS-e1sdwkLDUY0wA4gZyIkRp-ekjsLZ8k6g_Zvso/edit#heading=h.lndohpg8at5u)
---
 synapse/handlers/read_marker.py             | 82 +++++++++++++++++++++++++++++
 synapse/rest/client/v2_alpha/read_marker.py | 71 +++++++++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 synapse/handlers/read_marker.py
 create mode 100644 synapse/rest/client/v2_alpha/read_marker.py

(limited to 'synapse/rest')

diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
new file mode 100644
index 0000000000..77164f3f49
--- /dev/null
+++ b/synapse/handlers/read_marker.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import BaseHandler
+
+from twisted.internet import defer
+
+from synapse.util.logcontext import PreserveLoggingContext
+from synapse.types import get_domain_from_id
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReadMarkerHandler(BaseHandler):
+    def __init__(self, hs):
+        super(ReadMarkerHandler, self).__init__(hs)
+
+        self.server_name = hs.config.server_name
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def received_client_read_marker(self, room_id, user_id, event_id):
+        """NEEDS DOC
+        """
+
+        room_id = read_marker["room_id"]
+        user_id = read_marker["user_id"]
+        event_id = read_marker["event_id"]
+
+        # Get ordering for existing read marker
+        account_data = yield self.store.get_account_data_for_room(user_id, room_id)
+        existing_read_marker = account_data["m.read_marker"]
+
+        if existing_read_marker:
+            # Get ordering for new read marker
+            res = self.store._simple_select_one_txn(
+                txn,
+                table="events",
+                retcols=["topological_ordering", "stream_ordering"],
+                keyvalues={"event_id": event_id},
+                allow_none=True
+            )
+            new_to = int(res["topological_ordering"]) if res else None
+            new_so = int(res["stream_ordering"]) if res else None
+
+            res = self.store._simple_select_one_txn(
+                txn,
+                table="events",
+                retcols=["topological_ordering", "stream_ordering"],
+                keyvalues={"event_id": existing_read_marker.content.marker},
+                allow_none=True
+            )
+            existing_to = int(res["topological_ordering"]) if res else None
+            existing_so = int(res["stream_ordering"]) if res else None
+
+            if new_to > existing_to:
+                return False
+            elif new_to == existing_to and new_so >= existing_so:
+                return False
+
+        # Update account data
+        content = {
+            "marker": event_id
+        }
+        yield self.store.add_account_data_to_room(
+            user_id, room_id, "m.read_marker", content
+        )
diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
new file mode 100644
index 0000000000..02408eaf10
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.errors import SynapseError
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from ._base import client_v2_patterns
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReceiptRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns(
+        "/rooms/(?P<room_id>[^/]*)"
+        "/read_marker$"
+    )
+
+    def __init__(self, hs):
+        super(ReceiptRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.receipts_handler = hs.get_receipts_handler()
+        self.read_marker_handler = hs.get_read_marker_handler()
+        self.presence_handler = hs.get_presence_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, room_id, receipt_type, event_id):
+        requester = yield self.auth.get_user_by_req(request)
+
+        yield self.presence_handler.bump_presence_active_time(requester.user)
+
+        body = parse_json_object_from_request(request)
+
+        if "m.read" in body:
+            read_event_id = body["m.read"];
+            yield self.receipts_handler.received_client_receipt(
+                room_id,
+                "m.read",
+                user_id=requester.user.to_string(),
+                event_id=read_event_id
+            )
+
+        if "m.read_marker" in body:
+            read_marker_event_id = body["m.read_marker"]
+            yield self.read_marker_handler.received_client_read_marker(
+                room_id,
+                user_id=requester.user.to_string(),
+                event_id=read_marker_event_id
+            )
+
+        defer.returnValue((200, {}))
+
+
+def register_servlets(hs, http_server):
+    ReceiptRestServlet(hs).register(http_server)
-- 
cgit 1.4.1


From d892079844eec9dd7855ee66f3ad3225df4bdbc0 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Tue, 11 Apr 2017 15:01:39 +0100
Subject: Finish implementing RM endpoint

 - This change causes a 405 to be sent if "m.read_marker" is set via /account_data
 - This also fixes-up the RM endpoint so that it actually Works.
---
 synapse/handlers/read_marker.py              | 87 +++++++++++++++-------------
 synapse/rest/__init__.py                     |  2 +
 synapse/rest/client/v2_alpha/account_data.py |  8 ++-
 synapse/rest/client/v2_alpha/read_marker.py  | 13 ++---
 synapse/server.py                            |  5 ++
 5 files changed, 67 insertions(+), 48 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
index 77164f3f49..021faff376 100644
--- a/synapse/handlers/read_marker.py
+++ b/synapse/handlers/read_marker.py
@@ -18,65 +18,74 @@ from ._base import BaseHandler
 from twisted.internet import defer
 
 from synapse.util.logcontext import PreserveLoggingContext
+from synapse.util.async import Linearizer
 from synapse.types import get_domain_from_id
+from synapse.api.errors import SynapseError
 
 import logging
-
-
 logger = logging.getLogger(__name__)
 
-
 class ReadMarkerHandler(BaseHandler):
     def __init__(self, hs):
         super(ReadMarkerHandler, self).__init__(hs)
-
         self.server_name = hs.config.server_name
         self.store = hs.get_datastore()
+        self.read_marker_linearizer = Linearizer(name="read_marker")
+        self.notifier = hs.get_notifier()
 
     @defer.inlineCallbacks
     def received_client_read_marker(self, room_id, user_id, event_id):
-        """NEEDS DOC
-        """
+        """Updates the read marker for a given user in a given room if the event ID given
+        is ahead in the stream relative to the current read marker.
 
-        room_id = read_marker["room_id"]
-        user_id = read_marker["user_id"]
-        event_id = read_marker["event_id"]
+        This uses a notifier to indicate that account data should be sent down /sync if
+        the read marker has changed.
+        """
 
         # Get ordering for existing read marker
-        account_data = yield self.store.get_account_data_for_room(user_id, room_id)
-        existing_read_marker = account_data["m.read_marker"]
+        with (yield self.read_marker_linearizer.queue(room_id + "_" + user_id)):
+            account_data = yield self.store.get_account_data_for_room(user_id, room_id)
+            existing_read_marker = account_data["m.read_marker"]
 
-        if existing_read_marker:
-            # Get ordering for new read marker
-            res = self.store._simple_select_one_txn(
-                txn,
-                table="events",
-                retcols=["topological_ordering", "stream_ordering"],
-                keyvalues={"event_id": event_id},
-                allow_none=True
-            )
-            new_to = int(res["topological_ordering"]) if res else None
-            new_so = int(res["stream_ordering"]) if res else None
+            should_update = True
 
-            res = self.store._simple_select_one_txn(
-                txn,
+            res = yield self.store._simple_select_one(
                 table="events",
                 retcols=["topological_ordering", "stream_ordering"],
-                keyvalues={"event_id": existing_read_marker.content.marker},
+                keyvalues={"event_id": event_id},
                 allow_none=True
             )
-            existing_to = int(res["topological_ordering"]) if res else None
-            existing_so = int(res["stream_ordering"]) if res else None
-
-            if new_to > existing_to:
-                return False
-            elif new_to == existing_to and new_so >= existing_so:
-                return False
 
-        # Update account data
-        content = {
-            "marker": event_id
-        }
-        yield self.store.add_account_data_to_room(
-            user_id, room_id, "m.read_marker", content
-        )
+            if not res:
+                raise SynapseError(404, 'Event does not exist')
+
+            if existing_read_marker:
+                new_to = int(res["topological_ordering"])
+                new_so = int(res["stream_ordering"])
+
+                # Get ordering for existing read marker
+                res = yield self.store._simple_select_one(
+                    table="events",
+                    retcols=["topological_ordering", "stream_ordering"],
+                    keyvalues={"event_id": existing_read_marker['marker']},
+                    allow_none=True
+                )
+                existing_to = int(res["topological_ordering"]) if res else None
+                existing_so = int(res["stream_ordering"]) if res else None
+
+                # Prevent updating if the existing marker is ahead in the stream
+                if existing_to > new_to:
+                    should_update = False
+                elif existing_to == new_to and existing_so >= new_so:
+                    should_update = False
+
+            if should_update:
+                content = {
+                    "marker": event_id
+                }
+                max_id = yield self.store.add_account_data_to_room(
+                    user_id, room_id, "m.read_marker", content
+                )
+                self.notifier.on_new_event(
+                    "account_data_key", max_id, users=[user_id], rooms=[room_id]
+                )
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index f9f5a3e077..aa8d874f96 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -40,6 +40,7 @@ from synapse.rest.client.v2_alpha import (
     register,
     auth,
     receipts,
+    read_marker,
     keys,
     tokenrefresh,
     tags,
@@ -88,6 +89,7 @@ class ClientRestResource(JsonResource):
         register.register_servlets(hs, client_resource)
         auth.register_servlets(hs, client_resource)
         receipts.register_servlets(hs, client_resource)
+        read_marker.register_servlets(hs, client_resource)
         keys.register_servlets(hs, client_resource)
         tokenrefresh.register_servlets(hs, client_resource)
         tags.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
index b16079cece..a846ab1dc1 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -16,7 +16,7 @@
 from ._base import client_v2_patterns
 
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
-from synapse.api.errors import AuthError
+from synapse.api.errors import AuthError, SynapseError
 
 from twisted.internet import defer
 
@@ -82,6 +82,12 @@ class RoomAccountDataServlet(RestServlet):
 
         body = parse_json_object_from_request(request)
 
+        if account_data_type == "m.read_marker":
+            raise SynapseError(405,
+                "Cannot set m.read_marker through this API. "
+                "Use /rooms/!roomId:server.name/read_marker"
+            )
+
         max_id = yield self.store.add_account_data_to_room(
             user_id, room_id, account_data_type, body
         )
diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
index 02408eaf10..49ada9c047 100644
--- a/synapse/rest/client/v2_alpha/read_marker.py
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -25,14 +25,11 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-class ReceiptRestServlet(RestServlet):
-    PATTERNS = client_v2_patterns(
-        "/rooms/(?P<room_id>[^/]*)"
-        "/read_marker$"
-    )
+class ReadMarkerRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/rooms/(?P<room_id>[^/]*)/read_marker$")
 
     def __init__(self, hs):
-        super(ReceiptRestServlet, self).__init__()
+        super(ReadMarkerRestServlet, self).__init__()
         self.hs = hs
         self.auth = hs.get_auth()
         self.receipts_handler = hs.get_receipts_handler()
@@ -40,7 +37,7 @@ class ReceiptRestServlet(RestServlet):
         self.presence_handler = hs.get_presence_handler()
 
     @defer.inlineCallbacks
-    def on_POST(self, request, room_id, receipt_type, event_id):
+    def on_POST(self, request, room_id):
         requester = yield self.auth.get_user_by_req(request)
 
         yield self.presence_handler.bump_presence_active_time(requester.user)
@@ -68,4 +65,4 @@ class ReceiptRestServlet(RestServlet):
 
 
 def register_servlets(hs, http_server):
-    ReceiptRestServlet(hs).register(http_server)
+    ReadMarkerRestServlet(hs).register(http_server)
diff --git a/synapse/server.py b/synapse/server.py
index 6310152560..12754c89ae 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -48,6 +48,7 @@ from synapse.handlers.typing import TypingHandler
 from synapse.handlers.events import EventHandler, EventStreamHandler
 from synapse.handlers.initial_sync import InitialSyncHandler
 from synapse.handlers.receipts import ReceiptsHandler
+from synapse.handlers.read_marker import ReadMarkerHandler
 from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory
 from synapse.http.matrixfederationclient import MatrixFederationHttpClient
 from synapse.notifier import Notifier
@@ -133,6 +134,7 @@ class HomeServer(object):
         'receipts_handler',
         'macaroon_generator',
         'tcp_replication',
+        'read_marker_handler',
     ]
 
     def __init__(self, hostname, **kwargs):
@@ -291,6 +293,9 @@ class HomeServer(object):
     def build_receipts_handler(self):
         return ReceiptsHandler(self)
 
+    def build_read_marker_handler(self):
+        return ReadMarkerHandler(self)
+
     def build_tcp_replication(self):
         raise NotImplementedError()
 
-- 
cgit 1.4.1


From 012742302748a231593222beffc5ceff61966ffc Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Tue, 11 Apr 2017 17:07:07 +0100
Subject: flake8

---
 synapse/handlers/read_marker.py              | 3 +--
 synapse/rest/client/v2_alpha/account_data.py | 3 ++-
 synapse/rest/client/v2_alpha/read_marker.py  | 3 +--
 3 files changed, 4 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
index 021faff376..a97f6b9a79 100644
--- a/synapse/handlers/read_marker.py
+++ b/synapse/handlers/read_marker.py
@@ -17,14 +17,13 @@ from ._base import BaseHandler
 
 from twisted.internet import defer
 
-from synapse.util.logcontext import PreserveLoggingContext
 from synapse.util.async import Linearizer
-from synapse.types import get_domain_from_id
 from synapse.api.errors import SynapseError
 
 import logging
 logger = logging.getLogger(__name__)
 
+
 class ReadMarkerHandler(BaseHandler):
     def __init__(self, hs):
         super(ReadMarkerHandler, self).__init__(hs)
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
index a846ab1dc1..56cd347e78 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -83,7 +83,8 @@ class RoomAccountDataServlet(RestServlet):
         body = parse_json_object_from_request(request)
 
         if account_data_type == "m.read_marker":
-            raise SynapseError(405,
+            raise SynapseError(
+                405,
                 "Cannot set m.read_marker through this API. "
                 "Use /rooms/!roomId:server.name/read_marker"
             )
diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
index 49ada9c047..a568a9252a 100644
--- a/synapse/rest/client/v2_alpha/read_marker.py
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -15,7 +15,6 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from ._base import client_v2_patterns
 
@@ -45,7 +44,7 @@ class ReadMarkerRestServlet(RestServlet):
         body = parse_json_object_from_request(request)
 
         if "m.read" in body:
-            read_event_id = body["m.read"];
+            read_event_id = body["m.read"]
             yield self.receipts_handler.received_client_receipt(
                 room_id,
                 "m.read",
-- 
cgit 1.4.1


From b9676a75f6bf87a83bea75d541cc933ac6aad93a Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 12 Apr 2017 10:51:17 +0100
Subject: Move a space

---
 synapse/rest/client/v2_alpha/account_data.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
index 56cd347e78..1f9f42e661 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -85,8 +85,8 @@ class RoomAccountDataServlet(RestServlet):
         if account_data_type == "m.read_marker":
             raise SynapseError(
                 405,
-                "Cannot set m.read_marker through this API. "
-                "Use /rooms/!roomId:server.name/read_marker"
+                "Cannot set m.read_marker through this API."
+                " Use /rooms/!roomId:server.name/read_marker"
             )
 
         max_id = yield self.store.add_account_data_to_room(
-- 
cgit 1.4.1


From c0aba0a23e4babaff55af57ca8601518ad12d967 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 12 Apr 2017 10:52:11 +0100
Subject: Remove Unused ref to hs

---
 synapse/rest/client/v2_alpha/read_marker.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
index a568a9252a..bf5a96459c 100644
--- a/synapse/rest/client/v2_alpha/read_marker.py
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -29,7 +29,6 @@ class ReadMarkerRestServlet(RestServlet):
 
     def __init__(self, hs):
         super(ReadMarkerRestServlet, self).__init__()
-        self.hs = hs
         self.auth = hs.get_auth()
         self.receipts_handler = hs.get_receipts_handler()
         self.read_marker_handler = hs.get_read_marker_handler()
-- 
cgit 1.4.1


From cf6121e3da30a3bbfe50d68546cc0b026802e5c8 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 12 Apr 2017 14:02:03 +0100
Subject: More null-guard changes

---
 synapse/rest/client/v2_alpha/read_marker.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
index bf5a96459c..95e8e39974 100644
--- a/synapse/rest/client/v2_alpha/read_marker.py
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -42,8 +42,8 @@ class ReadMarkerRestServlet(RestServlet):
 
         body = parse_json_object_from_request(request)
 
-        if "m.read" in body:
-            read_event_id = body["m.read"]
+        read_event_id = body.get("m.read", None)
+        if read_event_id:
             yield self.receipts_handler.received_client_receipt(
                 room_id,
                 "m.read",
@@ -51,8 +51,8 @@ class ReadMarkerRestServlet(RestServlet):
                 event_id=read_event_id
             )
 
-        if "m.read_marker" in body:
-            read_marker_event_id = body["m.read_marker"]
+        read_marker_event_id = body.get("m.read_marker", None)
+        if read_marker_event_id:
             yield self.read_marker_handler.received_client_read_marker(
                 room_id,
                 user_id=requester.user.to_string(),
-- 
cgit 1.4.1


From 574d573ac2c7f345faf61ef549c9d2c39664b3a3 Mon Sep 17 00:00:00 2001
From: Anant Prakash <anantprakashjsr@gmail.com>
Date: Fri, 14 Apr 2017 23:42:42 +0530
Subject: Check that requested room_id exists

---
 synapse/rest/client/v1/directory.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py
index 8930f1826f..f15aa5c13f 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/v1/directory.py
@@ -39,6 +39,7 @@ class ClientDirectoryServer(ClientV1RestServlet):
 
     def __init__(self, hs):
         super(ClientDirectoryServer, self).__init__(hs)
+        self.store = hs.get_datastore()
         self.handlers = hs.get_handlers()
 
     @defer.inlineCallbacks
@@ -70,7 +71,10 @@ class ClientDirectoryServer(ClientV1RestServlet):
         logger.debug("Got servers: %s", servers)
 
         # TODO(erikj): Check types.
-        # TODO(erikj): Check that room exists
+
+        room = yield self.store.get_room(room_id)
+        if room is None:
+            raise SynapseError(400, "Room does not exist")
 
         dir_handler = self.handlers.directory_handler
 
-- 
cgit 1.4.1


From 3fb8784c9243da32d680b21773f35d885e8be480 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukebarnard1@users.noreply.github.com>
Date: Tue, 18 Apr 2017 17:46:15 +0100
Subject: m.read_marker -> m.fully_read (#2128)

Also:
 - change the REST endpoint to have a "S" on the end (so it's now /read_markers)
 - change the content of the m.read_up_to event to have the key "event_id" instead of "marker".
---
 synapse/handlers/read_marker.py              | 8 ++++----
 synapse/rest/client/v2_alpha/account_data.py | 6 +++---
 synapse/rest/client/v2_alpha/read_marker.py  | 4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
index 3f46a16b90..b5b0303d54 100644
--- a/synapse/handlers/read_marker.py
+++ b/synapse/handlers/read_marker.py
@@ -43,7 +43,7 @@ class ReadMarkerHandler(BaseHandler):
         with (yield self.read_marker_linearizer.queue((room_id, user_id))):
             account_data = yield self.store.get_account_data_for_room(user_id, room_id)
 
-            existing_read_marker = account_data.get("m.read_marker", None)
+            existing_read_marker = account_data.get("m.fully_read", None)
 
             should_update = True
 
@@ -51,14 +51,14 @@ class ReadMarkerHandler(BaseHandler):
                 # Only update if the new marker is ahead in the stream
                 should_update = yield self.store.is_event_after(
                     event_id,
-                    existing_read_marker['marker']
+                    existing_read_marker['event_id']
                 )
 
             if should_update:
                 content = {
-                    "marker": event_id
+                    "event_id": event_id
                 }
                 max_id = yield self.store.add_account_data_to_room(
-                    user_id, room_id, "m.read_marker", content
+                    user_id, room_id, "m.fully_read", content
                 )
                 self.notifier.on_new_event("account_data_key", max_id, users=[user_id])
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
index 1f9f42e661..0e0a187efd 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -82,11 +82,11 @@ class RoomAccountDataServlet(RestServlet):
 
         body = parse_json_object_from_request(request)
 
-        if account_data_type == "m.read_marker":
+        if account_data_type == "m.fully_read":
             raise SynapseError(
                 405,
-                "Cannot set m.read_marker through this API."
-                " Use /rooms/!roomId:server.name/read_marker"
+                "Cannot set m.fully_read through this API."
+                " Use /rooms/!roomId:server.name/read_markers"
             )
 
         max_id = yield self.store.add_account_data_to_room(
diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
index 95e8e39974..2f8784fe06 100644
--- a/synapse/rest/client/v2_alpha/read_marker.py
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 
 class ReadMarkerRestServlet(RestServlet):
-    PATTERNS = client_v2_patterns("/rooms/(?P<room_id>[^/]*)/read_marker$")
+    PATTERNS = client_v2_patterns("/rooms/(?P<room_id>[^/]*)/read_markers$")
 
     def __init__(self, hs):
         super(ReadMarkerRestServlet, self).__init__()
@@ -51,7 +51,7 @@ class ReadMarkerRestServlet(RestServlet):
                 event_id=read_event_id
             )
 
-        read_marker_event_id = body.get("m.read_marker", None)
+        read_marker_event_id = body.get("m.fully_read", None)
         if read_marker_event_id:
             yield self.read_marker_handler.received_client_read_marker(
                 room_id,
-- 
cgit 1.4.1


From e6e262763663a7aebd3e38ee9763faeff64459c8 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 24 Apr 2017 18:51:25 +0100
Subject: Fix code for reporting old verify keys in synapse

---
 synapse/rest/key/v2/local_key_resource.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py
index ff95269ba8..be68d9a096 100644
--- a/synapse/rest/key/v2/local_key_resource.py
+++ b/synapse/rest/key/v2/local_key_resource.py
@@ -84,12 +84,11 @@ class LocalKey(Resource):
             }
 
         old_verify_keys = {}
-        for key in self.config.old_signing_keys:
-            key_id = "%s:%s" % (key.alg, key.version)
+        for key_id, key in self.config.old_signing_keys.items():
             verify_key_bytes = key.encode()
             old_verify_keys[key_id] = {
                 u"key": encode_base64(verify_key_bytes),
-                u"expired_ts": key.expired,
+                u"expired_ts": key.expired_ts,
             }
 
         tls_fingerprints = self.config.tls_fingerprints
-- 
cgit 1.4.1


From d9aa645f86db733bb0419b5f5428ba9a9c799735 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 25 Apr 2017 14:38:51 +0100
Subject: Reduce size of joined_user cache

The _get_joined_users_from_context cache stores a mapping from user_id
to avatar_url and display_name. Instead of storing those in a dict,
store them in a namedtuple as that uses much less memory.

We also try converting the string to ascii to further reduce the size.
---
 synapse/push/bulk_push_rule_evaluator.py |  7 +++++--
 synapse/rest/client/v1/room.py           |  8 +++++++-
 synapse/storage/roommember.py            | 22 ++++++++++++++--------
 synapse/util/stringutils.py              | 14 ++++++++++++++
 4 files changed, 40 insertions(+), 11 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 78b095c903..503fef4261 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -87,8 +87,11 @@ class BulkPushRuleEvaluator:
         condition_cache = {}
 
         for uid, rules in self.rules_by_user.items():
-            display_name = room_members.get(uid, {}).get("display_name", None)
-            if not display_name:
+            display_name = None
+            profile_info = room_members.get(uid, {})
+            if profile_info:
+                display_name = profile_info.display_name
+            else:
                 # Handle the case where we are pushing a membership event to
                 # that user, as they might not be already joined.
                 if event.type == EventTypes.Member and event.state_key == uid:
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 0bdd6b5b36..c376ab8fd7 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -406,7 +406,13 @@ class JoinedRoomMemberListRestServlet(ClientV1RestServlet):
         users_with_profile = yield self.state.get_current_user_in_room(room_id)
 
         defer.returnValue((200, {
-            "joined": users_with_profile
+            "joined": {
+                user_id: {
+                    "avatar_url": profile.avatar_url,
+                    "display_name": profile.display_name,
+                }
+                for user_id, profile in users_with_profile.iteritems()
+            }
         }))
 
 
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 367dbbbcf6..68e724ac7f 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -19,6 +19,7 @@ from collections import namedtuple
 
 from ._base import SQLBaseStore
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
+from synapse.util.stringutils import to_ascii
 
 from synapse.api.constants import Membership, EventTypes
 from synapse.types import get_domain_from_id
@@ -35,6 +36,11 @@ RoomsForUser = namedtuple(
 )
 
 
+ProfileInfo = namedtuple(
+    "ProfileInfo", ("avatar_url", "display_name")
+)
+
+
 _MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
 
 
@@ -422,20 +428,20 @@ class RoomMemberStore(SQLBaseStore):
         )
 
         users_in_room = {
-            row["user_id"]: {
-                "display_name": row["display_name"],
-                "avatar_url": row["avatar_url"],
-            }
+            to_ascii(row["user_id"]): ProfileInfo(
+                avatar_url=to_ascii(row["avatar_url"]),
+                display_name=to_ascii(row["display_name"]),
+            )
             for row in rows
         }
 
         if event is not None and event.type == EventTypes.Member:
             if event.membership == Membership.JOIN:
                 if event.event_id in member_event_ids:
-                    users_in_room[event.state_key] = {
-                        "display_name": event.content.get("displayname", None),
-                        "avatar_url": event.content.get("avatar_url", None),
-                    }
+                    users_in_room[to_ascii(event.state_key)] = ProfileInfo(
+                        display_name=to_ascii(event.content.get("displayname", None)),
+                        avatar_url=to_ascii(event.content.get("avatar_url", None)),
+                    )
 
         defer.returnValue(users_in_room)
 
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index a100f151d4..95a6168e16 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -40,3 +40,17 @@ def is_ascii(s):
         return False
     else:
         return True
+
+
+def to_ascii(s):
+    """Converts a string to ascii if it is ascii, otherwise leave it alone.
+
+    If given None then will return None.
+    """
+    if s is None:
+        return None
+
+    try:
+        return s.encode("ascii")
+    except UnicodeEncodeError:
+        return s
-- 
cgit 1.4.1


From 34e682d3855562a4ca6e32f366a2ecfa1c684a3a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 26 Apr 2017 16:18:08 +0100
Subject: Fix invite state to always include all events

---
 synapse/events/utils.py              | 20 +++++++++++++++++++-
 synapse/handlers/message.py          |  9 +++------
 synapse/rest/client/v2_alpha/sync.py |  2 ++
 3 files changed, 24 insertions(+), 7 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 5bbaef8187..7ca7796558 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -225,7 +225,22 @@ def format_event_for_client_v2_without_room_id(d):
 
 def serialize_event(e, time_now_ms, as_client_event=True,
                     event_format=format_event_for_client_v1,
-                    token_id=None, only_event_fields=None):
+                    token_id=None, only_event_fields=None, is_invite=False):
+    """Serialize event for clients
+
+    Args:
+        e (EventBase)
+        time_now_ms (int)
+        as_client_event (bool)
+        event_format
+        token_id
+        only_event_fields
+        is_invite (bool): Whether this is an invite that is being sent to the
+            invitee
+
+    Returns:
+        dict
+    """
     # FIXME(erikj): To handle the case of presence events and the like
     if not isinstance(e, EventBase):
         return e
@@ -260,4 +275,7 @@ def serialize_event(e, time_now_ms, as_client_event=True,
             raise TypeError("only_event_fields must be a list of strings")
         d = only_fields(d, only_event_fields)
 
+    if not is_invite:
+        d["unsigned"].pop("invite_room_state", None)
+
     return d
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 348056add5..82a2ade1f6 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -531,9 +531,9 @@ class MessageHandler(BaseHandler):
 
                 state_to_include_ids = [
                     e_id
-                    for k, e_id in context.current_state_ids.items()
+                    for k, e_id in context.current_state_ids.iteritems()
                     if k[0] in self.hs.config.room_invite_state_types
-                    or k[0] == EventTypes.Member and k[1] == event.sender
+                    or k == (EventTypes.Member, event.sender)
                 ]
 
                 state_to_include = yield self.store.get_events(state_to_include_ids)
@@ -545,7 +545,7 @@ class MessageHandler(BaseHandler):
                         "content": e.content,
                         "sender": e.sender,
                     }
-                    for e in state_to_include.values()
+                    for e in state_to_include.itervalues()
                 ]
 
                 invitee = UserID.from_string(event.state_key)
@@ -618,6 +618,3 @@ class MessageHandler(BaseHandler):
             )
 
         preserve_fn(_notify)()
-
-        # If invite, remove room_state from unsigned before sending.
-        event.unsigned.pop("invite_room_state", None)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index a7a9e0a794..b5bcfade70 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -250,9 +250,11 @@ class SyncRestServlet(RestServlet):
         """
         invited = {}
         for room in rooms:
+            logger.info("invite: %r", room.invite)
             invite = serialize_event(
                 room.invite, time_now, token_id=token_id,
                 event_format=format_event_for_client_v2_without_room_id,
+                is_invite=True,
             )
             unsigned = dict(invite.get("unsigned", {}))
             invite["unsigned"] = unsigned
-- 
cgit 1.4.1


From 46826bb078ac72ac8aea0d76c9e1c1f2ad29df97 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 27 Apr 2017 17:25:44 +0100
Subject: Comment and remove spurious logging

---
 synapse/events/utils.py              | 3 +++
 synapse/rest/client/v2_alpha/sync.py | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index c42b73d11a..824f4a42e3 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -266,6 +266,9 @@ def serialize_event(e, time_now_ms, as_client_event=True,
             if txn_id is not None:
                 d["unsigned"]["transaction_id"] = txn_id
 
+    # If this is an invite for somebody else, then we don't care about the
+    # invite_room_state as that's meant solely for the invitee. Other clients
+    # will already have the state since they're in the room.
     if not is_invite:
         d["unsigned"].pop("invite_room_state", None)
 
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index b5bcfade70..f30eab76fd 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -250,7 +250,6 @@ class SyncRestServlet(RestServlet):
         """
         invited = {}
         for room in rooms:
-            logger.info("invite: %r", room.invite)
             invite = serialize_event(
                 room.invite, time_now, token_id=token_id,
                 event_format=format_event_for_client_v2_without_room_id,
-- 
cgit 1.4.1


From 3e5a62ecd8839fbfb56aa33b92127822a053ef6d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 2 May 2017 11:36:11 +0100
Subject: Add more granular event send metrics

---
 synapse/events/snapshot.py      |  3 +++
 synapse/handlers/message.py     | 10 ++++++++--
 synapse/handlers/room_member.py |  1 +
 synapse/rest/client/v1/room.py  |  1 +
 synapse/storage/events.py       | 16 ++++++++++++++++
 tests/storage/event_injector.py |  4 ++--
 tests/storage/test_events.py    |  2 +-
 7 files changed, 32 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py
index 6be18880b9..e9a732ff03 100644
--- a/synapse/events/snapshot.py
+++ b/synapse/events/snapshot.py
@@ -50,6 +50,7 @@ class EventContext(object):
         "prev_group",
         "delta_ids",
         "prev_state_events",
+        "app_service",
     ]
 
     def __init__(self):
@@ -68,3 +69,5 @@ class EventContext(object):
         self.delta_ids = None
 
         self.prev_state_events = None
+
+        self.app_service = None
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 82a2ade1f6..57265c6d7d 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -175,7 +175,8 @@ class MessageHandler(BaseHandler):
         defer.returnValue(chunk)
 
     @defer.inlineCallbacks
-    def create_event(self, event_dict, token_id=None, txn_id=None, prev_event_ids=None):
+    def create_event(self, requester, event_dict, token_id=None, txn_id=None,
+                     prev_event_ids=None):
         """
         Given a dict from a client, create a new event.
 
@@ -185,6 +186,7 @@ class MessageHandler(BaseHandler):
         Adds display names to Join membership events.
 
         Args:
+            requester
             event_dict (dict): An entire event
             token_id (str)
             txn_id (str)
@@ -226,6 +228,7 @@ class MessageHandler(BaseHandler):
 
             event, context = yield self._create_new_client_event(
                 builder=builder,
+                requester=requester,
                 prev_event_ids=prev_event_ids,
             )
 
@@ -319,6 +322,7 @@ class MessageHandler(BaseHandler):
         See self.create_event and self.send_nonmember_event.
         """
         event, context = yield self.create_event(
+            requester,
             event_dict,
             token_id=requester.access_token_id,
             txn_id=txn_id
@@ -416,7 +420,7 @@ class MessageHandler(BaseHandler):
 
     @measure_func("_create_new_client_event")
     @defer.inlineCallbacks
-    def _create_new_client_event(self, builder, prev_event_ids=None):
+    def _create_new_client_event(self, builder, requester=None, prev_event_ids=None):
         if prev_event_ids:
             prev_events = yield self.store.add_event_hashes(prev_event_ids)
             prev_max_depth = yield self.store.get_max_depth_of_events(prev_event_ids)
@@ -456,6 +460,8 @@ class MessageHandler(BaseHandler):
         state_handler = self.state_handler
 
         context = yield state_handler.compute_event_context(builder)
+        if requester:
+            context.app_service = requester.app_service
 
         if builder.is_state():
             builder.prev_state = yield self.store.add_event_hashes(
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 28b2c80a93..ab87632d99 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -70,6 +70,7 @@ class RoomMemberHandler(BaseHandler):
             content["kind"] = "guest"
 
         event, context = yield msg_handler.create_event(
+            requester,
             {
                 "type": EventTypes.Member,
                 "content": content,
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index c376ab8fd7..cd388770c8 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -164,6 +164,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
         else:
             msg_handler = self.handlers.message_handler
             event, context = yield msg_handler.create_event(
+                requester,
                 event_dict,
                 token_id=requester.access_token_id,
                 txn_id=txn_id,
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index a3790419dd..98707d40ee 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -29,6 +29,7 @@ from synapse.api.constants import EventTypes
 from synapse.api.errors import SynapseError
 from synapse.state import resolve_events
 from synapse.util.caches.descriptors import cached
+from synapse.types import get_domain_from_id
 
 from canonicaljson import encode_canonical_json
 from collections import deque, namedtuple, OrderedDict
@@ -49,6 +50,9 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 persist_event_counter = metrics.register_counter("persisted_events")
+event_counter = metrics.register_counter(
+    "persisted_events_sep", labels=["type", "origin_type", "origin_entity"]
+)
 
 
 def encode_json(json_object):
@@ -370,6 +374,18 @@ class EventsStore(SQLBaseStore):
                     new_forward_extremeties=new_forward_extremeties,
                 )
                 persist_event_counter.inc_by(len(chunk))
+                for event, context in chunk:
+                    if context.app_service:
+                        origin_type = "local"
+                        origin_entity = context.app_service.id
+                    elif self.hs.is_mine_id(event.sender):
+                        origin_type = "local"
+                        origin_entity = "*client*"
+                    else:
+                        origin_type = "remote"
+                        origin_entity = get_domain_from_id(event.sender)
+
+                    event_counter.inc(event.type, origin_type, origin_entity)
 
     @defer.inlineCallbacks
     def _calculate_new_extremeties(self, room_id, event_contexts, latest_event_ids):
diff --git a/tests/storage/event_injector.py b/tests/storage/event_injector.py
index 38556da9a7..024ac15069 100644
--- a/tests/storage/event_injector.py
+++ b/tests/storage/event_injector.py
@@ -27,10 +27,10 @@ class EventInjector:
         self.event_builder_factory = hs.get_event_builder_factory()
 
     @defer.inlineCallbacks
-    def create_room(self, room):
+    def create_room(self, room, user):
         builder = self.event_builder_factory.new({
             "type": EventTypes.Create,
-            "sender": "",
+            "sender": user.to_string(),
             "room_id": room.to_string(),
             "content": {},
         })
diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py
index 3762b38e37..14443b53bc 100644
--- a/tests/storage/test_events.py
+++ b/tests/storage/test_events.py
@@ -50,7 +50,7 @@ class EventsStoreTestCase(unittest.TestCase):
         # Create something to report
         room = RoomID.from_string("!abc123:test")
         user = UserID.from_string("@raccoonlover:test")
-        yield self.event_injector.create_room(room)
+        yield self.event_injector.create_room(room, user)
 
         self.base_event = yield self._get_last_stream_token()
 
-- 
cgit 1.4.1


From 34ed4f4206b4ff6830c381beabbaa7739b1a63f8 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 3 May 2017 11:55:44 +0100
Subject: Implement username availability checker

Outlined here: https://github.com/vector-im/riot-web/issues/3605#issuecomment-298679388

```HTTP
GET /_matrix/.../register/available
{
    "username": "desiredlocalpart123"
}
```

If available, the response looks like
```HTTP
HTTP/1.1 200 OK
{
    "available": true
}
```

Otherwise,
```HTTP
HTTP/1.1 429
{
    "errcode": "M_LIMIT_EXCEEDED",
    "error": "Too Many Requests",
    "retry_after_ms": 2000
}
```
or
```HTTP
HTTP/1.1 400
{
    "errcode": "M_USER_IN_USE",
    "error": "User ID already taken."
}

```
or
```HTTP
HTTP/1.1 400
{
    "errcode": "M_INVALID_USERNAME",
    "error": "Some reason for username being invalid"
}
```
---
 synapse/rest/client/v2_alpha/register.py | 36 ++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 3acf4eacdd..49fee04ec8 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -31,6 +31,7 @@ import logging
 import hmac
 from hashlib import sha1
 from synapse.util.async import run_on_reactor
+from synapse.util.ratelimitutils import FederationRateLimiter
 
 
 # We ought to be using hmac.compare_digest() but on older pythons it doesn't
@@ -115,6 +116,40 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet):
         defer.returnValue((200, ret))
 
 
+class UsernameAvailabilityRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/register/available")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(UsernameAvailabilityRestServlet, self).__init__()
+        self.hs = hs
+        self.registration_handler = hs.get_handlers().registration_handler
+        self.ratelimiter = FederationRateLimiter(
+            hs.get_clock(),
+            window_size=2000, # Time window of 2s
+            sleep_limit=1, # Artificially delay requests if rate > sleep_limit/window_size
+            sleep_msec=1000, # Amount of artificial delay to apply
+            reject_limit=1, # Error with 429 if more than reject_limit requests are queued
+            concurrent_requests=1, # Allow 1 request at a time
+        )
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        ip = self.hs.get_ip_from_request(request)
+        with self.ratelimiter.ratelimit(ip) as wait_deferred:
+            yield wait_deferred
+
+            body = parse_json_object_from_request(request)
+            assert_params_in_request(body, ['username'])
+
+            yield self.registration_handler.check_username(body['username'])
+
+            defer.returnValue((200, {"available": True}))
+
+
 class RegisterRestServlet(RestServlet):
     PATTERNS = client_v2_patterns("/register$")
 
@@ -555,4 +590,5 @@ class RegisterRestServlet(RestServlet):
 def register_servlets(hs, http_server):
     EmailRegisterRequestTokenRestServlet(hs).register(http_server)
     MsisdnRegisterRequestTokenRestServlet(hs).register(http_server)
+    UsernameAvailabilityRestServlet(hs).register(http_server)
     RegisterRestServlet(hs).register(http_server)
-- 
cgit 1.4.1


From 3669065466ea35d5337631a7a089c03a65e9ddb8 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 3 May 2017 18:05:49 +0100
Subject: Appease the flake8 gods

---
 synapse/rest/client/v2_alpha/register.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 49fee04ec8..38a739f2f8 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -129,11 +129,16 @@ class UsernameAvailabilityRestServlet(RestServlet):
         self.registration_handler = hs.get_handlers().registration_handler
         self.ratelimiter = FederationRateLimiter(
             hs.get_clock(),
-            window_size=2000, # Time window of 2s
-            sleep_limit=1, # Artificially delay requests if rate > sleep_limit/window_size
-            sleep_msec=1000, # Amount of artificial delay to apply
-            reject_limit=1, # Error with 429 if more than reject_limit requests are queued
-            concurrent_requests=1, # Allow 1 request at a time
+            # Time window of 2s
+            window_size=2000,
+            # Artificially delay requests if rate > sleep_limit/window_size
+            sleep_limit=1,
+            # Amount of artificial delay to apply
+            sleep_msec=1000,
+            # Error with 429 if more than reject_limit requests are queued
+            reject_limit=1,
+            # Allow 1 request at a time
+            concurrent_requests=1,
         )
 
     @defer.inlineCallbacks
-- 
cgit 1.4.1


From f7278e612e0238d918c2a0d71ca3fb8fb499c765 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 10 May 2017 11:40:18 +0100
Subject: Change register/available to POST (from GET)

---
 synapse/rest/client/v2_alpha/register.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 38a739f2f8..6a7cd96ea5 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -142,7 +142,7 @@ class UsernameAvailabilityRestServlet(RestServlet):
         )
 
     @defer.inlineCallbacks
-    def on_GET(self, request):
+    def on_POST(self, request):
         ip = self.hs.get_ip_from_request(request)
         with self.ratelimiter.ratelimit(ip) as wait_deferred:
             yield wait_deferred
-- 
cgit 1.4.1


From 369195caa5f52fc67c2496507fc99fccf4b0ede8 Mon Sep 17 00:00:00 2001
From: Luke Barnard <lukeb@openmarket.com>
Date: Wed, 10 May 2017 17:17:12 +0100
Subject: Modify register/available to be GET with query param

- GET is now the method for register/available
- a query parameter "username" is now used

Also, empty usernames are now handled with an error message on registration or via register/available: `User ID cannot be empty`
---
 synapse/handlers/register.py             | 7 +++++++
 synapse/rest/client/v2_alpha/register.py | 9 ++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 03c6a85fc6..dd84c5f5e9 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -54,6 +54,13 @@ class RegistrationHandler(BaseHandler):
                 Codes.INVALID_USERNAME
             )
 
+        if len(localpart) == 0:
+            raise SynapseError(
+                400,
+                "User ID cannot be empty",
+                Codes.INVALID_USERNAME
+            )
+
         if localpart[0] == '_':
             raise SynapseError(
                 400,
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 6a7cd96ea5..1421c18152 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -21,7 +21,7 @@ from synapse.api.auth import get_access_token_from_request, has_access_token
 from synapse.api.constants import LoginType
 from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
 from synapse.http.servlet import (
-    RestServlet, parse_json_object_from_request, assert_params_in_request
+    RestServlet, parse_json_object_from_request, assert_params_in_request, parse_string
 )
 from synapse.util.msisdn import phone_number_to_msisdn
 
@@ -142,15 +142,14 @@ class UsernameAvailabilityRestServlet(RestServlet):
         )
 
     @defer.inlineCallbacks
-    def on_POST(self, request):
+    def on_GET(self, request):
         ip = self.hs.get_ip_from_request(request)
         with self.ratelimiter.ratelimit(ip) as wait_deferred:
             yield wait_deferred
 
-            body = parse_json_object_from_request(request)
-            assert_params_in_request(body, ['username'])
+            username = parse_string(request, "username", required=True)
 
-            yield self.registration_handler.check_username(body['username'])
+            yield self.registration_handler.check_username(username)
 
             defer.returnValue((200, {"available": True}))
 
-- 
cgit 1.4.1


From 9da4316ca574f2d552136941d92ce722fabfe29e Mon Sep 17 00:00:00 2001
From: Pablo Saavedra <psaavedra@igalia.com>
Date: Sat, 13 May 2017 18:17:54 +0200
Subject: Configurable maximum number of events requested by /sync and
 /messages (#2220)

Set the limit on the returned events in the timeline in the get and sync
operations. The default value is -1, means no upper limit.

For example, using `filter_timeline_limit: 5000`:

POST /_matrix/client/r0/user/user:id/filter
{
room: {
    timeline: {
      limit: 1000000000000000000
    }
}
}

GET /_matrix/client/r0/user/user:id/filter/filter:id

{
room: {
    timeline: {
      limit: 5000
    }
}
}

The server cuts down the room.timeline.limit.
---
 synapse/config/server.py               | 6 ++++++
 synapse/rest/client/v2_alpha/_base.py  | 8 ++++++++
 synapse/rest/client/v2_alpha/filter.py | 4 ++++
 synapse/rest/client/v2_alpha/sync.py   | 3 +++
 4 files changed, 21 insertions(+)

(limited to 'synapse/rest')

diff --git a/synapse/config/server.py b/synapse/config/server.py
index 25e6666238..3910b9dc31 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -35,6 +35,8 @@ class ServerConfig(Config):
         # "disable" federation
         self.send_federation = config.get("send_federation", True)
 
+        self.filter_timeline_limit = config.get("filter_timeline_limit", -1)
+
         if self.public_baseurl is not None:
             if self.public_baseurl[-1] != '/':
                 self.public_baseurl += '/'
@@ -161,6 +163,10 @@ class ServerConfig(Config):
         # The GC threshold parameters to pass to `gc.set_threshold`, if defined
         # gc_thresholds: [700, 10, 10]
 
+        # Set the limit on the returned events in the timeline in the get
+        # and sync operations. The default value is -1, means no upper limit.
+        # filter_timeline_limit: 5000
+
         # List of ports that Synapse should listen on, their purpose and their
         # configuration.
         listeners:
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 20e765f48f..279e2a7751 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -47,3 +47,11 @@ def client_v2_patterns(path_regex, releases=(0,),
         new_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/r%d" % release)
         patterns.append(re.compile("^" + new_prefix + path_regex))
     return patterns
+
+
+def set_timeline_upper_limit(filter_json, filter_timeline_limit):
+    if filter_timeline_limit < 0:
+        return  # no upper limits
+    if 'room' in filter_json and 'limit' in filter_json['room']:
+        filter_json['room']["limit"] = min(filter_json['room']["limit"],
+                                           filter_timeline_limit)
diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py
index b4084fec62..364d20d8e6 100644
--- a/synapse/rest/client/v2_alpha/filter.py
+++ b/synapse/rest/client/v2_alpha/filter.py
@@ -20,6 +20,7 @@ from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.types import UserID
 
 from ._base import client_v2_patterns
+from ._base import set_timeline_upper_limit
 
 import logging
 
@@ -85,6 +86,9 @@ class CreateFilterRestServlet(RestServlet):
             raise AuthError(403, "Can only create filters for local users")
 
         content = parse_json_object_from_request(request)
+        set_timeline_upper_limit(content,
+                                 self.hs.config.filter_timeline_limit)
+
         filter_id = yield self.filtering.add_user_filter(
             user_localpart=target_user.localpart,
             user_filter=content,
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index f30eab76fd..f5e7349c5c 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -28,6 +28,7 @@ from synapse.api.filtering import FilterCollection, DEFAULT_FILTER_COLLECTION
 from synapse.api.errors import SynapseError
 from synapse.api.constants import PresenceState
 from ._base import client_v2_patterns
+from ._base import set_timeline_upper_limit
 
 import itertools
 import logging
@@ -121,6 +122,8 @@ class SyncRestServlet(RestServlet):
             if filter_id.startswith('{'):
                 try:
                     filter_object = json.loads(filter_id)
+                    set_timeline_upper_limit(filter_object,
+                                             self.hs.config.filter_timeline_limit)
                 except:
                     raise SynapseError(400, "Invalid filter JSON")
                 self.filtering.check_valid_filter(filter_object)
-- 
cgit 1.4.1


From 627e6ea2b0b941c67f8752736993f82a5f123e76 Mon Sep 17 00:00:00 2001
From: Pablo Saavedra <psaavedra@igalia.com>
Date: Mon, 15 May 2017 14:51:43 +0200
Subject: Fixed implementation errors
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Added HS as property in SyncRestServlet
* Fixed set_timeline_upper_limit function implementat¡ion
---
 synapse/rest/client/v2_alpha/_base.py | 9 ++++++---
 synapse/rest/client/v2_alpha/sync.py  | 1 +
 2 files changed, 7 insertions(+), 3 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 279e2a7751..fd8f915655 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -52,6 +52,9 @@ def client_v2_patterns(path_regex, releases=(0,),
 def set_timeline_upper_limit(filter_json, filter_timeline_limit):
     if filter_timeline_limit < 0:
         return  # no upper limits
-    if 'room' in filter_json and 'limit' in filter_json['room']:
-        filter_json['room']["limit"] = min(filter_json['room']["limit"],
-                                           filter_timeline_limit)
+    if 'room' in filter_json \
+            and 'timeline' in filter_json['room'] \
+            and 'limit' in filter_json['room']['timeline']:
+        filter_json['room']['timeline']["limit"] = min(
+            filter_json['room']['timeline']['limit'],
+            filter_timeline_limit)
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index f5e7349c5c..771e127ab9 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -79,6 +79,7 @@ class SyncRestServlet(RestServlet):
 
     def __init__(self, hs):
         super(SyncRestServlet, self).__init__()
+        self.hs = hs
         self.auth = hs.get_auth()
         self.sync_handler = hs.get_sync_handler()
         self.clock = hs.get_clock()
-- 
cgit 1.4.1


From 224137fcf98b424c6b5d1ab4b76c24b6213c7174 Mon Sep 17 00:00:00 2001
From: Pablo Saavedra <psaavedra@igalia.com>
Date: Mon, 15 May 2017 16:21:02 +0200
Subject: Fixed syntax nits

---
 synapse/rest/client/v2_alpha/_base.py  | 5 ++---
 synapse/rest/client/v2_alpha/filter.py | 6 ++++--
 2 files changed, 6 insertions(+), 5 deletions(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index fd8f915655..1f5bc24cc3 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -52,9 +52,8 @@ def client_v2_patterns(path_regex, releases=(0,),
 def set_timeline_upper_limit(filter_json, filter_timeline_limit):
     if filter_timeline_limit < 0:
         return  # no upper limits
-    if 'room' in filter_json \
-            and 'timeline' in filter_json['room'] \
-            and 'limit' in filter_json['room']['timeline']:
+    timeline = filter_json.get('room', {}).get('timeline', {})
+    if 'limit' in timeline:
         filter_json['room']['timeline']["limit"] = min(
             filter_json['room']['timeline']['limit'],
             filter_timeline_limit)
diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py
index 364d20d8e6..d2b2fd66e6 100644
--- a/synapse/rest/client/v2_alpha/filter.py
+++ b/synapse/rest/client/v2_alpha/filter.py
@@ -86,8 +86,10 @@ class CreateFilterRestServlet(RestServlet):
             raise AuthError(403, "Can only create filters for local users")
 
         content = parse_json_object_from_request(request)
-        set_timeline_upper_limit(content,
-                                 self.hs.config.filter_timeline_limit)
+        set_timeline_upper_limit(
+            content,
+            self.hs.config.filter_timeline_limit
+        )
 
         filter_id = yield self.filtering.add_user_filter(
             user_localpart=target_user.localpart,
-- 
cgit 1.4.1


From d12ae7fd1c213c2505e3904aab98604fa86f42f5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 15 May 2017 15:42:18 +0100
Subject: Don't log exceptions for NotRetryingDestination

---
 synapse/rest/media/v1/media_repository.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'synapse/rest')

diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index c43b185e08..caca96c222 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -34,6 +34,7 @@ from synapse.api.errors import SynapseError, HttpResponseException, \
 from synapse.util.async import Linearizer
 from synapse.util.stringutils import is_ascii
 from synapse.util.logcontext import preserve_context_over_fn
+from synapse.util.retryutils import NotRetryingDestination
 
 import os
 import errno
@@ -181,7 +182,8 @@ class MediaRepository(object):
                     logger.exception("Failed to fetch remote media %s/%s",
                                      server_name, media_id)
                     raise
-
+                except NotRetryingDestination:
+                    logger.warn("Not retrying destination %r", server_name)
                 except Exception:
                     logger.exception("Failed to fetch remote media %s/%s",
                                      server_name, media_id)
-- 
cgit 1.4.1