Merge remote-tracking branch 'origin/release-v0.12.0' v0.12.0

12 files changed, 201 insertions, 75 deletions

diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py
index a108132346..c488b10d3c 100644
--- a/synapse/rest/client/v2_alpha/__init__.py
+++ b/synapse/rest/client/v2_alpha/__init__.py
@@ -12,37 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from . import (
-    sync,
-    filter,
-    account,
-    register,
-    auth,
-    receipts,
-    keys,
-    tokenrefresh,
-    tags,
-)
-
-from synapse.http.server import JsonResource
-
-
-class ClientV2AlphaRestResource(JsonResource):
-    """A resource for version 2 alpha of the matrix client API."""
-
-    def __init__(self, hs):
-        JsonResource.__init__(self, hs, canonical_json=False)
-        self.register_servlets(self, hs)
-
-    @staticmethod
-    def register_servlets(client_resource, hs):
-        sync.register_servlets(hs, client_resource)
-        filter.register_servlets(hs, client_resource)
-        account.register_servlets(hs, client_resource)
-        register.register_servlets(hs, client_resource)
-        auth.register_servlets(hs, client_resource)
-        receipts.register_servlets(hs, client_resource)
-        keys.register_servlets(hs, client_resource)
-        tokenrefresh.register_servlets(hs, client_resource)
-        tags.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 4540e8dcf7..7b8b879c03 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -27,7 +27,7 @@ import simplejson
 logger = logging.getLogger(__name__)
 
 
-def client_v2_pattern(path_regex):
+def client_v2_patterns(path_regex, releases=(0,)):
     """Creates a regex compiled client path with the correct client path
     prefix.
 
@@ -37,7 +37,13 @@ def client_v2_pattern(path_regex):
     Returns:
         SRE_Pattern
     """
-    return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex)
+    patterns = [re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex)]
+    unstable_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/unstable")
+    patterns.append(re.compile("^" + unstable_prefix + path_regex))
+    for release in releases:
+        new_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/r%d" % release)
+        patterns.append(re.compile("^" + new_prefix + path_regex))
+    return patterns
 
 
 def parse_request_allow_empty(request):
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 1970ad3458..3e1459d5b9 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -20,7 +20,7 @@ from synapse.api.errors import LoginError, SynapseError, Codes
 from synapse.http.servlet import RestServlet
 from synapse.util.async import run_on_reactor
 
-from ._base import client_v2_pattern, parse_json_dict_from_request
+from ._base import client_v2_patterns, parse_json_dict_from_request
 
 import logging
 
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 
 class PasswordRestServlet(RestServlet):
-    PATTERN = client_v2_pattern("/account/password")
+    PATTERNS = client_v2_patterns("/account/password")
 
     def __init__(self, hs):
         super(PasswordRestServlet, self).__init__()
@@ -89,7 +89,7 @@ class PasswordRestServlet(RestServlet):
 
 
 class ThreepidRestServlet(RestServlet):
-    PATTERN = client_v2_pattern("/account/3pid")
+    PATTERNS = client_v2_patterns("/account/3pid")
 
     def __init__(self, hs):
         super(ThreepidRestServlet, self).__init__()
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
new file mode 100644
index 0000000000..5b8f454bf1
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import client_v2_patterns
+
+from synapse.http.servlet import RestServlet
+from synapse.api.errors import AuthError, SynapseError
+
+from twisted.internet import defer
+
+import logging
+
+import simplejson as json
+
+logger = logging.getLogger(__name__)
+
+
+class AccountDataServlet(RestServlet):
+    """
+    PUT /user/{user_id}/account_data/{account_dataType} HTTP/1.1
+    """
+    PATTERNS = client_v2_patterns(
+        "/user/(?P<user_id>[^/]*)/account_data/(?P<account_data_type>[^/]*)"
+    )
+
+    def __init__(self, hs):
+        super(AccountDataServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+        self.notifier = hs.get_notifier()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, user_id, account_data_type):
+        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        if user_id != auth_user.to_string():
+            raise AuthError(403, "Cannot add account data for other users.")
+
+        try:
+            content_bytes = request.content.read()
+            body = json.loads(content_bytes)
+        except:
+            raise SynapseError(400, "Invalid JSON")
+
+        max_id = yield self.store.add_account_data_for_user(
+            user_id, account_data_type, body
+        )
+
+        yield self.notifier.on_new_event(
+            "account_data_key", max_id, users=[user_id]
+        )
+
+        defer.returnValue((200, {}))
+
+
+class RoomAccountDataServlet(RestServlet):
+    """
+    PUT /user/{user_id}/rooms/{room_id}/account_data/{account_dataType} HTTP/1.1
+    """
+    PATTERNS = client_v2_patterns(
+        "/user/(?P<user_id>[^/]*)"
+        "/rooms/(?P<room_id>[^/]*)"
+        "/account_data/(?P<account_data_type>[^/]*)"
+    )
+
+    def __init__(self, hs):
+        super(RoomAccountDataServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+        self.notifier = hs.get_notifier()
+
+    @defer.inlineCallbacks
+    def on_PUT(self, request, user_id, room_id, account_data_type):
+        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        if user_id != auth_user.to_string():
+            raise AuthError(403, "Cannot add account data for other users.")
+
+        try:
+            content_bytes = request.content.read()
+            body = json.loads(content_bytes)
+        except:
+            raise SynapseError(400, "Invalid JSON")
+
+        if not isinstance(body, dict):
+            raise ValueError("Expected a JSON object")
+
+        max_id = yield self.store.add_account_data_to_room(
+            user_id, room_id, account_data_type, body
+        )
+
+        yield self.notifier.on_new_event(
+            "account_data_key", max_id, users=[user_id]
+        )
+
+        defer.returnValue((200, {}))
+
+
+def register_servlets(hs, http_server):
+    AccountDataServlet(hs).register(http_server)
+    RoomAccountDataServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py
index 4c726f05f5..fb5947a141 100644
--- a/synapse/rest/client/v2_alpha/auth.py
+++ b/synapse/rest/client/v2_alpha/auth.py
@@ -20,7 +20,7 @@ from synapse.api.errors import SynapseError
 from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX
 from synapse.http.servlet import RestServlet
 
-from ._base import client_v2_pattern
+from ._base import client_v2_patterns
 
 import logging
 
@@ -97,7 +97,7 @@ class AuthRestServlet(RestServlet):
     cannot be handled in the normal flow (with requests to the same endpoint).
     Current use is for web fallback auth.
     """
-    PATTERN = client_v2_pattern("/auth/(?P<stagetype>[\w\.]*)/fallback/web")
+    PATTERNS = client_v2_patterns("/auth/(?P<stagetype>[\w\.]*)/fallback/web")
 
     def __init__(self, hs):
         super(AuthRestServlet, self).__init__()
diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py
index 97956a4b91..3cd0364b56 100644
--- a/synapse/rest/client/v2_alpha/filter.py
+++ b/synapse/rest/client/v2_alpha/filter.py
@@ -19,7 +19,7 @@ from synapse.api.errors import AuthError, SynapseError
 from synapse.http.servlet import RestServlet
 from synapse.types import UserID
 
-from ._base import client_v2_pattern
+from ._base import client_v2_patterns
 
 import simplejson as json
 import logging
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 
 class GetFilterRestServlet(RestServlet):
-    PATTERN = client_v2_pattern("/user/(?P<user_id>[^/]*)/filter/(?P<filter_id>[^/]*)")
+    PATTERNS = client_v2_patterns("/user/(?P<user_id>[^/]*)/filter/(?P<filter_id>[^/]*)")
 
     def __init__(self, hs):
         super(GetFilterRestServlet, self).__init__()
@@ -65,7 +65,7 @@ class GetFilterRestServlet(RestServlet):
 
 
 class CreateFilterRestServlet(RestServlet):
-    PATTERN = client_v2_pattern("/user/(?P<user_id>[^/]*)/filter")
+    PATTERNS = client_v2_patterns("/user/(?P<user_id>[^/]*)/filter")
 
     def __init__(self, hs):
         super(CreateFilterRestServlet, self).__init__()
diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py
index 820d33336f..753f2988a1 100644
--- a/synapse/rest/client/v2_alpha/keys.py
+++ b/synapse/rest/client/v2_alpha/keys.py
@@ -21,7 +21,7 @@ from synapse.types import UserID
 
 from canonicaljson import encode_canonical_json
 
-from ._base import client_v2_pattern
+from ._base import client_v2_patterns
 
 import simplejson as json
 import logging
@@ -54,7 +54,7 @@ class KeyUploadServlet(RestServlet):
       },
     }
     """
-    PATTERN = client_v2_pattern("/keys/upload/(?P<device_id>[^/]*)")
+    PATTERNS = client_v2_patterns("/keys/upload/(?P<device_id>[^/]*)", releases=())
 
     def __init__(self, hs):
         super(KeyUploadServlet, self).__init__()
@@ -154,12 +154,13 @@ class KeyQueryServlet(RestServlet):
     } } } } } }
     """
 
-    PATTERN = client_v2_pattern(
+    PATTERNS = client_v2_patterns(
         "/keys/query(?:"
         "/(?P<user_id>[^/]*)(?:"
         "/(?P<device_id>[^/]*)"
         ")?"
-        ")?"
+        ")?",
+        releases=()
     )
 
     def __init__(self, hs):
@@ -245,10 +246,11 @@ class OneTimeKeyServlet(RestServlet):
     } } } }
 
     """
-    PATTERN = client_v2_pattern(
+    PATTERNS = client_v2_patterns(
         "/keys/claim(?:/?|(?:/"
         "(?P<user_id>[^/]*)/(?P<device_id>[^/]*)/(?P<algorithm>[^/]*)"
-        ")?)"
+        ")?)",
+        releases=()
     )
 
     def __init__(self, hs):
diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py
index 788acd4adb..aa214e13b6 100644
--- a/synapse/rest/client/v2_alpha/receipts.py
+++ b/synapse/rest/client/v2_alpha/receipts.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from synapse.api.errors import SynapseError
 from synapse.http.servlet import RestServlet
-from ._base import client_v2_pattern
+from ._base import client_v2_patterns
 
 import logging
 
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 
 class ReceiptRestServlet(RestServlet):
-    PATTERN = client_v2_pattern(
+    PATTERNS = client_v2_patterns(
         "/rooms/(?P<room_id>[^/]*)"
         "/receipt/(?P<receipt_type>[^/]*)"
         "/(?P<event_id>[^/]*)$"
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index f899376311..b2b89652c6 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -19,7 +19,7 @@ from synapse.api.constants import LoginType
 from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
 from synapse.http.servlet import RestServlet
 
-from ._base import client_v2_pattern, parse_json_dict_from_request
+from ._base import client_v2_patterns, parse_json_dict_from_request
 
 import logging
 import hmac
@@ -41,7 +41,7 @@ logger = logging.getLogger(__name__)
 
 
 class RegisterRestServlet(RestServlet):
-    PATTERN = client_v2_pattern("/register")
+    PATTERNS = client_v2_patterns("/register")
 
     def __init__(self, hs):
         super(RegisterRestServlet, self).__init__()
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index 775f49885b..35a70ffad1 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -25,11 +25,14 @@ from synapse.events.utils import (
     serialize_event, format_event_for_client_v2_without_room_id,
 )
 from synapse.api.filtering import FilterCollection
-from ._base import client_v2_pattern
+from synapse.api.errors import SynapseError
+from ._base import client_v2_patterns
 
 import copy
 import logging
 
+import ujson as json
+
 logger = logging.getLogger(__name__)
 
 
@@ -48,7 +51,7 @@ class SyncRestServlet(RestServlet):
           "next_batch": // batch token for the next /sync
           "presence": // presence data for the user.
           "rooms": {
-            "joined": { // Joined rooms being updated.
+            "join": { // Joined rooms being updated.
               "${room_id}": { // Id of the room being updated
                 "event_map": // Map of EventID -> event JSON.
                 "timeline": { // The recent events in the room if gap is "true"
@@ -63,13 +66,13 @@ class SyncRestServlet(RestServlet):
                 "ephemeral": {"events": []} // list of event objects
               }
             },
-            "invited": {}, // Invited rooms being updated.
-            "archived": {} // Archived rooms being updated.
+            "invite": {}, // Invited rooms being updated.
+            "leave": {} // Archived rooms being updated.
           }
         }
     """
 
-    PATTERN = client_v2_pattern("/sync$")
+    PATTERNS = client_v2_patterns("/sync$")
     ALLOWED_PRESENCE = set(["online", "offline"])
 
     def __init__(self, hs):
@@ -82,7 +85,9 @@ class SyncRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        user, token_id, _ = yield self.auth.get_user_by_req(request)
+        user, token_id, is_guest = yield self.auth.get_user_by_req(
+            request, allow_guest=True
+        )
 
         timeout = parse_integer(request, "timeout", default=0)
         since = parse_string(request, "since")
@@ -100,15 +105,29 @@ class SyncRestServlet(RestServlet):
             )
         )
 
-        try:
-            filter = yield self.filtering.get_user_filter(
-                user.localpart, filter_id
+        if filter_id and filter_id.startswith('{'):
+            try:
+                filter_object = json.loads(filter_id)
+            except:
+                raise SynapseError(400, "Invalid filter JSON")
+            self.filtering._check_valid_filter(filter_object)
+            filter = FilterCollection(filter_object)
+        else:
+            try:
+                filter = yield self.filtering.get_user_filter(
+                    user.localpart, filter_id
+                )
+            except:
+                filter = FilterCollection({})
+
+        if is_guest and filter.list_rooms() is None:
+            raise SynapseError(
+                400, "Guest users must provide a list of rooms in the filter"
             )
-        except:
-            filter = FilterCollection({})
 
         sync_config = SyncConfig(
             user=user,
+            is_guest=is_guest,
             filter=filter,
         )
 
@@ -144,6 +163,9 @@ class SyncRestServlet(RestServlet):
         )
 
         response_content = {
+            "account_data": self.encode_account_data(
+                sync_result.account_data, filter, time_now
+            ),
             "presence": self.encode_presence(
                 sync_result.presence, filter, time_now
             ),
@@ -165,6 +187,9 @@ class SyncRestServlet(RestServlet):
             formatted.append(event)
         return {"events": filter.filter_presence(formatted)}
 
+    def encode_account_data(self, events, filter, time_now):
+        return {"events": filter.filter_account_data(events)}
+
     def encode_joined(self, rooms, filter, time_now, token_id):
         """
         Encode the joined rooms in a sync result
@@ -333,20 +358,36 @@ class SyncRestServlet(RestServlet):
                 continue
 
             prev_event_id = timeline_event.unsigned.get("replaces_state", None)
-            logger.debug("Replacing %s with %s in state dict",
-                         timeline_event.event_id, prev_event_id)
 
-            if prev_event_id is None:
+            prev_content = timeline_event.unsigned.get('prev_content')
+            prev_sender = timeline_event.unsigned.get('prev_sender')
+            # Empircally it seems possible for the event to have a
+            # "replaces_state" key but not a prev_content or prev_sender
+            # markjh conjectures that it could be due to the server not
+            # having a copy of that event.
+            # If this is the case the we ignore the previous event. This will
+            # cause the displayname calculations on the client to be incorrect
+            if prev_event_id is None or not prev_content or not prev_sender:
+                logger.debug(
+                    "Removing %r from the state dict, as it is missing"
+                    " prev_content (prev_event_id=%r)",
+                    timeline_event.event_id, prev_event_id
+                )
                 del result[event_key]
             else:
+                logger.debug(
+                    "Replacing %r with %r in state dict",
+                    timeline_event.event_id, prev_event_id
+                )
                 result[event_key] = FrozenEvent({
                     "type": timeline_event.type,
                     "state_key": timeline_event.state_key,
-                    "content": timeline_event.unsigned['prev_content'],
-                    "sender": timeline_event.unsigned['prev_sender'],
+                    "content": prev_content,
+                    "sender": prev_sender,
                     "event_id": prev_event_id,
                     "room_id": timeline_event.room_id,
                 })
+
             logger.debug("New value: %r", result.get(event_key))
 
         return result
diff --git a/synapse/rest/client/v2_alpha/tags.py b/synapse/rest/client/v2_alpha/tags.py
index ba7223be11..b5d0db5569 100644
--- a/synapse/rest/client/v2_alpha/tags.py
+++ b/synapse/rest/client/v2_alpha/tags.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import client_v2_pattern
+from ._base import client_v2_patterns
 
 from synapse.http.servlet import RestServlet
 from synapse.api.errors import AuthError, SynapseError
@@ -31,7 +31,7 @@ class TagListServlet(RestServlet):
     """
     GET /user/{user_id}/rooms/{room_id}/tags HTTP/1.1
     """
-    PATTERN = client_v2_pattern(
+    PATTERNS = client_v2_patterns(
         "/user/(?P<user_id>[^/]*)/rooms/(?P<room_id>[^/]*)/tags"
     )
 
@@ -56,7 +56,7 @@ class TagServlet(RestServlet):
     PUT /user/{user_id}/rooms/{room_id}/tags/{tag} HTTP/1.1
     DELETE /user/{user_id}/rooms/{room_id}/tags/{tag} HTTP/1.1
     """
-    PATTERN = client_v2_pattern(
+    PATTERNS = client_v2_patterns(
         "/user/(?P<user_id>[^/]*)/rooms/(?P<room_id>[^/]*)/tags/(?P<tag>[^/]*)"
     )
 
diff --git a/synapse/rest/client/v2_alpha/tokenrefresh.py b/synapse/rest/client/v2_alpha/tokenrefresh.py
index 901e777983..5a63afd51e 100644
--- a/synapse/rest/client/v2_alpha/tokenrefresh.py
+++ b/synapse/rest/client/v2_alpha/tokenrefresh.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from synapse.api.errors import AuthError, StoreError, SynapseError
 from synapse.http.servlet import RestServlet
 
-from ._base import client_v2_pattern, parse_json_dict_from_request
+from ._base import client_v2_patterns, parse_json_dict_from_request
 
 
 class TokenRefreshRestServlet(RestServlet):
@@ -26,7 +26,7 @@ class TokenRefreshRestServlet(RestServlet):
     Exchanges refresh tokens for a pair of an access token and a new refresh
     token.
     """
-    PATTERN = client_v2_pattern("/tokenrefresh")
+    PATTERNS = client_v2_patterns("/tokenrefresh")
 
     def __init__(self, hs):
         super(TokenRefreshRestServlet, self).__init__()