Merge branch 'release-v0.14.0' of github.com:matrix-org/synapse v0.14.0

11 files changed, 89 insertions, 99 deletions

diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 24af322126..b6faa2b0e6 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -17,11 +17,9 @@
 """
 
 from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX
-from synapse.api.errors import SynapseError
 import re
 
 import logging
-import simplejson
 
 
 logger = logging.getLogger(__name__)
@@ -44,23 +42,3 @@ def client_v2_patterns(path_regex, releases=(0,)):
         new_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/r%d" % release)
         patterns.append(re.compile("^" + new_prefix + path_regex))
     return patterns
-
-
-def parse_request_allow_empty(request):
-    content = request.content.read()
-    if content is None or content == '':
-        return None
-    try:
-        return simplejson.loads(content)
-    except simplejson.JSONDecodeError:
-        raise SynapseError(400, "Content not JSON.")
-
-
-def parse_json_dict_from_request(request):
-    try:
-        content = simplejson.loads(request.content.read())
-        if type(content) != dict:
-            raise SynapseError(400, "Content must be a JSON object.")
-        return content
-    except simplejson.JSONDecodeError:
-        raise SynapseError(400, "Content not JSON.")
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index a614b79d45..7f8a6a4cf7 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -17,10 +17,10 @@ from twisted.internet import defer
 
 from synapse.api.constants import LoginType
 from synapse.api.errors import LoginError, SynapseError, Codes
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.util.async import run_on_reactor
 
-from ._base import client_v2_patterns, parse_json_dict_from_request
+from ._base import client_v2_patterns
 
 import logging
 
@@ -41,9 +41,9 @@ class PasswordRestServlet(RestServlet):
     def on_POST(self, request):
         yield run_on_reactor()
 
-        body = parse_json_dict_from_request(request)
+        body = parse_json_object_from_request(request)
 
-        authed, result, params = yield self.auth_handler.check_auth([
+        authed, result, params, _ = yield self.auth_handler.check_auth([
             [LoginType.PASSWORD],
             [LoginType.EMAIL_IDENTITY]
         ], body, self.hs.get_ip_from_request(request))
@@ -79,7 +79,7 @@ class PasswordRestServlet(RestServlet):
         new_password = params['new_password']
 
         yield self.auth_handler.set_password(
-            user_id, new_password
+            user_id, new_password, requester
         )
 
         defer.returnValue((200, {}))
@@ -114,7 +114,7 @@ class ThreepidRestServlet(RestServlet):
     def on_POST(self, request):
         yield run_on_reactor()
 
-        body = parse_json_dict_from_request(request)
+        body = parse_json_object_from_request(request)
 
         threePidCreds = body.get('threePidCreds')
         threePidCreds = body.get('three_pid_creds', threePidCreds)
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
index 1456881c1a..b16079cece 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -15,15 +15,13 @@
 
 from ._base import client_v2_patterns
 
-from synapse.http.servlet import RestServlet
-from synapse.api.errors import AuthError, SynapseError
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.api.errors import AuthError
 
 from twisted.internet import defer
 
 import logging
 
-import simplejson as json
-
 logger = logging.getLogger(__name__)
 
 
@@ -47,11 +45,7 @@ class AccountDataServlet(RestServlet):
         if user_id != requester.user.to_string():
             raise AuthError(403, "Cannot add account data for other users.")
 
-        try:
-            content_bytes = request.content.read()
-            body = json.loads(content_bytes)
-        except:
-            raise SynapseError(400, "Invalid JSON")
+        body = parse_json_object_from_request(request)
 
         max_id = yield self.store.add_account_data_for_user(
             user_id, account_data_type, body
@@ -86,14 +80,7 @@ class RoomAccountDataServlet(RestServlet):
         if user_id != requester.user.to_string():
             raise AuthError(403, "Cannot add account data for other users.")
 
-        try:
-            content_bytes = request.content.read()
-            body = json.loads(content_bytes)
-        except:
-            raise SynapseError(400, "Invalid JSON")
-
-        if not isinstance(body, dict):
-            raise ValueError("Expected a JSON object")
+        body = parse_json_object_from_request(request)
 
         max_id = yield self.store.add_account_data_to_room(
             user_id, room_id, account_data_type, body
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py
index ff71c40b43..78181b7b18 100644
--- a/synapse/rest/client/v2_alpha/auth.py
+++ b/synapse/rest/client/v2_alpha/auth.py
@@ -18,6 +18,7 @@ from twisted.internet import defer
 from synapse.api.constants import LoginType
 from synapse.api.errors import SynapseError
 from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX
+from synapse.http.server import finish_request
 from synapse.http.servlet import RestServlet
 
 from ._base import client_v2_patterns
@@ -130,7 +131,7 @@ class AuthRestServlet(RestServlet):
             request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
 
             request.write(html_bytes)
-            request.finish()
+            finish_request(request)
             defer.returnValue(None)
         else:
             raise SynapseError(404, "Unknown auth stage type")
@@ -176,7 +177,7 @@ class AuthRestServlet(RestServlet):
             request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
 
             request.write(html_bytes)
-            request.finish()
+            finish_request(request)
 
             defer.returnValue(None)
         else:
diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py
index 7c94f6ec41..510f8b2c74 100644
--- a/synapse/rest/client/v2_alpha/filter.py
+++ b/synapse/rest/client/v2_alpha/filter.py
@@ -16,12 +16,11 @@
 from twisted.internet import defer
 
 from synapse.api.errors import AuthError, SynapseError
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.types import UserID
 
 from ._base import client_v2_patterns
 
-import simplejson as json
 import logging
 
 
@@ -84,12 +83,7 @@ class CreateFilterRestServlet(RestServlet):
         if not self.hs.is_mine(target_user):
             raise SynapseError(400, "Can only create filters for local users")
 
-        try:
-            content = json.loads(request.content.read())
-
-            # TODO(paul): check for required keys and invalid keys
-        except:
-            raise SynapseError(400, "Invalid filter definition")
+        content = parse_json_object_from_request(request)
 
         filter_id = yield self.filtering.add_user_filter(
             user_localpart=target_user.localpart,
diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py
index f989b08614..89ab39491c 100644
--- a/synapse/rest/client/v2_alpha/keys.py
+++ b/synapse/rest/client/v2_alpha/keys.py
@@ -15,16 +15,15 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.types import UserID
 
 from canonicaljson import encode_canonical_json
 
 from ._base import client_v2_patterns
 
-import simplejson as json
 import logging
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
@@ -68,10 +67,9 @@ class KeyUploadServlet(RestServlet):
         user_id = requester.user.to_string()
         # TODO: Check that the device_id matches that in the authentication
         # or derive the device_id from the authentication instead.
-        try:
-            body = json.loads(request.content.read())
-        except:
-            raise SynapseError(400, "Invalid key JSON")
+
+        body = parse_json_object_from_request(request)
+
         time_now = self.clock.time_msec()
 
         # TODO: Validate the JSON to make sure it has the right keys.
@@ -173,10 +171,7 @@ class KeyQueryServlet(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request, user_id, device_id):
         yield self.auth.get_user_by_req(request)
-        try:
-            body = json.loads(request.content.read())
-        except:
-            raise SynapseError(400, "Invalid key JSON")
+        body = parse_json_object_from_request(request)
         result = yield self.handle_request(body)
         defer.returnValue(result)
 
@@ -272,10 +267,7 @@ class OneTimeKeyServlet(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request, user_id, device_id, algorithm):
         yield self.auth.get_user_by_req(request)
-        try:
-            body = json.loads(request.content.read())
-        except:
-            raise SynapseError(400, "Invalid key JSON")
+        body = parse_json_object_from_request(request)
         result = yield self.handle_request(body)
         defer.returnValue(result)
 
diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py
index eb4b369a3d..b831d8c95e 100644
--- a/synapse/rest/client/v2_alpha/receipts.py
+++ b/synapse/rest/client/v2_alpha/receipts.py
@@ -37,6 +37,7 @@ class ReceiptRestServlet(RestServlet):
         self.hs = hs
         self.auth = hs.get_auth()
         self.receipts_handler = hs.get_handlers().receipts_handler
+        self.presence_handler = hs.get_handlers().presence_handler
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, receipt_type, event_id):
@@ -45,6 +46,8 @@ class ReceiptRestServlet(RestServlet):
         if receipt_type != "m.read":
             raise SynapseError(400, "Receipt type must be 'm.read'")
 
+        yield self.presence_handler.bump_presence_active_time(requester.user)
+
         yield self.receipts_handler.received_client_receipt(
             room_id,
             receipt_type,
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index ec5c21fa1f..d32c06c882 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -17,9 +17,9 @@ from twisted.internet import defer
 
 from synapse.api.constants import LoginType
 from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
-from ._base import client_v2_patterns, parse_json_dict_from_request
+from ._base import client_v2_patterns
 
 import logging
 import hmac
@@ -73,7 +73,7 @@ class RegisterRestServlet(RestServlet):
             ret = yield self.onEmailTokenRequest(request)
             defer.returnValue(ret)
 
-        body = parse_json_dict_from_request(request)
+        body = parse_json_object_from_request(request)
 
         # we do basic sanity checks here because the auth layer will store these
         # in sessions. Pull out the username/password provided to us.
@@ -122,10 +122,22 @@ class RegisterRestServlet(RestServlet):
 
         guest_access_token = body.get("guest_access_token", None)
 
+        session_id = self.auth_handler.get_session_id(body)
+        registered_user_id = None
+        if session_id:
+            # if we get a registered user id out of here, it means we previously
+            # registered a user for this session, so we could just return the
+            # user here. We carry on and go through the auth checks though,
+            # for paranoia.
+            registered_user_id = self.auth_handler.get_session_data(
+                session_id, "registered_user_id", None
+            )
+
         if desired_username is not None:
             yield self.registration_handler.check_username(
                 desired_username,
-                guest_access_token=guest_access_token
+                guest_access_token=guest_access_token,
+                assigned_user_id=registered_user_id,
             )
 
         if self.hs.config.enable_registration_captcha:
@@ -139,7 +151,7 @@ class RegisterRestServlet(RestServlet):
                 [LoginType.EMAIL_IDENTITY]
             ]
 
-        authed, result, params = yield self.auth_handler.check_auth(
+        authed, result, params, session_id = yield self.auth_handler.check_auth(
             flows, body, self.hs.get_ip_from_request(request)
         )
 
@@ -147,6 +159,22 @@ class RegisterRestServlet(RestServlet):
             defer.returnValue((401, result))
             return
 
+        if registered_user_id is not None:
+            logger.info(
+                "Already registered user ID %r for this session",
+                registered_user_id
+            )
+            access_token = yield self.auth_handler.issue_access_token(registered_user_id)
+            refresh_token = yield self.auth_handler.issue_refresh_token(
+                registered_user_id
+            )
+            defer.returnValue((200, {
+                "user_id": registered_user_id,
+                "access_token": access_token,
+                "home_server": self.hs.hostname,
+                "refresh_token": refresh_token,
+            }))
+
         # NB: This may be from the auth handler and NOT from the POST
         if 'password' not in params:
             raise SynapseError(400, "Missing password.", Codes.MISSING_PARAM)
@@ -161,6 +189,12 @@ class RegisterRestServlet(RestServlet):
             guest_access_token=guest_access_token,
         )
 
+        # remember that we've now registered that user account, and with what
+        # user ID (since the user may not have specified)
+        self.auth_handler.set_session_data(
+            session_id, "registered_user_id", user_id
+        )
+
         if result and LoginType.EMAIL_IDENTITY in result:
             threepid = result[LoginType.EMAIL_IDENTITY]
 
@@ -187,7 +221,7 @@ class RegisterRestServlet(RestServlet):
             else:
                 logger.info("bind_email not specified: not binding email")
 
-        result = self._create_registration_details(user_id, token)
+        result = yield self._create_registration_details(user_id, token)
         defer.returnValue((200, result))
 
     def on_OPTIONS(self, _):
@@ -198,7 +232,7 @@ class RegisterRestServlet(RestServlet):
         (user_id, token) = yield self.registration_handler.appservice_register(
             username, as_token
         )
-        defer.returnValue(self._create_registration_details(user_id, token))
+        defer.returnValue((yield self._create_registration_details(user_id, token)))
 
     @defer.inlineCallbacks
     def _do_shared_secret_registration(self, username, password, mac):
@@ -225,18 +259,21 @@ class RegisterRestServlet(RestServlet):
         (user_id, token) = yield self.registration_handler.register(
             localpart=username, password=password
         )
-        defer.returnValue(self._create_registration_details(user_id, token))
+        defer.returnValue((yield self._create_registration_details(user_id, token)))
 
+    @defer.inlineCallbacks
     def _create_registration_details(self, user_id, token):
-        return {
+        refresh_token = yield self.auth_handler.issue_refresh_token(user_id)
+        defer.returnValue({
             "user_id": user_id,
             "access_token": token,
             "home_server": self.hs.hostname,
-        }
+            "refresh_token": refresh_token,
+        })
 
     @defer.inlineCallbacks
     def onEmailTokenRequest(self, request):
-        body = parse_json_dict_from_request(request)
+        body = parse_json_object_from_request(request)
 
         required = ['id_server', 'client_secret', 'email', 'send_attempt']
         absent = []
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index accbc6cfac..de4a020ad4 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -25,6 +25,7 @@ from synapse.events.utils import (
 )
 from synapse.api.filtering import FilterCollection, DEFAULT_FILTER_COLLECTION
 from synapse.api.errors import SynapseError
+from synapse.api.constants import PresenceState
 from ._base import client_v2_patterns
 
 import copy
@@ -82,6 +83,7 @@ class SyncRestServlet(RestServlet):
         self.sync_handler = hs.get_handlers().sync_handler
         self.clock = hs.get_clock()
         self.filtering = hs.get_filtering()
+        self.presence_handler = hs.get_handlers().presence_handler
 
     @defer.inlineCallbacks
     def on_GET(self, request):
@@ -139,17 +141,19 @@ class SyncRestServlet(RestServlet):
         else:
             since_token = None
 
-        if set_presence == "online":
-            yield self.event_stream_handler.started_stream(user)
+        affect_presence = set_presence != PresenceState.OFFLINE
 
-        try:
+        if affect_presence:
+            yield self.presence_handler.set_state(user, {"presence": set_presence})
+
+        context = yield self.presence_handler.user_syncing(
+            user.to_string(), affect_presence=affect_presence,
+        )
+        with context:
             sync_result = yield self.sync_handler.wait_for_sync_for_user(
                 sync_config, since_token=since_token, timeout=timeout,
                 full_state=full_state
             )
-        finally:
-            if set_presence == "online":
-                self.event_stream_handler.stopped_stream(user)
 
         time_now = self.clock.time_msec()
 
diff --git a/synapse/rest/client/v2_alpha/tags.py b/synapse/rest/client/v2_alpha/tags.py
index 79c436a8cf..dac8603b07 100644
--- a/synapse/rest/client/v2_alpha/tags.py
+++ b/synapse/rest/client/v2_alpha/tags.py
@@ -15,15 +15,13 @@
 
 from ._base import client_v2_patterns
 
-from synapse.http.servlet import RestServlet
-from synapse.api.errors import AuthError, SynapseError
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.api.errors import AuthError
 
 from twisted.internet import defer
 
 import logging
 
-import simplejson as json
-
 logger = logging.getLogger(__name__)
 
 
@@ -72,11 +70,7 @@ class TagServlet(RestServlet):
         if user_id != requester.user.to_string():
             raise AuthError(403, "Cannot add tags for other users.")
 
-        try:
-            content_bytes = request.content.read()
-            body = json.loads(content_bytes)
-        except:
-            raise SynapseError(400, "Invalid tag JSON")
+        body = parse_json_object_from_request(request)
 
         max_id = yield self.store.add_tag_to_room(user_id, room_id, tag, body)
 
diff --git a/synapse/rest/client/v2_alpha/tokenrefresh.py b/synapse/rest/client/v2_alpha/tokenrefresh.py
index 3553f6b040..a158c2209a 100644
--- a/synapse/rest/client/v2_alpha/tokenrefresh.py
+++ b/synapse/rest/client/v2_alpha/tokenrefresh.py
@@ -16,9 +16,9 @@
 from twisted.internet import defer
 
 from synapse.api.errors import AuthError, StoreError, SynapseError
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
 
-from ._base import client_v2_patterns, parse_json_dict_from_request
+from ._base import client_v2_patterns
 
 
 class TokenRefreshRestServlet(RestServlet):
@@ -35,7 +35,7 @@ class TokenRefreshRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        body = parse_json_dict_from_request(request)
+        body = parse_json_object_from_request(request)
         try:
             old_refresh_token = body["refresh_token"]
             auth_handler = self.hs.get_handlers().auth_handler