Merge branch 'release-v0.13.0' of github.com:matrix-org/synapse v0.13.0

15 files changed, 195 insertions, 162 deletions

diff --git a/synapse/rest/client/v1/__init__.py b/synapse/rest/client/v1/__init__.py
index c488b10d3c..bfebb0f644 100644
--- a/synapse/rest/client/v1/__init__.py
+++ b/synapse/rest/client/v1/__init__.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 886199a6da..e2f5eb7b29 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,8 +31,9 @@ class WhoisRestServlet(ClientV1RestServlet):
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
         target_user = UserID.from_string(user_id)
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
-        is_admin = yield self.auth.is_server_admin(auth_user)
+        requester = yield self.auth.get_user_by_req(request)
+        auth_user = requester.user
+        is_admin = yield self.auth.is_server_admin(requester.user)
 
         if not is_admin and target_user != auth_user:
             raise AuthError(403, "You are not a server admin")
diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py
index 6273ce0795..1c020b7e2c 100644
--- a/synapse/rest/client/v1/base.py
+++ b/synapse/rest/client/v1/base.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py
index f488e2dd41..74ec1e50e0 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/v1/directory.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -69,9 +69,9 @@ class ClientDirectoryServer(ClientV1RestServlet):
 
         try:
             # try to auth as a user
-            user, _, _ = yield self.auth.get_user_by_req(request)
+            requester = yield self.auth.get_user_by_req(request)
             try:
-                user_id = user.to_string()
+                user_id = requester.user.to_string()
                 yield dir_handler.create_association(
                     user_id, room_alias, room_id, servers
                 )
@@ -116,8 +116,8 @@ class ClientDirectoryServer(ClientV1RestServlet):
             # fallback to default user behaviour if they aren't an AS
             pass
 
-        user, _, _ = yield self.auth.get_user_by_req(request)
-
+        requester = yield self.auth.get_user_by_req(request)
+        user = requester.user
         is_admin = yield self.auth.is_server_admin(user)
         if not is_admin:
             raise AuthError(403, "You need to be a server admin")
diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py
index 41b97e7d15..d1afa0f0d5 100644
--- a/synapse/rest/client/v1/events.py
+++ b/synapse/rest/client/v1/events.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,14 +34,16 @@ class EventStreamRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        auth_user, _, is_guest = yield self.auth.get_user_by_req(
+        requester = yield self.auth.get_user_by_req(
             request,
-            allow_guest=True
+            allow_guest=True,
         )
+        is_guest = requester.is_guest
         room_id = None
         if is_guest:
             if "room_id" not in request.args:
                 raise SynapseError(400, "Guest users must specify room_id param")
+        if "room_id" in request.args:
             room_id = request.args["room_id"][0]
         try:
             handler = self.handlers.event_stream_handler
@@ -56,9 +58,13 @@ class EventStreamRestServlet(ClientV1RestServlet):
             as_client_event = "raw" not in request.args
 
             chunk = yield handler.get_stream(
-                auth_user.to_string(), pagin_config, timeout=timeout,
-                as_client_event=as_client_event, affect_presence=(not is_guest),
-                room_id=room_id, is_guest=is_guest
+                requester.user.to_string(),
+                pagin_config,
+                timeout=timeout,
+                as_client_event=as_client_event,
+                affect_presence=(not is_guest),
+                room_id=room_id,
+                is_guest=is_guest,
             )
         except:
             logger.exception("Event stream failed")
@@ -80,9 +86,9 @@ class EventRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, event_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         handler = self.handlers.event_handler
-        event = yield handler.get_event(auth_user, event_id)
+        event = yield handler.get_event(requester.user, event_id)
 
         time_now = self.clock.time_msec()
         if event:
diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py
index 9ad3df8a9f..ad161bdbab 100644
--- a/synapse/rest/client/v1/initial_sync.py
+++ b/synapse/rest/client/v1/initial_sync.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,13 +25,13 @@ class InitialSyncRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         as_client_event = "raw" not in request.args
         pagination_config = PaginationConfig.from_request(request)
         handler = self.handlers.message_handler
         include_archived = request.args.get("archived", None) == ["true"]
         content = yield handler.snapshot_all_rooms(
-            user_id=user.to_string(),
+            user_id=requester.user.to_string(),
             pagin_config=pagination_config,
             as_client_event=as_client_event,
             include_archived=include_archived,
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index e8c35508cd..7199113dac 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -89,7 +89,7 @@ class LoginRestServlet(ClientV1RestServlet):
                                          LoginRestServlet.SAML2_TYPE):
                 relay_state = ""
                 if "relay_state" in login_submission:
-                    relay_state = "&RelayState="+urllib.quote(
+                    relay_state = "&RelayState=" + urllib.quote(
                                   login_submission["relay_state"])
                 result = {
                     "uri": "%s%s" % (self.idp_redirect_url, relay_state)
diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py
index e0949fe4bb..a6f8754e32 100644
--- a/synapse/rest/client/v1/presence.py
+++ b/synapse/rest/client/v1/presence.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,17 +32,17 @@ class PresenceStatusRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         user = UserID.from_string(user_id)
 
         state = yield self.handlers.presence_handler.get_state(
-            target_user=user, auth_user=auth_user)
+            target_user=user, auth_user=requester.user)
 
         defer.returnValue((200, state))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         user = UserID.from_string(user_id)
 
         state = {}
@@ -64,7 +64,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet):
             raise SynapseError(400, "Unable to parse state")
 
         yield self.handlers.presence_handler.set_state(
-            target_user=user, auth_user=auth_user, state=state)
+            target_user=user, auth_user=requester.user, state=state)
 
         defer.returnValue((200, {}))
 
@@ -77,13 +77,13 @@ class PresenceListRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         user = UserID.from_string(user_id)
 
         if not self.hs.is_mine(user):
             raise SynapseError(400, "User not hosted on this Home Server")
 
-        if auth_user != user:
+        if requester.user != user:
             raise SynapseError(400, "Cannot get another user's presence list")
 
         presence = yield self.handlers.presence_handler.get_presence_list(
@@ -97,13 +97,13 @@ class PresenceListRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         user = UserID.from_string(user_id)
 
         if not self.hs.is_mine(user):
             raise SynapseError(400, "User not hosted on this Home Server")
 
-        if auth_user != user:
+        if requester.user != user:
             raise SynapseError(
                 400, "Cannot modify another user's presence list")
 
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index e6c6e5d024..3c5a212920 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,11 +33,15 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
             user,
         )
 
-        defer.returnValue((200, {"displayname": displayname}))
+        ret = {}
+        if displayname is not None:
+            ret["displayname"] = displayname
+
+        defer.returnValue((200, ret))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         user = UserID.from_string(user_id)
 
         try:
@@ -47,7 +51,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
             defer.returnValue((400, "Unable to parse name"))
 
         yield self.handlers.profile_handler.set_displayname(
-            user, auth_user, new_name)
+            user, requester.user, new_name)
 
         defer.returnValue((200, {}))
 
@@ -66,11 +70,15 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
             user,
         )
 
-        defer.returnValue((200, {"avatar_url": avatar_url}))
+        ret = {}
+        if avatar_url is not None:
+            ret["avatar_url"] = avatar_url
+
+        defer.returnValue((200, ret))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         user = UserID.from_string(user_id)
 
         try:
@@ -80,7 +88,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet):
             defer.returnValue((400, "Unable to parse name"))
 
         yield self.handlers.profile_handler.set_avatar_url(
-            user, auth_user, new_name)
+            user, requester.user, new_name)
 
         defer.returnValue((200, {}))
 
@@ -102,10 +110,13 @@ class ProfileRestServlet(ClientV1RestServlet):
             user,
         )
 
-        defer.returnValue((200, {
-            "displayname": displayname,
-            "avatar_url": avatar_url
-        }))
+        ret = {}
+        if displayname is not None:
+            ret["displayname"] = displayname
+        if avatar_url is not None:
+            ret["avatar_url"] = avatar_url
+
+        defer.returnValue((200, ret))
 
 
 def register_servlets(hs, http_server):
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index 9270bdd079..96633a176c 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ from synapse.push.rulekinds import (
     PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP
 )
 
+import copy
 import simplejson as json
 
 
@@ -43,7 +44,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
         except InvalidRuleException as e:
             raise SynapseError(400, e.message)
 
-        user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         if '/' in spec['rule_id'] or '\\' in spec['rule_id']:
             raise SynapseError(400, "rule_id may not contain slashes")
@@ -51,7 +52,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
         content = _parse_json(request)
 
         if 'attr' in spec:
-            self.set_rule_attr(user.to_string(), spec, content)
+            yield self.set_rule_attr(requester.user.to_string(), spec, content)
             defer.returnValue((200, {}))
 
         try:
@@ -65,15 +66,16 @@ class PushRuleRestServlet(ClientV1RestServlet):
             raise SynapseError(400, e.message)
 
         before = request.args.get("before", None)
-        if before and len(before):
-            before = before[0]
+        if before:
+            before = _namespaced_rule_id(spec, before[0])
+
         after = request.args.get("after", None)
-        if after and len(after):
-            after = after[0]
+        if after:
+            after = _namespaced_rule_id(spec, after[0])
 
         try:
             yield self.hs.get_datastore().add_push_rule(
-                user_name=user.to_string(),
+                user_id=requester.user.to_string(),
                 rule_id=_namespaced_rule_id_from_spec(spec),
                 priority_class=priority_class,
                 conditions=conditions,
@@ -92,13 +94,13 @@ class PushRuleRestServlet(ClientV1RestServlet):
     def on_DELETE(self, request):
         spec = _rule_spec_from_path(request.postpath)
 
-        user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         namespaced_rule_id = _namespaced_rule_id_from_spec(spec)
 
         try:
             yield self.hs.get_datastore().delete_push_rule(
-                user.to_string(), namespaced_rule_id
+                requester.user.to_string(), namespaced_rule_id
             )
             defer.returnValue((200, {}))
         except StoreError as e:
@@ -109,7 +111,8 @@ class PushRuleRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
+        user = requester.user
 
         # we build up the full structure and then decide which bits of it
         # to send which means doing unnecessary work sometimes but is
@@ -125,7 +128,8 @@ class PushRuleRestServlet(ClientV1RestServlet):
             rule["actions"] = json.loads(rawrule["actions"])
             ruleslist.append(rule)
 
-        ruleslist = baserules.list_with_base_rules(ruleslist, user)
+        # We're going to be mutating this a lot, so do a deep copy
+        ruleslist = copy.deepcopy(baserules.list_with_base_rules(ruleslist))
 
         rules = {'global': {}, 'device': {}}
 
@@ -139,6 +143,16 @@ class PushRuleRestServlet(ClientV1RestServlet):
 
             template_name = _priority_class_to_template_name(r['priority_class'])
 
+            # Remove internal stuff.
+            for c in r["conditions"]:
+                c.pop("_id", None)
+
+                pattern_type = c.pop("pattern_type", None)
+                if pattern_type == "user_id":
+                    c["pattern"] = user.to_string()
+                elif pattern_type == "user_localpart":
+                    c["pattern"] = user.localpart
+
             if r['priority_class'] > PRIORITY_CLASS_MAP['override']:
                 # per-device rule
                 profile_tag = _profile_tag_from_conditions(r["conditions"])
@@ -205,7 +219,7 @@ class PushRuleRestServlet(ClientV1RestServlet):
     def on_OPTIONS(self, _):
         return 200, {}
 
-    def set_rule_attr(self, user_name, spec, val):
+    def set_rule_attr(self, user_id, spec, val):
         if spec['attr'] == 'enabled':
             if isinstance(val, dict) and "enabled" in val:
                 val = val["enabled"]
@@ -215,16 +229,16 @@ class PushRuleRestServlet(ClientV1RestServlet):
                 # bools directly, so let's not break them.
                 raise SynapseError(400, "Value for 'enabled' must be boolean")
             namespaced_rule_id = _namespaced_rule_id_from_spec(spec)
-            self.hs.get_datastore().set_push_rule_enabled(
-                user_name, namespaced_rule_id, val
+            return self.hs.get_datastore().set_push_rule_enabled(
+                user_id, namespaced_rule_id, val
             )
         else:
             raise UnrecognizedRequestError()
 
-    def get_rule_attr(self, user_name, namespaced_rule_id, attr):
+    def get_rule_attr(self, user_id, namespaced_rule_id, attr):
         if attr == 'enabled':
             return self.hs.get_datastore().get_push_rule_enabled_by_user_rule_id(
-                user_name, namespaced_rule_id
+                user_id, namespaced_rule_id
             )
         else:
             raise UnrecognizedRequestError()
@@ -439,11 +453,15 @@ def _strip_device_condition(rule):
 
 
 def _namespaced_rule_id_from_spec(spec):
+    return _namespaced_rule_id(spec, spec['rule_id'])
+
+
+def _namespaced_rule_id(spec, rule_id):
     if spec['scope'] == 'global':
         scope = 'global'
     else:
         scope = 'device/%s' % (spec['profile_tag'])
-    return "%s/%s/%s" % (scope, spec['template'], spec['rule_id'])
+    return "%s/%s/%s" % (scope, spec['template'], rule_id)
 
 
 def _rule_id_from_namespaced(in_rule_id):
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index d6d1ad528e..5547f1b112 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,7 +30,8 @@ class PusherRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        user, token_id, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
+        user = requester.user
 
         content = _parse_json(request)
 
@@ -40,7 +41,7 @@ class PusherRestServlet(ClientV1RestServlet):
                 and 'kind' in content and
                 content['kind'] is None):
             yield pusher_pool.remove_pusher(
-                content['app_id'], content['pushkey'], user_name=user.to_string()
+                content['app_id'], content['pushkey'], user_id=user.to_string()
             )
             defer.returnValue((200, {}))
 
@@ -51,7 +52,7 @@ class PusherRestServlet(ClientV1RestServlet):
             if i not in content:
                 missing.append(i)
         if len(missing):
-            raise SynapseError(400, "Missing parameters: "+','.join(missing),
+            raise SynapseError(400, "Missing parameters: " + ','.join(missing),
                                errcode=Codes.MISSING_PARAM)
 
         logger.debug("set pushkey %s to kind %s", content['pushkey'], content['kind'])
@@ -70,8 +71,8 @@ class PusherRestServlet(ClientV1RestServlet):
 
         try:
             yield pusher_pool.add_pusher(
-                user_name=user.to_string(),
-                access_token=token_id,
+                user_id=user.to_string(),
+                access_token=requester.access_token_id,
                 profile_tag=content['profile_tag'],
                 kind=content['kind'],
                 app_id=content['app_id'],
@@ -82,7 +83,7 @@ class PusherRestServlet(ClientV1RestServlet):
                 data=content['data']
             )
         except PusherConfigException as pce:
-            raise SynapseError(400, "Config Error: "+pce.message,
+            raise SynapseError(400, "Config Error: " + pce.message,
                                errcode=Codes.MISSING_PARAM)
 
         defer.returnValue((200, {}))
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index 4b02311e05..6d6d03c34c 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,7 +38,8 @@ logger = logging.getLogger(__name__)
 if hasattr(hmac, "compare_digest"):
     compare_digest = hmac.compare_digest
 else:
-    compare_digest = lambda a, b: a == b
+    def compare_digest(a, b):
+        return a == b
 
 
 class RegisterRestServlet(ClientV1RestServlet):
@@ -58,7 +59,7 @@ class RegisterRestServlet(ClientV1RestServlet):
         # }
         # TODO: persistent storage
         self.sessions = {}
-        self.disable_registration = hs.config.disable_registration
+        self.enable_registration = hs.config.enable_registration
 
     def on_GET(self, request):
         if self.hs.config.enable_registration_captcha:
@@ -112,7 +113,7 @@ class RegisterRestServlet(ClientV1RestServlet):
             is_using_shared_secret = login_type == LoginType.SHARED_SECRET
 
             can_register = (
-                not self.disable_registration
+                self.enable_registration
                 or is_application_server
                 or is_using_shared_secret
             )
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 6fe53f70e5..81bfe377bd 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -61,10 +61,14 @@ class RoomCreateRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         room_config = self.get_room_config(request)
-        info = yield self.make_room(room_config, auth_user, None)
+        info = yield self.make_room(
+            room_config,
+            requester.user,
+            None,
+        )
         room_config.update(info)
         defer.returnValue((200, info))
 
@@ -124,15 +128,15 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, event_type, state_key):
-        user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
 
         msg_handler = self.handlers.message_handler
         data = yield msg_handler.get_room_data(
-            user_id=user.to_string(),
+            user_id=requester.user.to_string(),
             room_id=room_id,
             event_type=event_type,
             state_key=state_key,
-            is_guest=is_guest,
+            is_guest=requester.is_guest,
         )
 
         if not data:
@@ -143,7 +147,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, event_type, state_key, txn_id=None):
-        user, token_id, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         content = _parse_json(request)
 
@@ -151,7 +155,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
             "type": event_type,
             "content": content,
             "room_id": room_id,
-            "sender": user.to_string(),
+            "sender": requester.user.to_string(),
         }
 
         if state_key is not None:
@@ -159,7 +163,7 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
 
         msg_handler = self.handlers.message_handler
         yield msg_handler.create_and_send_event(
-            event_dict, token_id=token_id, txn_id=txn_id,
+            event_dict, token_id=requester.access_token_id, txn_id=txn_id,
         )
 
         defer.returnValue((200, {}))
@@ -175,7 +179,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, event_type, txn_id=None):
-        user, token_id, _ = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         content = _parse_json(request)
 
         msg_handler = self.handlers.message_handler
@@ -184,9 +188,9 @@ class RoomSendEventRestServlet(ClientV1RestServlet):
                 "type": event_type,
                 "content": content,
                 "room_id": room_id,
-                "sender": user.to_string(),
+                "sender": requester.user.to_string(),
             },
-            token_id=token_id,
+            token_id=requester.access_token_id,
             txn_id=txn_id,
         )
 
@@ -220,9 +224,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_identifier, txn_id=None):
-        user, token_id, is_guest = yield self.auth.get_user_by_req(
+        requester = yield self.auth.get_user_by_req(
             request,
-            allow_guest=True
+            allow_guest=True,
         )
 
         # the identifier could be a room alias or a room id. Try one then the
@@ -241,24 +245,27 @@ class JoinRoomAliasServlet(ClientV1RestServlet):
 
         if is_room_alias:
             handler = self.handlers.room_member_handler
-            ret_dict = yield handler.join_room_alias(user, identifier)
+            ret_dict = yield handler.join_room_alias(
+                requester.user,
+                identifier,
+            )
             defer.returnValue((200, ret_dict))
         else:  # room id
             msg_handler = self.handlers.message_handler
             content = {"membership": Membership.JOIN}
-            if is_guest:
+            if requester.is_guest:
                 content["kind"] = "guest"
             yield msg_handler.create_and_send_event(
                 {
                     "type": EventTypes.Member,
                     "content": content,
                     "room_id": identifier.to_string(),
-                    "sender": user.to_string(),
-                    "state_key": user.to_string(),
+                    "sender": requester.user.to_string(),
+                    "state_key": requester.user.to_string(),
                 },
-                token_id=token_id,
+                token_id=requester.access_token_id,
                 txn_id=txn_id,
-                is_guest=is_guest,
+                is_guest=requester.is_guest,
             )
 
             defer.returnValue((200, {"room_id": identifier.to_string()}))
@@ -296,11 +303,11 @@ class RoomMemberListRestServlet(ClientV1RestServlet):
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
         # TODO support Pagination stream API (limit/tokens)
-        user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         handler = self.handlers.message_handler
         events = yield handler.get_state_events(
             room_id=room_id,
-            user_id=user.to_string(),
+            user_id=requester.user.to_string(),
         )
 
         chunk = []
@@ -315,7 +322,8 @@ class RoomMemberListRestServlet(ClientV1RestServlet):
             try:
                 presence_handler = self.handlers.presence_handler
                 presence_state = yield presence_handler.get_state(
-                    target_user=target_user, auth_user=user
+                    target_user=target_user,
+                    auth_user=requester.user,
                 )
                 event["content"].update(presence_state)
             except:
@@ -332,7 +340,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
-        user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         pagination_config = PaginationConfig.from_request(
             request, default_limit=10,
         )
@@ -340,8 +348,7 @@ class RoomMessageListRestServlet(ClientV1RestServlet):
         handler = self.handlers.message_handler
         msgs = yield handler.get_messages(
             room_id=room_id,
-            user_id=user.to_string(),
-            is_guest=is_guest,
+            requester=requester,
             pagin_config=pagination_config,
             as_client_event=as_client_event
         )
@@ -355,13 +362,13 @@ class RoomStateRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
-        user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         handler = self.handlers.message_handler
         # Get all the current state for this room
         events = yield handler.get_state_events(
             room_id=room_id,
-            user_id=user.to_string(),
-            is_guest=is_guest,
+            user_id=requester.user.to_string(),
+            is_guest=requester.is_guest,
         )
         defer.returnValue((200, events))
 
@@ -372,13 +379,12 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
-        user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
         pagination_config = PaginationConfig.from_request(request)
         content = yield self.handlers.message_handler.room_initial_sync(
             room_id=room_id,
-            user_id=user.to_string(),
+            requester=requester,
             pagin_config=pagination_config,
-            is_guest=is_guest,
         )
         defer.returnValue((200, content))
 
@@ -394,18 +400,28 @@ class RoomEventContext(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id, event_id):
-        user, _, is_guest = yield self.auth.get_user_by_req(request, allow_guest=True)
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
 
         limit = int(request.args.get("limit", [10])[0])
 
         results = yield self.handlers.room_context_handler.get_event_context(
-            user, room_id, event_id, limit, is_guest
+            requester.user,
+            room_id,
+            event_id,
+            limit,
+            requester.is_guest,
         )
 
+        if not results:
+            raise SynapseError(
+                404, "Event not found.", errcode=Codes.NOT_FOUND
+            )
+
         time_now = self.clock.time_msec()
         results["events_before"] = [
             serialize_event(event, time_now) for event in results["events_before"]
         ]
+        results["event"] = serialize_event(results["event"], time_now)
         results["events_after"] = [
             serialize_event(event, time_now) for event in results["events_after"]
         ]
@@ -413,8 +429,6 @@ class RoomEventContext(ClientV1RestServlet):
             serialize_event(event, time_now) for event in results["state"]
         ]
 
-        logger.info("Responding with %r", results)
-
         defer.returnValue((200, results))
 
 
@@ -424,74 +438,51 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
     def register(self, http_server):
         # /rooms/$roomid/[invite|join|leave]
         PATTERNS = ("/rooms/(?P<room_id>[^/]*)/"
-                    "(?P<membership_action>join|invite|leave|ban|kick|forget)")
+                    "(?P<membership_action>join|invite|leave|ban|unban|kick|forget)")
         register_txn_path(self, PATTERNS, http_server)
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, membership_action, txn_id=None):
-        user, token_id, is_guest = yield self.auth.get_user_by_req(
+        requester = yield self.auth.get_user_by_req(
             request,
-            allow_guest=True
+            allow_guest=True,
         )
 
-        effective_membership_action = membership_action
-
-        if is_guest and membership_action not in {Membership.JOIN, Membership.LEAVE}:
+        if requester.is_guest and membership_action not in {
+            Membership.JOIN,
+            Membership.LEAVE
+        }:
             raise AuthError(403, "Guest access not allowed")
 
         content = _parse_json(request)
 
-        # target user is you unless it is an invite
-        state_key = user.to_string()
-
         if membership_action == "invite" and self._has_3pid_invite_keys(content):
             yield self.handlers.room_member_handler.do_3pid_invite(
                 room_id,
-                user,
+                requester.user,
                 content["medium"],
                 content["address"],
                 content["id_server"],
-                token_id,
+                requester.access_token_id,
                 txn_id
             )
             defer.returnValue((200, {}))
             return
-        elif membership_action in ["invite", "ban", "kick"]:
-            if "user_id" in content:
-                state_key = content["user_id"]
-            else:
-                raise SynapseError(400, "Missing user_id key.")
-
-            # make sure it looks like a user ID; it'll throw if it's invalid.
-            UserID.from_string(state_key)
-
-            if membership_action == "kick":
-                effective_membership_action = "leave"
-        elif membership_action == "forget":
-            effective_membership_action = "leave"
 
-        msg_handler = self.handlers.message_handler
-
-        content = {"membership": unicode(effective_membership_action)}
-        if is_guest:
-            content["kind"] = "guest"
+        target = requester.user
+        if membership_action in ["invite", "ban", "unban", "kick"]:
+            if "user_id" not in content:
+                raise SynapseError(400, "Missing user_id key.")
+            target = UserID.from_string(content["user_id"])
 
-        yield msg_handler.create_and_send_event(
-            {
-                "type": EventTypes.Member,
-                "content": content,
-                "room_id": room_id,
-                "sender": user.to_string(),
-                "state_key": state_key,
-            },
-            token_id=token_id,
+        yield self.handlers.room_member_handler.update_membership(
+            requester=requester,
+            target=target,
+            room_id=room_id,
+            action=membership_action,
             txn_id=txn_id,
-            is_guest=is_guest,
         )
 
-        if membership_action == "forget":
-            yield self.handlers.room_member_handler.forget(user, room_id)
-
         defer.returnValue((200, {}))
 
     def _has_3pid_invite_keys(self, content):
@@ -524,7 +515,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request, room_id, event_id, txn_id=None):
-        user, token_id, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
         content = _parse_json(request)
 
         msg_handler = self.handlers.message_handler
@@ -533,10 +524,10 @@ class RoomRedactEventRestServlet(ClientV1RestServlet):
                 "type": EventTypes.Redaction,
                 "content": content,
                 "room_id": room_id,
-                "sender": user.to_string(),
+                "sender": requester.user.to_string(),
                 "redacts": event_id,
             },
-            token_id=token_id,
+            token_id=requester.access_token_id,
             txn_id=txn_id,
         )
 
@@ -564,7 +555,7 @@ class RoomTypingRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, user_id):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         room_id = urllib.unquote(room_id)
         target_user = UserID.from_string(urllib.unquote(user_id))
@@ -576,14 +567,14 @@ class RoomTypingRestServlet(ClientV1RestServlet):
         if content["typing"]:
             yield typing_handler.started_typing(
                 target_user=target_user,
-                auth_user=auth_user,
+                auth_user=requester.user,
                 room_id=room_id,
                 timeout=content.get("timeout", 30000),
             )
         else:
             yield typing_handler.stopped_typing(
                 target_user=target_user,
-                auth_user=auth_user,
+                auth_user=requester.user,
                 room_id=room_id,
             )
 
@@ -597,12 +588,16 @@ class SearchRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         content = _parse_json(request)
 
         batch = request.args.get("next_batch", [None])[0]
-        results = yield self.handlers.search_handler.search(auth_user, content, batch)
+        results = yield self.handlers.search_handler.search(
+            requester.user,
+            content,
+            batch,
+        )
 
         defer.returnValue((200, results))
 
diff --git a/synapse/rest/client/v1/transactions.py b/synapse/rest/client/v1/transactions.py
index b861069b89..bdccf464a5 100644
--- a/synapse/rest/client/v1/transactions.py
+++ b/synapse/rest/client/v1/transactions.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py
index 1567a03c89..ec4cf8db79 100644
--- a/synapse/rest/client/v1/voip.py
+++ b/synapse/rest/client/v1/voip.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ class VoipRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        auth_user, _, _ = yield self.auth.get_user_by_req(request)
+        requester = yield self.auth.get_user_by_req(request)
 
         turnUris = self.hs.config.turn_uris
         turnSecret = self.hs.config.turn_shared_secret
@@ -37,7 +37,7 @@ class VoipRestServlet(ClientV1RestServlet):
             defer.returnValue((200, {}))
 
         expiry = (self.hs.get_clock().time_msec() + userLifetime) / 1000
-        username = "%d:%s" % (expiry, auth_user.to_string())
+        username = "%d:%s" % (expiry, requester.user.to_string())
 
         mac = hmac.new(turnSecret, msg=username, digestmod=hashlib.sha1)
         # We need to use standard padded base64 encoding here