Merge branch 'release-v0.2.2' of github.com:matrix-org/synapse v0.2.2

28 files changed, 475 insertions, 67 deletions

diff --git a/synapse/__init__.py b/synapse/__init__.py
index 440e633966..1ed9cdcdf3 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a synapse home server.
 """
 
-__version__ = "0.2.1"
+__version__ = "0.2.2"
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index b4eda3df01..8f32191b57 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -18,8 +18,8 @@
 from twisted.internet import defer
 
 from synapse.api.constants import Membership, JoinRules
-from synapse.api.errors import AuthError, StoreError, Codes
-from synapse.api.events.room import RoomMemberEvent
+from synapse.api.errors import AuthError, StoreError, Codes, SynapseError
+from synapse.api.events.room import RoomMemberEvent, RoomPowerLevelsEvent
 from synapse.util.logutils import log_function
 
 import logging
@@ -67,6 +67,9 @@ class Auth(object):
                 else:
                     yield self._can_send_event(event)
 
+                if event.type == RoomPowerLevelsEvent.TYPE:
+                    yield self._check_power_levels(event)
+
                 defer.returnValue(True)
             else:
                 raise AuthError(500, "Unknown event: %s" % event)
@@ -172,7 +175,7 @@ class Auth(object):
                 if kick_level:
                     kick_level = int(kick_level)
                 else:
-                    kick_level = 5
+                    kick_level = 50
 
                 if user_level < kick_level:
                     raise AuthError(
@@ -189,7 +192,7 @@ class Auth(object):
             if ban_level:
                 ban_level = int(ban_level)
             else:
-                ban_level = 5  # FIXME (erikj): What should we do here?
+                ban_level = 50  # FIXME (erikj): What should we do here?
 
             if user_level < ban_level:
                 raise AuthError(403, "You don't have permission to ban")
@@ -305,7 +308,9 @@ class Auth(object):
         else:
             user_level = 0
 
-        logger.debug("Checking power level for %s, %s", event.user_id, user_level)
+        logger.debug(
+            "Checking power level for %s, %s", event.user_id, user_level
+        )
         if current_state and hasattr(current_state, "required_power_level"):
             req = current_state.required_power_level
 
@@ -315,3 +320,101 @@ class Auth(object):
                     403,
                     "You don't have permission to change that state"
                 )
+
+    @defer.inlineCallbacks
+    def _check_power_levels(self, event):
+        for k, v in event.content.items():
+            if k == "default":
+                continue
+
+            # FIXME (erikj): We don't want hsob_Ts in content.
+            if k == "hsob_ts":
+                continue
+
+            try:
+                self.hs.parse_userid(k)
+            except:
+                raise SynapseError(400, "Not a valid user_id: %s" % (k,))
+
+            try:
+                int(v)
+            except:
+                raise SynapseError(400, "Not a valid power level: %s" % (v,))
+
+        current_state = yield self.store.get_current_state(
+            event.room_id,
+            event.type,
+            event.state_key,
+        )
+
+        if not current_state:
+            return
+        else:
+            current_state = current_state[0]
+
+        user_level = yield self.store.get_power_level(
+            event.room_id,
+            event.user_id,
+        )
+
+        if user_level:
+            user_level = int(user_level)
+        else:
+            user_level = 0
+
+        old_list = current_state.content
+
+        # FIXME (erikj)
+        old_people = {k: v for k, v in old_list.items() if k.startswith("@")}
+        new_people = {
+            k: v for k, v in event.content.items()
+            if k.startswith("@")
+        }
+
+        removed = set(old_people.keys()) - set(new_people.keys())
+        added = set(old_people.keys()) - set(new_people.keys())
+        same = set(old_people.keys()) & set(new_people.keys())
+
+        for r in removed:
+            if int(old_list.content[r]) > user_level:
+                raise AuthError(
+                    403,
+                    "You don't have permission to remove user: %s" % (r, )
+                )
+
+        for n in added:
+            if int(event.content[n]) > user_level:
+                raise AuthError(
+                    403,
+                    "You don't have permission to add ops level greater "
+                    "than your own"
+                )
+
+        for s in same:
+            if int(event.content[s]) != int(old_list[s]):
+                if int(event.content[s]) > user_level:
+                    raise AuthError(
+                        403,
+                        "You don't have permission to add ops level greater "
+                        "than your own"
+                    )
+
+        if "default" in old_list:
+            old_default = int(old_list["default"])
+
+            if old_default > user_level:
+                raise AuthError(
+                    403,
+                    "You don't have permission to add ops level greater than "
+                    "your own"
+                )
+
+            if "default" in event.content:
+                new_default = int(event.content["default"])
+
+                if new_default > user_level:
+                    raise AuthError(
+                        403,
+                        "You don't have permission to add ops level greater "
+                        "than your own"
+                    )
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 84afe4fa37..88175602c4 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -29,6 +29,8 @@ class Codes(object):
     NOT_FOUND = "M_NOT_FOUND"
     UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN"
     LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED"
+    CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED"
+    CAPTCHA_INVALID = "M_CAPTCHA_INVALID"
 
 
 class CodeMessageException(Exception):
@@ -101,6 +103,19 @@ class StoreError(SynapseError):
     pass
 
 
+class InvalidCaptchaError(SynapseError):
+    def __init__(self, code=400, msg="Invalid captcha.", error_url=None,
+                 errcode=Codes.CAPTCHA_INVALID):
+        super(InvalidCaptchaError, self).__init__(code, msg, errcode)
+        self.error_url = error_url
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            error_url=self.error_url,
+        )
+
 class LimitExceededError(SynapseError):
     """A client has sent too many requests and is being throttled.
     """
diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py
index f95468fc65..5f300de108 100644
--- a/synapse/api/events/__init__.py
+++ b/synapse/api/events/__init__.py
@@ -157,7 +157,12 @@ class SynapseEvent(JsonEncodedObject):
 
 
 class SynapseStateEvent(SynapseEvent):
-     def __init__(self, **kwargs):
+
+    valid_keys = SynapseEvent.valid_keys + [
+        "prev_content",
+    ]
+
+    def __init__(self, **kwargs):
         if "state_key" not in kwargs:
             kwargs["state_key"] = ""
         super(SynapseStateEvent, self).__init__(**kwargs)
diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py
index a3b293e024..5e38cdbc44 100644
--- a/synapse/api/events/factory.py
+++ b/synapse/api/events/factory.py
@@ -47,11 +47,14 @@ class EventFactory(object):
             self._event_list[event_class.TYPE] = event_class
 
         self.clock = hs.get_clock()
+        self.hs = hs
 
     def create_event(self, etype=None, **kwargs):
         kwargs["type"] = etype
         if "event_id" not in kwargs:
-            kwargs["event_id"] = random_string(10)
+            kwargs["event_id"] = "%s@%s" % (
+                random_string(10), self.hs.hostname
+            )
 
         if "ts" not in kwargs:
             kwargs["ts"] = int(self.clock.time_msec())
diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py
index 33f0f0cb99..3a4dbc58ce 100644
--- a/synapse/api/events/room.py
+++ b/synapse/api/events/room.py
@@ -173,3 +173,10 @@ class RoomOpsPowerLevelsEvent(SynapseStateEvent):
 
     def get_content_template(self):
         return {}
+
+
+class RoomAliasesEvent(SynapseStateEvent):
+    TYPE = "m.room.aliases"
+
+    def get_content_template(self):
+        return {}
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 49cf928cc1..d675d8c8f9 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -57,7 +57,7 @@ SCHEMAS = [
 
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 2
+SCHEMA_VERSION = 3
 
 
 class SynapseHomeServer(HomeServer):
diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py
new file mode 100644
index 0000000000..a97a5bab1e
--- /dev/null
+++ b/synapse/config/captcha.py
@@ -0,0 +1,42 @@
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+
+class CaptchaConfig(Config):
+
+    def __init__(self, args):
+        super(CaptchaConfig, self).__init__(args)
+        self.recaptcha_private_key = args.recaptcha_private_key
+        self.enable_registration_captcha = args.enable_registration_captcha
+        self.captcha_ip_origin_is_x_forwarded = args.captcha_ip_origin_is_x_forwarded
+
+    @classmethod
+    def add_arguments(cls, parser):
+        super(CaptchaConfig, cls).add_arguments(parser)
+        group = parser.add_argument_group("recaptcha")
+        group.add_argument(
+            "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY",
+            help="The matching private key for the web client's public key."
+        )
+        group.add_argument(
+            "--enable-registration-captcha", type=bool, default=False,
+            help="Enables ReCaptcha checks when registering, preventing signup "+
+            "unless a captcha is answered. Requires a valid ReCaptcha public/private key."
+        )
+        group.add_argument(
+            "--captcha_ip_origin_is_x_forwarded", type=bool, default=False,
+            help="When checking captchas, use the X-Forwarded-For (XFF) header as the client IP "+
+            "and not the actual client IP."
+        )
\ No newline at end of file
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 76e2cdeddd..e16f2c733b 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -19,9 +19,10 @@ from .logger import LoggingConfig
 from .database import DatabaseConfig
 from .ratelimiting import RatelimitConfig
 from .repository import ContentRepositoryConfig
+from .captcha import CaptchaConfig
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
-                       RatelimitConfig, ContentRepositoryConfig):
+                       RatelimitConfig, ContentRepositoryConfig, CaptchaConfig):
     pass
 
 if __name__=='__main__':
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 9989fe8670..de4d23bbb3 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -42,9 +42,6 @@ class BaseHandler(object):
                 retry_after_ms=int(1000*(time_allowed - time_now)),
             )
 
-
-class BaseRoomHandler(BaseHandler):
-
     @defer.inlineCallbacks
     def _on_new_room_event(self, event, snapshot, extra_destinations=[],
                            extra_users=[]):
diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py
index 1b9e831fc0..4ab00a761a 100644
--- a/synapse/handlers/directory.py
+++ b/synapse/handlers/directory.py
@@ -19,8 +19,10 @@ from ._base import BaseHandler
 
 from synapse.api.errors import SynapseError
 from synapse.http.client import HttpClient
+from synapse.api.events.room import RoomAliasesEvent
 
 import logging
+import sqlite3
 
 
 logger = logging.getLogger(__name__)
@@ -37,7 +39,8 @@ class DirectoryHandler(BaseHandler):
         )
 
     @defer.inlineCallbacks
-    def create_association(self, room_alias, room_id, servers=None):
+    def create_association(self, user_id, room_alias, room_id, servers=None):
+
         # TODO(erikj): Do auth.
 
         if not room_alias.is_mine:
@@ -54,12 +57,37 @@ class DirectoryHandler(BaseHandler):
         if not servers:
             raise SynapseError(400, "Failed to get server list")
 
-        yield self.store.create_room_alias_association(
-            room_alias,
-            room_id,
-            servers
+
+        try:
+            yield self.store.create_room_alias_association(
+                room_alias,
+                room_id,
+                servers
+            )
+        except sqlite3.IntegrityError:
+            defer.returnValue("Already exists")
+
+        # TODO: Send the room event.
+
+        aliases = yield self.store.get_aliases_for_room(room_id)
+
+        event = self.event_factory.create_event(
+            etype=RoomAliasesEvent.TYPE,
+            state_key=self.hs.hostname,
+            room_id=room_id,
+            user_id=user_id,
+            content={"aliases": aliases},
+        )
+
+        snapshot = yield self.store.snapshot_room(
+            room_id=room_id,
+            user_id=user_id,
         )
 
+        yield self.state_handler.handle_new_event(event, snapshot)
+        yield self._on_new_room_event(event, snapshot, extra_users=[user_id])
+
+
     @defer.inlineCallbacks
     def get_association(self, room_alias):
         room_id = None
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index dad2bbd1a4..87fc04478b 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -19,7 +19,7 @@ from synapse.api.constants import Membership
 from synapse.api.events.room import RoomTopicEvent
 from synapse.api.errors import RoomError
 from synapse.streams.config import PaginationConfig
-from ._base import BaseRoomHandler
+from ._base import BaseHandler
 
 import logging
 
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 
 
 
-class MessageHandler(BaseRoomHandler):
+class MessageHandler(BaseHandler):
 
     def __init__(self, hs):
         super(MessageHandler, self).__init__(hs)
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index c79bb6ff76..b2af09f090 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -796,11 +796,12 @@ class PresenceEventSource(object):
         updates = []
         # TODO(paul): use a DeferredList ? How to limit concurrency.
         for observed_user in cachemap.keys():
-            if not (from_key < cachemap[observed_user].serial):
+            cached = cachemap[observed_user]
+            if not (from_key < cached.serial):
                 continue
 
             if (yield self.is_visible(observer_user, observed_user)):
-                updates.append((observed_user, cachemap[observed_user]))
+                updates.append((observed_user, cached))
 
         # TODO(paul): limit
 
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index bee052274f..0b841d6d3a 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -17,7 +17,9 @@
 from twisted.internet import defer
 
 from synapse.types import UserID
-from synapse.api.errors import SynapseError, RegistrationError
+from synapse.api.errors import (
+    SynapseError, RegistrationError, InvalidCaptchaError
+)
 from ._base import BaseHandler
 import synapse.util.stringutils as stringutils
 from synapse.http.client import PlainHttpClient
@@ -38,7 +40,8 @@ class RegistrationHandler(BaseHandler):
         self.distributor.declare("registered_user")
 
     @defer.inlineCallbacks
-    def register(self, localpart=None, password=None, threepidCreds=None):
+    def register(self, localpart=None, password=None, threepidCreds=None, 
+                 captcha_info={}):
         """Registers a new client on the server.
 
         Args:
@@ -51,10 +54,26 @@ class RegistrationHandler(BaseHandler):
         Raises:
             RegistrationError if there was a problem registering.
         """
+        if captcha_info:
+            captcha_response = yield self._validate_captcha(
+                captcha_info["ip"], 
+                captcha_info["private_key"],
+                captcha_info["challenge"],
+                captcha_info["response"]
+            )
+            if not captcha_response["valid"]:
+                logger.info("Invalid captcha entered from %s. Error: %s", 
+                            captcha_info["ip"], captcha_response["error_url"])
+                raise InvalidCaptchaError(
+                    error_url=captcha_response["error_url"]
+                )
+            else:
+                logger.info("Valid captcha entered from %s", captcha_info["ip"])
 
         if threepidCreds:
             for c in threepidCreds:
-                logger.info("validating theeepidcred sid %s on id server %s", c['sid'], c['idServer'])
+                logger.info("validating theeepidcred sid %s on id server %s",
+                            c['sid'], c['idServer'])
                 try:
                     threepid = yield self._threepid_from_creds(c)
                 except:
@@ -63,7 +82,8 @@ class RegistrationHandler(BaseHandler):
                     
                 if not threepid:
                     raise RegistrationError(400, "Couldn't validate 3pid")
-                logger.info("got threepid medium %s address %s", threepid['medium'], threepid['address'])
+                logger.info("got threepid medium %s address %s", 
+                            threepid['medium'], threepid['address'])
 
         password_hash = None
         if password:
@@ -131,7 +151,8 @@ class RegistrationHandler(BaseHandler):
         # XXX: make this configurable!
         trustedIdServers = [ 'matrix.org:8090' ]
         if not creds['idServer'] in trustedIdServers:
-            logger.warn('%s is not a trusted ID server: rejecting 3pid credentials', creds['idServer'])
+            logger.warn('%s is not a trusted ID server: rejecting 3pid '+
+                        'credentials', creds['idServer'])
             defer.returnValue(None)
         data = yield httpCli.get_json(
             creds['idServer'],
@@ -149,9 +170,44 @@ class RegistrationHandler(BaseHandler):
         data = yield httpCli.post_urlencoded_get_json(
             creds['idServer'],
             "/_matrix/identity/api/v1/3pid/bind",
-            { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], 'mxid':mxid }
+            { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], 
+            'mxid':mxid }
         )
         defer.returnValue(data)
         
+    @defer.inlineCallbacks
+    def _validate_captcha(self, ip_addr, private_key, challenge, response):
+        """Validates the captcha provided.
+        
+        Returns:
+            dict: Containing 'valid'(bool) and 'error_url'(str) if invalid.
+        
+        """
+        response = yield self._submit_captcha(ip_addr, private_key, challenge, 
+                                              response)
+        # parse Google's response. Lovely format..
+        lines = response.split('\n')
+        json = {
+            "valid": lines[0] == 'true',
+            "error_url": "http://www.google.com/recaptcha/api/challenge?"+
+                         "error=%s" % lines[1]
+        }
+        defer.returnValue(json)
+        
+    @defer.inlineCallbacks
+    def _submit_captcha(self, ip_addr, private_key, challenge, response):
+        client = PlainHttpClient(self.hs)
+        data = yield client.post_urlencoded_get_raw(
+            "www.google.com:80",
+            "/recaptcha/api/verify",
+            accept_partial=True,  # twisted dislikes google's response, no content length.
+            args={ 
+                'privatekey': private_key, 
+                'remoteip': ip_addr,
+                'challenge': challenge,
+                'response': response
+            }
+        )
+        defer.returnValue(data)
         
 
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 8171e9eb45..a0d0f2af16 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -25,14 +25,14 @@ from synapse.api.events.room import (
     RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent, RoomNameEvent,
 )
 from synapse.util import stringutils
-from ._base import BaseRoomHandler
+from ._base import BaseHandler
 
 import logging
 
 logger = logging.getLogger(__name__)
 
 
-class RoomCreationHandler(BaseRoomHandler):
+class RoomCreationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def create_room(self, user_id, room_id, config):
@@ -65,6 +65,13 @@ class RoomCreationHandler(BaseRoomHandler):
         else:
             room_alias = None
 
+        invite_list = config.get("invite", [])
+        for i in invite_list:
+            try:
+                self.hs.parse_userid(i)
+            except:
+                raise SynapseError(400, "Invalid user_id: %s" % (i,))
+
         is_public = config.get("visibility", None) == "public"
 
         if room_id:
@@ -105,7 +112,9 @@ class RoomCreationHandler(BaseRoomHandler):
         )
 
         if room_alias:
-            yield self.store.create_room_alias_association(
+            directory_handler = self.hs.get_handlers().directory_handler
+            yield directory_handler.create_association(
+                user_id=user_id,
                 room_id=room_id,
                 room_alias=room_alias,
                 servers=[self.hs.hostname],
@@ -132,7 +141,7 @@ class RoomCreationHandler(BaseRoomHandler):
                 etype=RoomNameEvent.TYPE,
                 room_id=room_id,
                 user_id=user_id,
-                required_power_level=5,
+                required_power_level=50,
                 content={"name": name},
             )
 
@@ -143,7 +152,7 @@ class RoomCreationHandler(BaseRoomHandler):
                 etype=RoomNameEvent.TYPE,
                 room_id=room_id,
                 user_id=user_id,
-                required_power_level=5,
+                required_power_level=50,
                 content={"name": name},
             )
 
@@ -155,7 +164,7 @@ class RoomCreationHandler(BaseRoomHandler):
                 etype=RoomTopicEvent.TYPE,
                 room_id=room_id,
                 user_id=user_id,
-                required_power_level=5,
+                required_power_level=50,
                 content={"topic": topic},
             )
 
@@ -176,6 +185,25 @@ class RoomCreationHandler(BaseRoomHandler):
             do_auth=False
         )
 
+        content = {"membership": Membership.INVITE}
+        for invitee in invite_list:
+            invite_event = self.event_factory.create_event(
+                etype=RoomMemberEvent.TYPE,
+                state_key=invitee,
+                room_id=room_id,
+                user_id=user_id,
+                content=content
+            )
+
+            yield self.hs.get_handlers().room_member_handler.change_membership(
+                invite_event,
+                do_auth=False
+            )
+
+        yield self.hs.get_handlers().room_member_handler.change_membership(
+            join_event,
+            do_auth=False
+        )
         result = {"room_id": room_id}
         if room_alias:
             result["room_alias"] = room_alias.to_string()
@@ -186,7 +214,7 @@ class RoomCreationHandler(BaseRoomHandler):
         event_keys = {
             "room_id": room_id,
             "user_id": creator.to_string(),
-            "required_power_level": 10,
+            "required_power_level": 100,
         }
 
         def create(etype, **content):
@@ -203,7 +231,7 @@ class RoomCreationHandler(BaseRoomHandler):
 
         power_levels_event = self.event_factory.create_event(
             etype=RoomPowerLevelsEvent.TYPE,
-            content={creator.to_string(): 10, "default": 0},
+            content={creator.to_string(): 100, "default": 0},
             **event_keys
         )
 
@@ -215,7 +243,7 @@ class RoomCreationHandler(BaseRoomHandler):
 
         add_state_event = create(
             etype=RoomAddStateLevelEvent.TYPE,
-            level=10,
+            level=100,
         )
 
         send_event = create(
@@ -225,8 +253,8 @@ class RoomCreationHandler(BaseRoomHandler):
 
         ops = create(
             etype=RoomOpsPowerLevelsEvent.TYPE,
-            ban_level=5,
-            kick_level=5,
+            ban_level=50,
+            kick_level=50,
         )
 
         return [
@@ -239,7 +267,7 @@ class RoomCreationHandler(BaseRoomHandler):
         ]
 
 
-class RoomMemberHandler(BaseRoomHandler):
+class RoomMemberHandler(BaseHandler):
     # TODO(paul): This handler currently contains a messy conflation of
     #   low-level API that works on UserID objects and so on, and REST-level
     #   API that takes ID strings and returns pagination chunks. These concerns
@@ -560,7 +588,7 @@ class RoomMemberHandler(BaseRoomHandler):
             extra_users=[target_user]
         )
 
-class RoomListHandler(BaseRoomHandler):
+class RoomListHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def get_public_room_list(self):
diff --git a/synapse/http/client.py b/synapse/http/client.py
index ebf1aa47c4..ece6318e00 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -16,7 +16,7 @@
 
 from twisted.internet import defer, reactor
 from twisted.internet.error import DNSLookupError
-from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer
+from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer, PartialDownloadError
 from twisted.web.http_headers import Headers
 
 from synapse.http.endpoint import matrix_endpoint
@@ -188,6 +188,32 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
 
         defer.returnValue(json.loads(body))
+        
+    # XXX FIXME : I'm so sorry.
+    @defer.inlineCallbacks
+    def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}):
+        if destination in _destination_mappings:
+            destination = _destination_mappings[destination]
+
+        query_bytes = urllib.urlencode(args, True)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "POST",
+            path.encode("ascii"),
+            producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
+            headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]}
+        )
+
+        try:
+            body = yield readBody(response)
+            defer.returnValue(body)
+        except PartialDownloadError as e:
+            if accept_partial:
+                defer.returnValue(e.response)
+            else:
+                raise e
+        
 
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes, param_bytes=b"",
diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py
index 18df7c8d8b..31849246a1 100644
--- a/synapse/rest/directory.py
+++ b/synapse/rest/directory.py
@@ -45,6 +45,8 @@ class ClientDirectoryServer(RestServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_alias):
+        user = yield self.auth.get_user_by_req(request)
+
         content = _parse_json(request)
         if not "room_id" in content:
             raise SynapseError(400, "Missing room_id key",
@@ -69,12 +71,13 @@ class ClientDirectoryServer(RestServlet):
 
         try:
             yield dir_handler.create_association(
-                room_alias, room_id, servers
+                user.to_string(), room_alias, room_id, servers
             )
         except SynapseError as e:
             raise e
         except:
             logger.exception("Failed to create association")
+            raise
 
         defer.returnValue((200, {}))
 
diff --git a/synapse/rest/login.py b/synapse/rest/login.py
index c7bf901c8e..ba49afcaa7 100644
--- a/synapse/rest/login.py
+++ b/synapse/rest/login.py
@@ -70,7 +70,7 @@ class LoginFallbackRestServlet(RestServlet):
     def on_GET(self, request):
         # TODO(kegan): This should be returning some HTML which is capable of
         # hitting LoginRestServlet
-        return (200, "")
+        return (200, {})
 
 
 def _parse_json(request):
diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py
index 2e17f87fa1..dad5a208c7 100644
--- a/synapse/rest/profile.py
+++ b/synapse/rest/profile.py
@@ -51,7 +51,7 @@ class ProfileDisplaynameRestServlet(RestServlet):
         yield self.handlers.profile_handler.set_displayname(
             user, auth_user, new_name)
 
-        defer.returnValue((200, ""))
+        defer.returnValue((200, {}))
 
     def on_OPTIONS(self, request, user_id):
         return (200, {})
@@ -86,7 +86,7 @@ class ProfileAvatarURLRestServlet(RestServlet):
         yield self.handlers.profile_handler.set_avatar_url(
             user, auth_user, new_name)
 
-        defer.returnValue((200, ""))
+        defer.returnValue((200, {}))
 
     def on_OPTIONS(self, request, user_id):
         return (200, {})
diff --git a/synapse/rest/register.py b/synapse/rest/register.py
index b8de3b250d..48d3c6eca0 100644
--- a/synapse/rest/register.py
+++ b/synapse/rest/register.py
@@ -16,7 +16,7 @@
 """This module contains REST servlets to do with registration: /register"""
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError
+from synapse.api.errors import SynapseError, Codes
 from base import RestServlet, client_path_pattern
 
 import json
@@ -50,12 +50,44 @@ class RegisterRestServlet(RestServlet):
         threepidCreds = None
         if 'threepidCreds' in register_json:
             threepidCreds = register_json['threepidCreds']
+            
+        captcha = {}
+        if self.hs.config.enable_registration_captcha:
+            challenge = None
+            user_response = None
+            try:
+                captcha_type = register_json["captcha"]["type"]
+                if captcha_type != "m.login.recaptcha":
+                    raise SynapseError(400, "Sorry, only m.login.recaptcha " +
+                                       "requests are supported.")
+                challenge = register_json["captcha"]["challenge"]
+                user_response = register_json["captcha"]["response"]
+            except KeyError:
+                raise SynapseError(400, "Captcha response is required",
+                                   errcode=Codes.CAPTCHA_NEEDED)
+            
+            # TODO determine the source IP : May be an X-Forwarding-For header depending on config
+            ip_addr = request.getClientIP()
+            if self.hs.config.captcha_ip_origin_is_x_forwarded:
+                # use the header
+                if request.requestHeaders.hasHeader("X-Forwarded-For"):
+                    ip_addr = request.requestHeaders.getRawHeaders(
+                        "X-Forwarded-For")[0]
+            
+            captcha = {
+                "ip": ip_addr,
+                "private_key": self.hs.config.recaptcha_private_key,
+                "challenge": challenge,
+                "response": user_response
+            }
+            
 
         handler = self.handlers.registration_handler
         (user_id, token) = yield handler.register(
             localpart=desired_user_id,
             password=password,
-            threepidCreds=threepidCreds)
+            threepidCreds=threepidCreds,
+            captcha_info=captcha)
 
         result = {
             "user_id": user_id,
diff --git a/synapse/rest/room.py b/synapse/rest/room.py
index 308b447090..cef700c81c 100644
--- a/synapse/rest/room.py
+++ b/synapse/rest/room.py
@@ -154,14 +154,14 @@ class RoomStateEventRestServlet(RestServlet):
             # membership events are special
             handler = self.handlers.room_member_handler
             yield handler.change_membership(event)
-            defer.returnValue((200, ""))
+            defer.returnValue((200, {}))
         else:
             # store random bits of state
             msg_handler = self.handlers.message_handler
             yield msg_handler.store_room_data(
                 event=event
             )
-            defer.returnValue((200, ""))
+            defer.returnValue((200, {}))
 
 
 # TODO: Needs unit testing for generic events + feedback
@@ -249,7 +249,7 @@ class JoinRoomAliasServlet(RestServlet):
             )
             handler = self.handlers.room_member_handler
             yield handler.change_membership(event)
-            defer.returnValue((200, ""))
+            defer.returnValue((200, {}))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_identifier, txn_id):
@@ -416,7 +416,7 @@ class RoomMembershipRestServlet(RestServlet):
         )
         handler = self.handlers.room_member_handler
         yield handler.change_membership(event)
-        defer.returnValue((200, ""))
+        defer.returnValue((200, {}))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, room_id, membership_action, txn_id):
diff --git a/synapse/state.py b/synapse/state.py
index 36d8210eb5..5dcff27367 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -16,7 +16,7 @@
 
 from twisted.internet import defer
 
-from synapse.federation.pdu_codec import encode_event_id
+from synapse.federation.pdu_codec import encode_event_id, decode_event_id
 from synapse.util.logutils import log_function
 
 from collections import namedtuple
@@ -87,9 +87,11 @@ class StateHandler(object):
         # than the power level of the user
         # power_level = self._get_power_level_for_event(event)
 
+        pdu_id, origin = decode_event_id(event.event_id, self.server_name)
+
         yield self.store.update_current_state(
-            pdu_id=event.event_id,
-            origin=self.server_name,
+            pdu_id=pdu_id,
+            origin=origin,
             context=key.context,
             pdu_type=key.type,
             state_key=key.state_key
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index d97014f4da..81c3c94b2e 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -81,7 +81,7 @@ class DataStore(RoomMemberStore, RoomStore,
         defer.returnValue(latest)
 
     @defer.inlineCallbacks
-    def get_event(self, event_id):
+    def get_event(self, event_id, allow_none=False):
         events_dict = yield self._simple_select_one(
             "events",
             {"event_id": event_id},
@@ -92,8 +92,12 @@ class DataStore(RoomMemberStore, RoomStore,
                 "content",
                 "unrecognized_keys"
             ],
+            allow_none=allow_none,
         )
 
+        if not events_dict:
+            defer.returnValue(None)
+
         event = self._parse_event_from_row(events_dict)
         defer.returnValue(event)
 
@@ -220,7 +224,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
         results = yield self._execute_and_decode(sql, *args)
 
-        defer.returnValue([self._parse_event_from_row(r) for r in results])
+        events = yield self._parse_events(results)
+        defer.returnValue(events)
 
     @defer.inlineCallbacks
     def _get_min_token(self):
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index bae50e7d1f..8037225079 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -312,6 +312,25 @@ class SQLBaseStore(object):
             **d
         )
 
+    def _parse_events(self, rows):
+        return self._db_pool.runInteraction(self._parse_events_txn, rows)
+
+    def _parse_events_txn(self, txn, rows):
+        events = [self._parse_event_from_row(r) for r in rows]
+
+        sql = "SELECT * FROM events WHERE event_id = ?"
+
+        for ev in events:
+           if hasattr(ev, "prev_state"):
+                # Load previous state_content. 
+                # TODO: Should we be pulling this out above?
+                cursor = txn.execute(sql, (ev.prev_state,))
+                prevs = self.cursor_to_dict(cursor)
+                if prevs:
+                    prev = self._parse_event_from_row(prevs[0])
+                    ev.prev_content = prev.content
+
+        return events
 
 class Table(object):
     """ A base class used to store information about a particular table.
diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py
index bf55449253..540eb4c2c4 100644
--- a/synapse/storage/directory.py
+++ b/synapse/storage/directory.py
@@ -92,3 +92,10 @@ class DirectoryStore(SQLBaseStore):
                     "server": server,
                 }
             )
+
+    def get_aliases_for_room(self, room_id):
+        return self._simple_select_onecol(
+            "room_aliases",
+            {"room_id": room_id},
+            "room_alias",
+        )
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 75c9a60101..9a393e2568 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -88,7 +88,7 @@ class RoomMemberStore(SQLBaseStore):
         txn.execute(sql, (user_id, room_id))
         rows = self.cursor_to_dict(txn)
         if rows:
-            return self._parse_event_from_row(rows[0])
+            return self._parse_events_txn(txn, rows)[0]
         else:
             return None
 
@@ -161,7 +161,7 @@ class RoomMemberStore(SQLBaseStore):
 
         # logger.debug("_get_members_query Got rows %s", rows)
 
-        results = [self._parse_event_from_row(r) for r in rows]
+        results = yield self._parse_events(rows)
         defer.returnValue(results)
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql
new file mode 100644
index 0000000000..cade295989
--- /dev/null
+++ b/synapse/storage/schema/delta/v3.sql
@@ -0,0 +1,27 @@
+/* Copyright 2014 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+CREATE INDEX IF NOT EXISTS room_aliases_alias ON room_aliases(room_alias);
+CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id);
+
+
+CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias);
+
+DELETE FROM room_aliases WHERE rowid NOT IN (SELECT max(rowid) FROM room_aliases GROUP BY room_alias, room_id);
+
+CREATE UNIQUE INDEX IF NOT EXISTS room_aliases_uniq ON room_aliases(room_alias, room_id);
+
+PRAGMA user_version = 3;
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 2cb0067a67..aff6dc9855 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -188,7 +188,7 @@ class StreamStore(SQLBaseStore):
             user_id, user_id, from_id, to_id
         )
 
-        ret = [self._parse_event_from_row(r) for r in rows]
+        ret = yield self._parse_events(rows)
 
         if rows:
             key = "s%d" % max([r["stream_ordering"] for r in rows])
@@ -243,9 +243,11 @@ class StreamStore(SQLBaseStore):
             # TODO (erikj): We should work out what to do here instead.
             next_token = to_key if to_key else from_key
 
+        events = yield self._parse_events(rows)
+
         defer.returnValue(
             (
-                [self._parse_event_from_row(r) for r in rows],
+                events,
                 next_token
             )
         )
@@ -277,12 +279,11 @@ class StreamStore(SQLBaseStore):
         else:
             token = (end_token, end_token)
 
-        defer.returnValue(
-            (
-                [self._parse_event_from_row(r) for r in rows],
-                token
-            )
-        )
+        events = yield self._parse_events(rows)
+
+        ret = (events, token)
+
+        defer.returnValue(ret)
 
     def get_room_events_max_id(self):
         return self._db_pool.runInteraction(self._get_room_events_max_id_txn)