Merge branch 'release-v0.3.0' of github.com:matrix-org/synapse v0.3.0

24 files changed, 599 insertions, 158 deletions

diff --git a/synapse/__init__.py b/synapse/__init__.py
index d60267ebe4..8ef176ea6f 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -16,4 +16,4 @@
 """ This is a reference implementation of a synapse home server.
 """
 
-__version__ = "0.2.3"
+__version__ = "0.3.0"
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index fcef062fc9..618d3d7577 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -50,3 +50,12 @@ class JoinRules(object):
     KNOCK = u"knock"
     INVITE = u"invite"
     PRIVATE = u"private"
+
+
+class LoginType(object):
+    PASSWORD = u"m.login.password"
+    OAUTH = u"m.login.oauth2"
+    EMAIL_CODE = u"m.login.email.code"
+    EMAIL_URL = u"m.login.email.url"
+    EMAIL_IDENTITY = u"m.login.email.identity"
+    RECAPTCHA = u"m.login.recaptcha"
\ No newline at end of file
diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py
index 5f300de108..a9991e9c94 100644
--- a/synapse/api/events/__init__.py
+++ b/synapse/api/events/__init__.py
@@ -17,6 +17,19 @@ from synapse.api.errors import SynapseError, Codes
 from synapse.util.jsonobject import JsonEncodedObject
 
 
+def serialize_event(hs, e):
+    # FIXME(erikj): To handle the case of presence events and the like
+    if not isinstance(e, SynapseEvent):
+        return e
+
+    d = e.get_dict()
+    if "age_ts" in d:
+        d["age"] = int(hs.get_clock().time_msec()) - d["age_ts"]
+        del d["age_ts"]
+
+    return d
+
+
 class SynapseEvent(JsonEncodedObject):
 
     """Base class for Synapse events. These are JSON objects which must abide
@@ -43,6 +56,8 @@ class SynapseEvent(JsonEncodedObject):
         "content",  # HTTP body, JSON
         "state_key",
         "required_power_level",
+        "age_ts",
+        "prev_content",
     ]
 
     internal_keys = [
@@ -158,10 +173,6 @@ class SynapseEvent(JsonEncodedObject):
 
 class SynapseStateEvent(SynapseEvent):
 
-    valid_keys = SynapseEvent.valid_keys + [
-        "prev_content",
-    ]
-
     def __init__(self, **kwargs):
         if "state_key" not in kwargs:
             kwargs["state_key"] = ""
diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py
index 5e38cdbc44..d3d96d73eb 100644
--- a/synapse/api/events/factory.py
+++ b/synapse/api/events/factory.py
@@ -59,6 +59,14 @@ class EventFactory(object):
         if "ts" not in kwargs:
             kwargs["ts"] = int(self.clock.time_msec())
 
+        # The "age" key is a delta timestamp that should be converted into an
+        # absolute timestamp the minute we see it.
+        if "age" in kwargs:
+            kwargs["age_ts"] = int(self.clock.time_msec()) - int(kwargs["age"])
+            del kwargs["age"]
+        elif "age_ts" not in kwargs:
+            kwargs["age_ts"] = int(self.clock.time_msec())
+
         if etype in self._event_list:
             handler = self._event_list[etype]
         else:
diff --git a/synapse/config/email.py b/synapse/config/email.py
new file mode 100644
index 0000000000..9bcc5a8fea
--- /dev/null
+++ b/synapse/config/email.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import Config
+
+
+class EmailConfig(Config):
+
+    def __init__(self, args):
+        super(EmailConfig, self).__init__(args)
+        self.email_from_address = args.email_from_address
+        self.email_smtp_server = args.email_smtp_server
+
+    @classmethod
+    def add_arguments(cls, parser):
+        super(EmailConfig, cls).add_arguments(parser)
+        email_group = parser.add_argument_group("email")
+        email_group.add_argument(
+            "--email-from-address",
+            default="FROM@EXAMPLE.COM",
+            help="The address to send emails from (e.g. for password resets)."
+        )
+        email_group.add_argument(
+            "--email-smtp-server",
+            default="",
+            help="The SMTP server to send emails from (e.g. for password resets)."
+        )
\ No newline at end of file
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index e16f2c733b..4b810a2302 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -20,11 +20,15 @@ from .database import DatabaseConfig
 from .ratelimiting import RatelimitConfig
 from .repository import ContentRepositoryConfig
 from .captcha import CaptchaConfig
+from .email import EmailConfig
+
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
-                       RatelimitConfig, ContentRepositoryConfig, CaptchaConfig):
+                       RatelimitConfig, ContentRepositoryConfig, CaptchaConfig,
+                       EmailConfig):
     pass
 
-if __name__=='__main__':
+
+if __name__ == '__main__':
     import sys
     HomeServerConfig.load_config("Generate config", sys.argv[1:], "HomeServer")
diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py
index e12510017f..96b82f00cb 100644
--- a/synapse/federation/replication.py
+++ b/synapse/federation/replication.py
@@ -291,6 +291,13 @@ class ReplicationLayer(object):
     def on_incoming_transaction(self, transaction_data):
         transaction = Transaction(**transaction_data)
 
+        for p in transaction.pdus:
+            if "age" in p:
+                p["age_ts"] = int(self._clock.time_msec()) - int(p["age"])
+                del p["age"]
+
+        pdu_list = [Pdu(**p) for p in transaction.pdus]
+
         logger.debug("[%s] Got transaction", transaction.transaction_id)
 
         response = yield self.transaction_actions.have_responded(transaction)
@@ -303,8 +310,6 @@ class ReplicationLayer(object):
 
         logger.debug("[%s] Transacition is new", transaction.transaction_id)
 
-        pdu_list = [Pdu(**p) for p in transaction.pdus]
-
         dl = []
         for pdu in pdu_list:
             dl.append(self._handle_new_pdu(pdu))
@@ -405,9 +410,14 @@ class ReplicationLayer(object):
         """Returns a new Transaction containing the given PDUs suitable for
         transmission.
         """
+        pdus = [p.get_dict() for p in pdu_list]
+        for p in pdus:
+            if "age_ts" in pdus:
+                p["age"] = int(self.clock.time_msec()) - p["age_ts"]
+
         return Transaction(
-            pdus=[p.get_dict() for p in pdu_list],
             origin=self.server_name,
+            pdus=pdus,
             ts=int(self._clock.time_msec()),
             destination=None,
         )
@@ -593,8 +603,21 @@ class _TransactionQueue(object):
             logger.debug("TX [%s] Sending transaction...", destination)
 
             # Actually send the transaction
+
+            # FIXME (erikj): This is a bit of a hack to make the Pdu age
+            # keys work
+            def cb(transaction):
+                now = int(self._clock.time_msec())
+                if "pdus" in transaction:
+                    for p in transaction["pdus"]:
+                        if "age_ts" in p:
+                            p["age"] = now - int(p["age_ts"])
+
+                return transaction
+
             code, response = yield self.transport_layer.send_transaction(
-                transaction
+                transaction,
+                on_send_callback=cb,
             )
 
             logger.debug("TX [%s] Sent transaction", destination)
diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py
index 6e62ae7c74..afc777ec9e 100644
--- a/synapse/federation/transport.py
+++ b/synapse/federation/transport.py
@@ -144,7 +144,7 @@ class TransportLayer(object):
 
     @defer.inlineCallbacks
     @log_function
-    def send_transaction(self, transaction):
+    def send_transaction(self, transaction, on_send_callback=None):
         """ Sends the given Transaction to it's destination
 
         Args:
@@ -165,10 +165,23 @@ class TransportLayer(object):
 
         data = transaction.get_dict()
 
+        # FIXME (erikj): This is a bit of a hack to make the Pdu age
+        # keys work
+        def cb(destination, method, path_bytes, producer):
+            if not on_send_callback:
+                return
+
+            transaction = json.loads(producer.body)
+
+            new_transaction = on_send_callback(transaction)
+
+            producer.reset(new_transaction)
+
         code, response = yield self.client.put_json(
             transaction.destination,
             path=PREFIX + "/send/%s/" % transaction.transaction_id,
-            data=data
+            data=data,
+            on_send_callback=cb,
         )
 
         logger.debug(
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index fd24a11fb8..93dcd40324 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -15,7 +15,6 @@
 
 from twisted.internet import defer
 
-from synapse.api.events import SynapseEvent
 from synapse.util.logutils import log_function
 
 from ._base import BaseHandler
@@ -71,10 +70,7 @@ class EventStreamHandler(BaseHandler):
                 auth_user, room_ids, pagin_config, timeout
             )
 
-            chunks = [
-                e.get_dict() if isinstance(e, SynapseEvent) else e
-                for e in events
-            ]
+            chunks = [self.hs.serialize_event(e) for e in events]
 
             chunk = {
                 "chunk": chunks,
@@ -92,7 +88,9 @@ class EventStreamHandler(BaseHandler):
                 # 10 seconds of grace to allow the client to reconnect again
                 #   before we think they're gone
                 def _later():
-                    logger.debug("_later stopped_user_eventstream %s", auth_user)
+                    logger.debug(
+                        "_later stopped_user_eventstream %s", auth_user
+                    )
                     self.distributor.fire(
                         "stopped_user_eventstream", auth_user
                     )
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 59cbf71d78..001c6c110c 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -93,22 +93,18 @@ class FederationHandler(BaseHandler):
         """
         event = self.pdu_codec.event_from_pdu(pdu)
 
+        logger.debug("Got event: %s", event.event_id)
+
         with (yield self.lock_manager.lock(pdu.context)):
             if event.is_state and not backfilled:
                 is_new_state = yield self.state_handler.handle_new_state(
                     pdu
                 )
-                if not is_new_state:
-                    return
             else:
                 is_new_state = False
         # TODO: Implement something in federation that allows us to
         # respond to PDU.
 
-        if hasattr(event, "state_key") and not is_new_state:
-            logger.debug("Ignoring old state.")
-            return
-
         target_is_mine = False
         if hasattr(event, "target_host"):
             target_is_mine = event.target_host == self.hs.hostname
@@ -139,7 +135,11 @@ class FederationHandler(BaseHandler):
 
         else:
             with (yield self.room_lock.lock(event.room_id)):
-                yield self.store.persist_event(event, backfilled)
+                yield self.store.persist_event(
+                    event,
+                    backfilled,
+                    is_new_state=is_new_state
+                )
 
             room = yield self.store.get_room(event.room_id)
 
diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 6ee7ce5a2d..80ffdd2726 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -17,9 +17,13 @@ from twisted.internet import defer
 
 from ._base import BaseHandler
 from synapse.api.errors import LoginError, Codes
+from synapse.http.client import PlainHttpClient
+from synapse.util.emailutils import EmailException
+import synapse.util.emailutils as emailutils
 
 import bcrypt
 import logging
+import urllib
 
 logger = logging.getLogger(__name__)
 
@@ -62,4 +66,41 @@ class LoginHandler(BaseHandler):
             defer.returnValue(token)
         else:
             logger.warn("Failed password login for user %s", user)
-            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
\ No newline at end of file
+            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+
+    @defer.inlineCallbacks
+    def reset_password(self, user_id, email):
+        is_valid = yield self._check_valid_association(user_id, email)
+        logger.info("reset_password user=%s email=%s valid=%s", user_id, email,
+                    is_valid)
+        if is_valid:
+            try:
+                # send an email out
+                emailutils.send_email(
+                    smtp_server=self.hs.config.email_smtp_server,
+                    from_addr=self.hs.config.email_from_address,
+                    to_addr=email,
+                    subject="Password Reset",
+                    body="TODO."
+                )
+            except EmailException as e:
+                logger.exception(e)
+
+    @defer.inlineCallbacks
+    def _check_valid_association(self, user_id, email):
+        identity = yield self._query_email(email)
+        if identity and "mxid" in identity:
+            if identity["mxid"] == user_id:
+                defer.returnValue(True)
+                return
+        defer.returnValue(False)
+
+    @defer.inlineCallbacks
+    def _query_email(self, email):
+        httpCli = PlainHttpClient(self.hs)
+        data = yield httpCli.get_json(
+            'matrix.org:8090',  # TODO FIXME This should be configurable.
+            "/_matrix/identity/api/v1/lookup?medium=email&address=" +
+            "%s" % urllib.quote(email)
+        )
+        defer.returnValue(data)
\ No newline at end of file
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 87fc04478b..14fae689f2 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -124,7 +124,7 @@ class MessageHandler(BaseHandler):
         )
 
         chunk = {
-            "chunk": [e.get_dict() for e in events],
+            "chunk": [self.hs.serialize_event(e) for e in events],
             "start": pagin_config.from_token.to_string(),
             "end": next_token.to_string(),
         }
@@ -268,6 +268,9 @@ class MessageHandler(BaseHandler):
             user, pagination_config, None
         )
 
+        public_rooms = yield self.store.get_rooms(is_public=True)
+        public_room_ids = [r["room_id"] for r in public_rooms]
+
         limit = pagin_config.limit
         if not limit:
             limit = 10
@@ -276,6 +279,8 @@ class MessageHandler(BaseHandler):
             d = {
                 "room_id": event.room_id,
                 "membership": event.membership,
+                "visibility": ("public" if event.room_id in
+                              public_room_ids else "private"),
             }
 
             if event.membership == Membership.INVITE:
@@ -296,7 +301,7 @@ class MessageHandler(BaseHandler):
                 end_token = now_token.copy_and_replace("room_key", token[1])
 
                 d["messages"] = {
-                    "chunk": [m.get_dict() for m in messages],
+                    "chunk": [self.hs.serialize_event(m) for m in messages],
                     "start": start_token.to_string(),
                     "end": end_token.to_string(),
                 }
@@ -304,7 +309,7 @@ class MessageHandler(BaseHandler):
                 current_state = yield self.store.get_current_state(
                     event.room_id
                 )
-                d["state"] = [c.get_dict() for c in current_state]
+                d["state"] = [self.hs.serialize_event(c) for c in current_state]
             except:
                 logger.exception("Failed to get snapshot")
 
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index 023d8c0cf2..dab9b03f04 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -15,9 +15,9 @@
 
 from twisted.internet import defer
 
-from synapse.api.errors import SynapseError, AuthError
-
-from synapse.api.errors import CodeMessageException
+from synapse.api.errors import SynapseError, AuthError, CodeMessageException
+from synapse.api.constants import Membership
+from synapse.api.events.room import RoomMemberEvent
 
 from ._base import BaseHandler
 
@@ -97,6 +97,8 @@ class ProfileHandler(BaseHandler):
             }
         )
 
+        yield self._update_join_states(target_user)
+
     @defer.inlineCallbacks
     def get_avatar_url(self, target_user):
         if target_user.is_mine:
@@ -144,6 +146,8 @@ class ProfileHandler(BaseHandler):
             }
         )
 
+        yield self._update_join_states(target_user)
+
     @defer.inlineCallbacks
     def collect_presencelike_data(self, user, state):
         if not user.is_mine:
@@ -180,3 +184,39 @@ class ProfileHandler(BaseHandler):
             )
 
         defer.returnValue(response)
+
+    @defer.inlineCallbacks
+    def _update_join_states(self, user):
+        if not user.is_mine:
+            return
+
+        joins = yield self.store.get_rooms_for_user_where_membership_is(
+            user.to_string(),
+            [Membership.JOIN],
+        )
+
+        for j in joins:
+            snapshot = yield self.store.snapshot_room(
+                j.room_id, j.state_key, RoomMemberEvent.TYPE,
+                j.state_key
+            )
+
+            content = {
+                "membership": j.content["membership"],
+                "prev": j.content["membership"],
+            }
+
+            yield self.distributor.fire(
+                "collect_presencelike_data", user, content
+            )
+
+            new_event = self.event_factory.create_event(
+                etype=j.type,
+                room_id=j.room_id,
+                state_key=j.state_key,
+                content=content,
+                user_id=j.state_key,
+            )
+
+            yield self.state_handler.handle_new_event(new_event, snapshot)
+            yield self._on_new_room_event(new_event, snapshot)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 0b841d6d3a..a019d770d4 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -40,8 +40,7 @@ class RegistrationHandler(BaseHandler):
         self.distributor.declare("registered_user")
 
     @defer.inlineCallbacks
-    def register(self, localpart=None, password=None, threepidCreds=None, 
-                 captcha_info={}):
+    def register(self, localpart=None, password=None):
         """Registers a new client on the server.
 
         Args:
@@ -54,37 +53,6 @@ class RegistrationHandler(BaseHandler):
         Raises:
             RegistrationError if there was a problem registering.
         """
-        if captcha_info:
-            captcha_response = yield self._validate_captcha(
-                captcha_info["ip"], 
-                captcha_info["private_key"],
-                captcha_info["challenge"],
-                captcha_info["response"]
-            )
-            if not captcha_response["valid"]:
-                logger.info("Invalid captcha entered from %s. Error: %s", 
-                            captcha_info["ip"], captcha_response["error_url"])
-                raise InvalidCaptchaError(
-                    error_url=captcha_response["error_url"]
-                )
-            else:
-                logger.info("Valid captcha entered from %s", captcha_info["ip"])
-
-        if threepidCreds:
-            for c in threepidCreds:
-                logger.info("validating theeepidcred sid %s on id server %s",
-                            c['sid'], c['idServer'])
-                try:
-                    threepid = yield self._threepid_from_creds(c)
-                except:
-                    logger.err()
-                    raise RegistrationError(400, "Couldn't validate 3pid")
-                    
-                if not threepid:
-                    raise RegistrationError(400, "Couldn't validate 3pid")
-                logger.info("got threepid medium %s address %s", 
-                            threepid['medium'], threepid['address'])
-
         password_hash = None
         if password:
             password_hash = bcrypt.hashpw(password, bcrypt.gensalt())
@@ -126,15 +94,54 @@ class RegistrationHandler(BaseHandler):
                         raise RegistrationError(
                             500, "Cannot generate user ID.")
 
-        # Now we have a matrix ID, bind it to the threepids we were given
-        if threepidCreds:
-            for c in threepidCreds:
-                # XXX: This should be a deferred list, shouldn't it?
-                yield self._bind_threepid(c, user_id)
-                
-
         defer.returnValue((user_id, token))
 
+    @defer.inlineCallbacks
+    def check_recaptcha(self, ip, private_key, challenge, response):
+        """Checks a recaptcha is correct."""
+
+        captcha_response = yield self._validate_captcha(
+            ip,
+            private_key,
+            challenge,
+            response
+        )
+        if not captcha_response["valid"]:
+            logger.info("Invalid captcha entered from %s. Error: %s",
+                        ip, captcha_response["error_url"])
+            raise InvalidCaptchaError(
+                error_url=captcha_response["error_url"]
+            )
+        else:
+            logger.info("Valid captcha entered from %s", ip)
+
+    @defer.inlineCallbacks
+    def register_email(self, threepidCreds):
+        """Registers emails with an identity server."""
+
+        for c in threepidCreds:
+            logger.info("validating theeepidcred sid %s on id server %s",
+                        c['sid'], c['idServer'])
+            try:
+                threepid = yield self._threepid_from_creds(c)
+            except:
+                logger.err()
+                raise RegistrationError(400, "Couldn't validate 3pid")
+
+            if not threepid:
+                raise RegistrationError(400, "Couldn't validate 3pid")
+            logger.info("got threepid medium %s address %s",
+                        threepid['medium'], threepid['address'])
+
+    @defer.inlineCallbacks
+    def bind_emails(self, user_id, threepidCreds):
+        """Links emails with a user ID and informs an identity server."""
+
+        # Now we have a matrix ID, bind it to the threepids we were given
+        for c in threepidCreds:
+            # XXX: This should be a deferred list, shouldn't it?
+            yield self._bind_threepid(c, user_id)
+
     def _generate_token(self, user_id):
         # urlsafe variant uses _ and - so use . as the separator and replace
         # all =s with .s so http clients don't quote =s when it is used as
@@ -149,17 +156,17 @@ class RegistrationHandler(BaseHandler):
     def _threepid_from_creds(self, creds):
         httpCli = PlainHttpClient(self.hs)
         # XXX: make this configurable!
-        trustedIdServers = [ 'matrix.org:8090' ]
+        trustedIdServers = ['matrix.org:8090']
         if not creds['idServer'] in trustedIdServers:
-            logger.warn('%s is not a trusted ID server: rejecting 3pid '+
+            logger.warn('%s is not a trusted ID server: rejecting 3pid ' +
                         'credentials', creds['idServer'])
             defer.returnValue(None)
         data = yield httpCli.get_json(
             creds['idServer'],
             "/_matrix/identity/api/v1/3pid/getValidated3pid",
-            { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'] }
+            {'sid': creds['sid'], 'clientSecret': creds['clientSecret']}
         )
-        
+
         if 'medium' in data:
             defer.returnValue(data)
         defer.returnValue(None)
@@ -170,44 +177,45 @@ class RegistrationHandler(BaseHandler):
         data = yield httpCli.post_urlencoded_get_json(
             creds['idServer'],
             "/_matrix/identity/api/v1/3pid/bind",
-            { 'sid': creds['sid'], 'clientSecret': creds['clientSecret'], 
-            'mxid':mxid }
+            {'sid': creds['sid'], 'clientSecret': creds['clientSecret'],
+            'mxid': mxid}
         )
         defer.returnValue(data)
-        
+
     @defer.inlineCallbacks
     def _validate_captcha(self, ip_addr, private_key, challenge, response):
         """Validates the captcha provided.
-        
+
         Returns:
             dict: Containing 'valid'(bool) and 'error_url'(str) if invalid.
-        
+
         """
-        response = yield self._submit_captcha(ip_addr, private_key, challenge, 
+        response = yield self._submit_captcha(ip_addr, private_key, challenge,
                                               response)
         # parse Google's response. Lovely format..
         lines = response.split('\n')
         json = {
             "valid": lines[0] == 'true',
-            "error_url": "http://www.google.com/recaptcha/api/challenge?"+
+            "error_url": "http://www.google.com/recaptcha/api/challenge?" +
                          "error=%s" % lines[1]
         }
         defer.returnValue(json)
-        
+
     @defer.inlineCallbacks
     def _submit_captcha(self, ip_addr, private_key, challenge, response):
         client = PlainHttpClient(self.hs)
         data = yield client.post_urlencoded_get_raw(
             "www.google.com:80",
             "/recaptcha/api/verify",
-            accept_partial=True,  # twisted dislikes google's response, no content length.
-            args={ 
-                'privatekey': private_key, 
+            # twisted dislikes google's response, no content length.
+            accept_partial=True,
+            args={
+                'privatekey': private_key,
                 'remoteip': ip_addr,
                 'challenge': challenge,
                 'response': response
             }
         )
         defer.returnValue(data)
-        
+
 
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 310cb46fe7..5bc1280432 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -335,7 +335,7 @@ class RoomMemberHandler(BaseHandler):
 
         member_list = yield self.store.get_room_members(room_id=room_id)
         event_list = [
-            entry.get_dict()
+            self.hs.serialize_event(entry)
             for entry in member_list
         ]
         chunk_data = {
diff --git a/synapse/http/client.py b/synapse/http/client.py
index ece6318e00..eb11bfd4d5 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -122,7 +122,7 @@ class TwistedHttpClient(HttpClient):
         self.hs = hs
 
     @defer.inlineCallbacks
-    def put_json(self, destination, path, data):
+    def put_json(self, destination, path, data, on_send_callback=None):
         if destination in _destination_mappings:
             destination = _destination_mappings[destination]
 
@@ -131,7 +131,8 @@ class TwistedHttpClient(HttpClient):
             "PUT",
             path.encode("ascii"),
             producer=_JsonProducer(data),
-            headers_dict={"Content-Type": ["application/json"]}
+            headers_dict={"Content-Type": ["application/json"]},
+            on_send_callback=on_send_callback,
         )
 
         logger.debug("Getting resp body")
@@ -218,7 +219,7 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes, param_bytes=b"",
                         query_bytes=b"", producer=None, headers_dict={},
-                        retry_on_dns_fail=True):
+                        retry_on_dns_fail=True, on_send_callback=None):
         """ Creates and sends a request to the given url
         """
         headers_dict[b"User-Agent"] = [b"Synapse"]
@@ -242,6 +243,9 @@ class TwistedHttpClient(HttpClient):
         endpoint = self._getEndpoint(reactor, destination);
 
         while True:
+            if on_send_callback:
+                on_send_callback(destination, method, path_bytes, producer)
+
             try:
                 response = yield self.agent.request(
                     destination,
@@ -310,6 +314,9 @@ class _JsonProducer(object):
     """ Used by the twisted http client to create the HTTP body from json
     """
     def __init__(self, jsn):
+        self.reset(jsn)
+
+    def reset(self, jsn):
         self.body = encode_canonical_json(jsn)
         self.length = len(self.body)
 
diff --git a/synapse/rest/events.py b/synapse/rest/events.py
index 7fde143200..097195d7cc 100644
--- a/synapse/rest/events.py
+++ b/synapse/rest/events.py
@@ -59,7 +59,7 @@ class EventRestServlet(RestServlet):
         event = yield handler.get_event(auth_user, event_id)
 
         if event:
-            defer.returnValue((200, event.get_dict()))
+            defer.returnValue((200, self.hs.serialize_event(event)))
         else:
             defer.returnValue((404, "Event not found."))
 
diff --git a/synapse/rest/login.py b/synapse/rest/login.py
index ba49afcaa7..ad71f6c61d 100644
--- a/synapse/rest/login.py
+++ b/synapse/rest/login.py
@@ -73,6 +73,27 @@ class LoginFallbackRestServlet(RestServlet):
         return (200, {})
 
 
+class PasswordResetRestServlet(RestServlet):
+    PATTERN = client_path_pattern("/login/reset")
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        reset_info = _parse_json(request)
+        try:
+            email = reset_info["email"]
+            user_id = reset_info["user_id"]
+            handler = self.handlers.login_handler
+            yield handler.reset_password(user_id, email)
+            # purposefully give no feedback to avoid people hammering different
+            # combinations.
+            defer.returnValue((200, {}))
+        except KeyError:
+            raise SynapseError(
+                400,
+                "Missing keys. Requires 'email' and 'user_id'."
+            )
+
+
 def _parse_json(request):
     try:
         content = json.loads(request.content.read())
@@ -85,3 +106,4 @@ def _parse_json(request):
 
 def register_servlets(hs, http_server):
     LoginRestServlet(hs).register(http_server)
+    # TODO PasswordResetRestServlet(hs).register(http_server)
diff --git a/synapse/rest/register.py b/synapse/rest/register.py
index 48d3c6eca0..af528a44f6 100644
--- a/synapse/rest/register.py
+++ b/synapse/rest/register.py
@@ -17,89 +17,218 @@
 from twisted.internet import defer
 
 from synapse.api.errors import SynapseError, Codes
+from synapse.api.constants import LoginType
 from base import RestServlet, client_path_pattern
+import synapse.util.stringutils as stringutils
 
 import json
+import logging
 import urllib
 
+logger = logging.getLogger(__name__)
+
 
 class RegisterRestServlet(RestServlet):
+    """Handles registration with the home server.
+
+    This servlet is in control of the registration flow; the registration
+    handler doesn't have a concept of multi-stages or sessions.
+    """
+
     PATTERN = client_path_pattern("/register$")
 
+    def __init__(self, hs):
+        super(RegisterRestServlet, self).__init__(hs)
+        # sessions are stored as:
+        # self.sessions = {
+        #   "session_id" : { __session_dict__ }
+        # }
+        # TODO: persistent storage
+        self.sessions = {}
+
+    def on_GET(self, request):
+        if self.hs.config.enable_registration_captcha:
+            return (200, {
+                "flows": [
+                    {
+                        "type": LoginType.RECAPTCHA,
+                        "stages": ([LoginType.RECAPTCHA,
+                                    LoginType.EMAIL_IDENTITY,
+                                    LoginType.PASSWORD])
+                    },
+                    {
+                        "type": LoginType.RECAPTCHA,
+                        "stages": [LoginType.RECAPTCHA, LoginType.PASSWORD]
+                    }
+                ]
+            })
+        else:
+            return (200, {
+                "flows": [
+                    {
+                        "type": LoginType.EMAIL_IDENTITY,
+                        "stages": ([LoginType.EMAIL_IDENTITY,
+                                    LoginType.PASSWORD])
+                    },
+                    {
+                        "type": LoginType.PASSWORD
+                    }
+                ]
+            })
+
     @defer.inlineCallbacks
     def on_POST(self, request):
-        desired_user_id = None
-        password = None
+        register_json = _parse_json(request)
+
+        session = (register_json["session"] if "session" in register_json
+                  else None)
+        login_type = None
+        if "type" not in register_json:
+            raise SynapseError(400, "Missing 'type' key.")
+
         try:
-            register_json = json.loads(request.content.read())
-            if "password" in register_json:
-                password = register_json["password"].encode("utf-8")
-
-            if type(register_json["user_id"]) == unicode:
-                desired_user_id = register_json["user_id"].encode("utf-8")
-                if urllib.quote(desired_user_id) != desired_user_id:
-                    raise SynapseError(
-                        400,
-                        "User ID must only contain characters which do not " +
-                        "require URL encoding.")
-        except ValueError:
-            defer.returnValue((400, "No JSON object."))
-        except KeyError:
-            pass  # user_id is optional
+            login_type = register_json["type"]
+            stages = {
+                LoginType.RECAPTCHA: self._do_recaptcha,
+                LoginType.PASSWORD: self._do_password,
+                LoginType.EMAIL_IDENTITY: self._do_email_identity
+            }
 
-        threepidCreds = None
-        if 'threepidCreds' in register_json:
-            threepidCreds = register_json['threepidCreds']
-            
-        captcha = {}
-        if self.hs.config.enable_registration_captcha:
-            challenge = None
-            user_response = None
-            try:
-                captcha_type = register_json["captcha"]["type"]
-                if captcha_type != "m.login.recaptcha":
-                    raise SynapseError(400, "Sorry, only m.login.recaptcha " +
-                                       "requests are supported.")
-                challenge = register_json["captcha"]["challenge"]
-                user_response = register_json["captcha"]["response"]
-            except KeyError:
-                raise SynapseError(400, "Captcha response is required",
-                                   errcode=Codes.CAPTCHA_NEEDED)
-            
-            # TODO determine the source IP : May be an X-Forwarding-For header depending on config
-            ip_addr = request.getClientIP()
-            if self.hs.config.captcha_ip_origin_is_x_forwarded:
-                # use the header
-                if request.requestHeaders.hasHeader("X-Forwarded-For"):
-                    ip_addr = request.requestHeaders.getRawHeaders(
-                        "X-Forwarded-For")[0]
-            
-            captcha = {
-                "ip": ip_addr,
-                "private_key": self.hs.config.recaptcha_private_key,
-                "challenge": challenge,
-                "response": user_response
+            session_info = self._get_session_info(request, session)
+            logger.debug("%s : session info %s   request info %s",
+                         login_type, session_info, register_json)
+            response = yield stages[login_type](
+                request,
+                register_json,
+                session_info
+            )
+
+            if "access_token" not in response:
+                # isn't a final response
+                response["session"] = session_info["id"]
+
+            defer.returnValue((200, response))
+        except KeyError as e:
+            logger.exception(e)
+            raise SynapseError(400, "Missing JSON keys for login type %s." % login_type)
+
+    def on_OPTIONS(self, request):
+        return (200, {})
+
+    def _get_session_info(self, request, session_id):
+        if not session_id:
+            # create a new session
+            while session_id is None or session_id in self.sessions:
+                session_id = stringutils.random_string(24)
+            self.sessions[session_id] = {
+                "id": session_id,
+                LoginType.EMAIL_IDENTITY: False,
+                LoginType.RECAPTCHA: False
             }
-            
 
+        return self.sessions[session_id]
+
+    def _save_session(self, session):
+        # TODO: Persistent storage
+        logger.debug("Saving session %s", session)
+        self.sessions[session["id"]] = session
+
+    def _remove_session(self, session):
+        logger.debug("Removing session %s", session)
+        self.sessions.pop(session["id"])
+
+    @defer.inlineCallbacks
+    def _do_recaptcha(self, request, register_json, session):
+        if not self.hs.config.enable_registration_captcha:
+            raise SynapseError(400, "Captcha not required.")
+
+        challenge = None
+        user_response = None
+        try:
+            challenge = register_json["challenge"]
+            user_response = register_json["response"]
+        except KeyError:
+            raise SynapseError(400, "Captcha response is required",
+                               errcode=Codes.CAPTCHA_NEEDED)
+
+        # May be an X-Forwarding-For header depending on config
+        ip_addr = request.getClientIP()
+        if self.hs.config.captcha_ip_origin_is_x_forwarded:
+            # use the header
+            if request.requestHeaders.hasHeader("X-Forwarded-For"):
+                ip_addr = request.requestHeaders.getRawHeaders(
+                    "X-Forwarded-For")[0]
+
+        handler = self.handlers.registration_handler
+        yield handler.check_recaptcha(
+            ip_addr,
+            self.hs.config.recaptcha_private_key,
+            challenge,
+            user_response
+        )
+        session[LoginType.RECAPTCHA] = True  # mark captcha as done
+        self._save_session(session)
+        defer.returnValue({
+            "next": [LoginType.PASSWORD, LoginType.EMAIL_IDENTITY]
+        })
+
+    @defer.inlineCallbacks
+    def _do_email_identity(self, request, register_json, session):
+        if (self.hs.config.enable_registration_captcha and
+                not session[LoginType.RECAPTCHA]):
+            raise SynapseError(400, "Captcha is required.")
+
+        threepidCreds = register_json['threepidCreds']
+        handler = self.handlers.registration_handler
+        yield handler.register_email(threepidCreds)
+        session["threepidCreds"] = threepidCreds  # store creds for next stage
+        session[LoginType.EMAIL_IDENTITY] = True  # mark email as done
+        self._save_session(session)
+        defer.returnValue({
+            "next": LoginType.PASSWORD
+        })
+
+    @defer.inlineCallbacks
+    def _do_password(self, request, register_json, session):
+        if (self.hs.config.enable_registration_captcha and
+                not session[LoginType.RECAPTCHA]):
+            # captcha should've been done by this stage!
+            raise SynapseError(400, "Captcha is required.")
+
+        password = register_json["password"].encode("utf-8")
+        desired_user_id = (register_json["user"].encode("utf-8") if "user"
+                          in register_json else None)
+        if desired_user_id and urllib.quote(desired_user_id) != desired_user_id:
+            raise SynapseError(
+                400,
+                "User ID must only contain characters which do not " +
+                "require URL encoding.")
         handler = self.handlers.registration_handler
         (user_id, token) = yield handler.register(
             localpart=desired_user_id,
-            password=password,
-            threepidCreds=threepidCreds,
-            captcha_info=captcha)
+            password=password
+        )
+
+        if session[LoginType.EMAIL_IDENTITY]:
+            yield handler.bind_emails(user_id, session["threepidCreds"])
 
         result = {
             "user_id": user_id,
             "access_token": token,
             "home_server": self.hs.hostname,
         }
-        defer.returnValue(
-            (200, result)
-        )
+        self._remove_session(session)
+        defer.returnValue(result)
 
-    def on_OPTIONS(self, request):
-        return (200, {})
+
+def _parse_json(request):
+    try:
+        content = json.loads(request.content.read())
+        if type(content) != dict:
+            raise SynapseError(400, "Content must be a JSON object.")
+        return content
+    except ValueError:
+        raise SynapseError(400, "Content not JSON.")
 
 
 def register_servlets(hs, http_server):
diff --git a/synapse/rest/room.py b/synapse/rest/room.py
index cef700c81c..ecb1e346d9 100644
--- a/synapse/rest/room.py
+++ b/synapse/rest/room.py
@@ -378,7 +378,7 @@ class RoomTriggerBackfill(RestServlet):
         handler = self.handlers.federation_handler
         events = yield handler.backfill(remote_server, room_id, limit)
 
-        res = [event.get_dict() for event in events]
+        res = [self.hs.serialize_event(event) for event in events]
         defer.returnValue((200, res))
 
 
diff --git a/synapse/server.py b/synapse/server.py
index 83368ea5a7..7c185537aa 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -20,6 +20,7 @@
 
 # Imports required for the default HomeServer() implementation
 from synapse.federation import initialize_http_replication
+from synapse.api.events import serialize_event
 from synapse.api.events.factory import EventFactory
 from synapse.notifier import Notifier
 from synapse.api.auth import Auth
@@ -138,6 +139,9 @@ class BaseHomeServer(object):
         object."""
         return RoomID.from_string(s, hs=self)
 
+    def serialize_event(self, e):
+        return serialize_event(self, e)
+
 # Build magic accessors for every dependency
 for depname in BaseHomeServer.DEPENDENCIES:
     BaseHomeServer._make_dependency_method(depname)
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 9201a377b6..1cede2809d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -68,7 +68,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
     @defer.inlineCallbacks
     @log_function
-    def persist_event(self, event=None, backfilled=False, pdu=None):
+    def persist_event(self, event=None, backfilled=False, pdu=None,
+                      is_new_state=True):
         stream_ordering = None
         if backfilled:
             if not self.min_token_deferred.called:
@@ -83,6 +84,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 event=event,
                 backfilled=backfilled,
                 stream_ordering=stream_ordering,
+                is_new_state=is_new_state,
             )
         except _RollbackButIsFineException as e:
             pass
@@ -109,12 +111,14 @@ class DataStore(RoomMemberStore, RoomStore,
         defer.returnValue(event)
 
     def _persist_pdu_event_txn(self, txn, pdu=None, event=None,
-                               backfilled=False, stream_ordering=None):
+                               backfilled=False, stream_ordering=None,
+                               is_new_state=True):
         if pdu is not None:
             self._persist_event_pdu_txn(txn, pdu)
         if event is not None:
             return self._persist_event_txn(
-                txn, event, backfilled, stream_ordering
+                txn, event, backfilled, stream_ordering,
+                is_new_state=is_new_state,
             )
 
     def _persist_event_pdu_txn(self, txn, pdu):
@@ -141,7 +145,8 @@ class DataStore(RoomMemberStore, RoomStore,
         self._update_min_depth_for_context_txn(txn, pdu.context, pdu.depth)
 
     @log_function
-    def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None):
+    def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None,
+                           is_new_state=True):
         if event.type == RoomMemberEvent.TYPE:
             self._store_room_member_txn(txn, event)
         elif event.type == FeedbackEvent.TYPE:
@@ -195,7 +200,7 @@ class DataStore(RoomMemberStore, RoomStore,
             )
             raise _RollbackButIsFineException("_persist_event")
 
-        if not backfilled and hasattr(event, "state_key"):
+        if is_new_state and hasattr(event, "state_key"):
             vals = {
                 "event_id": event.event_id,
                 "room_id": event.room_id,
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 8deaaf93bd..cf88bfc22b 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -315,6 +315,10 @@ class SQLBaseStore(object):
         d["content"] = json.loads(d["content"])
         del d["unrecognized_keys"]
 
+        if "age_ts" not in d:
+            # For compatibility
+            d["age_ts"] = d["ts"] if "ts" in d else 0
+
         return self.event_factory.create_event(
             etype=d["type"],
             **d
diff --git a/synapse/util/emailutils.py b/synapse/util/emailutils.py
new file mode 100644
index 0000000000..cdb0abd7ea
--- /dev/null
+++ b/synapse/util/emailutils.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" This module allows you to send out emails.
+"""
+import email.utils
+import smtplib
+import twisted.python.log
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class EmailException(Exception):
+    pass
+
+
+def send_email(smtp_server, from_addr, to_addr, subject, body):
+    """Sends an email.
+
+    Args:
+        smtp_server(str): The SMTP server to use.
+        from_addr(str): The address to send from.
+        to_addr(str): The address to send to.
+        subject(str): The subject of the email.
+        body(str): The plain text body of the email.
+    Raises:
+        EmailException if there was a problem sending the mail.
+    """
+    if not smtp_server or not from_addr or not to_addr:
+        raise EmailException("Need SMTP server, from and to addresses. Check " +
+                             "the config to set these.")
+
+    msg = MIMEMultipart('alternative')
+    msg['Subject'] = subject
+    msg['From'] = from_addr
+    msg['To'] = to_addr
+    plain_part = MIMEText(body)
+    msg.attach(plain_part)
+
+    raw_from = email.utils.parseaddr(from_addr)[1]
+    raw_to = email.utils.parseaddr(to_addr)[1]
+    if not raw_from or not raw_to:
+        raise EmailException("Couldn't parse from/to address.")
+
+    logger.info("Sending email to %s on server %s with subject %s",
+                to_addr, smtp_server, subject)
+
+    try:
+        smtp = smtplib.SMTP(smtp_server)
+        smtp.sendmail(raw_from, raw_to, msg.as_string())
+        smtp.quit()
+    except Exception as origException:
+        twisted.python.log.err()
+        ese = EmailException()
+        ese.cause = origException
+        raise ese
\ No newline at end of file