From 278149f53392d5e3b08f72106517afe2711c856c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Mar 2015 13:43:21 +0000
Subject: Sanitize TransactionStore

---
 synapse/handlers/federation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 15ba417e06..9a4773ac02 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -179,7 +179,7 @@ class FederationHandler(BaseHandler):
         # it's probably a good idea to mark it as not in retry-state
         # for sending (although this is a bit of a leap)
         retry_timings = yield self.store.get_destination_retry_timings(origin)
-        if (retry_timings and retry_timings.retry_last_ts):
+        if retry_timings and retry_timings["retry_last_ts"]:
             self.store.set_destination_retry_timings(origin, 0, 0)
 
         room = yield self.store.get_room(event.room_id)
-- 
cgit 1.5.1


From d98660a60daaf1cc8d83cb2d64daa5f20a34139c Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 23 Mar 2015 14:20:28 +0000
Subject: Implement password changing (finally) along with a start on making
 client/server auth more general.

---
 synapse/handlers/__init__.py             |   2 +
 synapse/handlers/auth.py                 | 109 +++++++++++++++++++++++++++++++
 synapse/handlers/login.py                |  49 ++------------
 synapse/rest/client/v2_alpha/__init__.py |   4 +-
 synapse/rest/client/v2_alpha/_base.py    |  12 ++++
 synapse/rest/client/v2_alpha/password.py |  76 +++++++++++++++++++++
 synapse/storage/registration.py          |  33 ++++++++--
 7 files changed, 236 insertions(+), 49 deletions(-)
 create mode 100644 synapse/handlers/auth.py
 create mode 100644 synapse/rest/client/v2_alpha/password.py

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 8d345bf936..336ce15701 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -29,6 +29,7 @@ from .typing import TypingNotificationHandler
 from .admin import AdminHandler
 from .appservice import ApplicationServicesHandler
 from .sync import SyncHandler
+from .auth import AuthHandler
 
 
 class Handlers(object):
@@ -58,3 +59,4 @@ class Handlers(object):
             hs, ApplicationServiceApi(hs)
         )
         self.sync_handler = SyncHandler(hs)
+        self.auth_handler = AuthHandler(hs)
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
new file mode 100644
index 0000000000..e4a73da9a7
--- /dev/null
+++ b/synapse/handlers/auth.py
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014, 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from ._base import BaseHandler
+from synapse.api.constants import LoginType
+from synapse.types import UserID
+from synapse.api.errors import LoginError, Codes
+
+import logging
+import bcrypt
+
+
+logger = logging.getLogger(__name__)
+
+
+class AuthHandler(BaseHandler):
+
+    def __init__(self, hs):
+        super(AuthHandler, self).__init__(hs)
+
+    @defer.inlineCallbacks
+    def check_auth(self, flows, clientdict):
+        """
+        Takes a dictionary sent by the client in the login / registration
+        protocol and handles the login flow.
+
+        Args:
+            flows: list of list of stages
+            authdict: The dictionary from the client root level, not the
+                      'auth' key: this method prompts for auth if none is sent.
+        Returns:
+            A tuple of authed, dict where authed is true if the client
+            has successfully completed an auth flow. If it is true, the dict
+            contains the authenticated credentials of each stage.
+            If authed is false, the dictionary is the server response to the
+            login request and should be passed back to the client.
+        """
+        types = {
+            LoginType.PASSWORD: self.check_password_auth
+        }
+
+        if 'auth' not in clientdict:
+            defer.returnValue((False, auth_dict_for_flows(flows)))
+
+        authdict = clientdict['auth']
+
+        # In future: support sessions & retrieve previously succeeded
+        # login types
+        creds = {}
+
+        # check auth type currently being presented
+        if 'type' not in authdict:
+            raise LoginError(400, "", Codes.MISSING_PARAM)
+        if authdict['type'] not in types:
+            raise LoginError(400, "", Codes.UNRECOGNIZED)
+        result = yield types[authdict['type']](authdict)
+        if result:
+            creds[authdict['type']] = result
+
+        for f in flows:
+            if len(set(f) - set(creds.keys())) == 0:
+                logger.info("Auth completed with creds: %r", creds)
+                defer.returnValue((True, creds))
+
+        ret = auth_dict_for_flows(flows)
+        ret['completed'] = creds.keys()
+        defer.returnValue((False, ret))
+
+    @defer.inlineCallbacks
+    def check_password_auth(self, authdict):
+        if "user" not in authdict or "password" not in authdict:
+            raise LoginError(400, "", Codes.MISSING_PARAM)
+
+        user = authdict["user"]
+        password = authdict["password"]
+        if not user.startswith('@'):
+            user = UserID.create(user, self.hs.hostname).to_string()
+
+        user_info = yield self.store.get_user_by_id(user_id=user)
+        if not user_info:
+            logger.warn("Attempted to login as %s but they do not exist", user)
+            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+
+        stored_hash = user_info[0]["password_hash"]
+        if bcrypt.checkpw(password, stored_hash):
+            defer.returnValue(user)
+        else:
+            logger.warn("Failed password login for user %s", user)
+            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+
+
+def auth_dict_for_flows(flows):
+    return {
+        "flows": {"stages": f for f in flows}
+    }
diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 7447800460..19b560d91e 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -69,48 +69,9 @@ class LoginHandler(BaseHandler):
             raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
     @defer.inlineCallbacks
-    def reset_password(self, user_id, email):
-        is_valid = yield self._check_valid_association(user_id, email)
-        logger.info("reset_password user=%s email=%s valid=%s", user_id, email,
-                    is_valid)
-        if is_valid:
-            try:
-                # send an email out
-                emailutils.send_email(
-                    smtp_server=self.hs.config.email_smtp_server,
-                    from_addr=self.hs.config.email_from_address,
-                    to_addr=email,
-                    subject="Password Reset",
-                    body="TODO."
-                )
-            except EmailException as e:
-                logger.exception(e)
+    def set_password(self, user_id, newpassword, token_id=None):
+        password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt())
 
-    @defer.inlineCallbacks
-    def _check_valid_association(self, user_id, email):
-        identity = yield self._query_email(email)
-        if identity and "mxid" in identity:
-            if identity["mxid"] == user_id:
-                defer.returnValue(True)
-                return
-        defer.returnValue(False)
-
-    @defer.inlineCallbacks
-    def _query_email(self, email):
-        http_client = SimpleHttpClient(self.hs)
-        try:
-            data = yield http_client.get_json(
-                # TODO FIXME This should be configurable.
-                # XXX: ID servers need to use HTTPS
-                "http://%s%s" % (
-                    "matrix.org:8090", "/_matrix/identity/api/v1/lookup"
-                ),
-                {
-                    'medium': 'email',
-                    'address': email
-                }
-            )
-            defer.returnValue(data)
-        except CodeMessageException as e:
-            data = json.loads(e.msg)
-            defer.returnValue(data)
+        yield self.store.user_set_password_hash(user_id, password_hash)
+        yield self.store.user_delete_access_tokens_apart_from(user_id, token_id)
+        yield self.store.flush_user(user_id)
diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py
index bca65f2a6a..041f538e20 100644
--- a/synapse/rest/client/v2_alpha/__init__.py
+++ b/synapse/rest/client/v2_alpha/__init__.py
@@ -15,7 +15,8 @@
 
 from . import (
     sync,
-    filter
+    filter,
+    password
 )
 
 from synapse.http.server import JsonResource
@@ -32,3 +33,4 @@ class ClientV2AlphaRestResource(JsonResource):
     def register_servlets(client_resource, hs):
         sync.register_servlets(hs, client_resource)
         filter.register_servlets(hs, client_resource)
+        password.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index 22dc5cb862..c772cc986f 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -17,9 +17,11 @@
 """
 
 from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX
+from synapse.api.errors import SynapseError
 import re
 
 import logging
+import simplejson
 
 
 logger = logging.getLogger(__name__)
@@ -36,3 +38,13 @@ def client_v2_pattern(path_regex):
         SRE_Pattern
     """
     return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex)
+
+
+def parse_json_dict_from_request(request):
+    try:
+        content = simplejson.loads(request.content.read())
+        if type(content) != dict:
+            raise SynapseError(400, "Content must be a JSON object.")
+        return content
+    except simplejson.JSONDecodeError:
+        raise SynapseError(400, "Content not JSON.")
diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py
new file mode 100644
index 0000000000..3663781c95
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/password.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.constants import LoginType
+from synapse.api.errors import LoginError, SynapseError, Codes
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_pattern, parse_json_dict_from_request
+
+import simplejson as json
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class PasswordRestServlet(RestServlet):
+    PATTERN = client_v2_pattern("/account/password")
+
+    def __init__(self, hs):
+        super(PasswordRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.auth_handler = hs.get_handlers().auth_handler
+        self.login_handler = hs.get_handlers().login_handler
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        body = parse_json_dict_from_request(request)
+
+        authed, result = yield self.auth_handler.check_auth([
+            [LoginType.PASSWORD]
+        ], body)
+
+        if not authed:
+            defer.returnValue((401, result))
+
+        auth_user = None
+
+        if LoginType.PASSWORD in result:
+            # if using password, they should also be logged in
+            auth_user, client = yield self.auth.get_user_by_req(request)
+            if auth_user.to_string() != result[LoginType.PASSWORD]:
+                raise LoginError(400, "", Codes.UNKNOWN)
+        else:
+            logger.error("Auth succeeded but no known type!", result.keys())
+            raise SynapseError(500, "", Codes.UNKNOWN)
+
+        user_id = auth_user.to_string()
+
+        if 'new_password' not in body:
+            raise SynapseError(400, "", Codes.MISSING_PARAM)
+        new_password = body['new_password']
+
+        self.login_handler.set_password(
+            user_id, new_password, client.token_id
+        )
+
+        defer.returnValue((200, {}))
+
+def register_servlets(hs, http_server):
+    PasswordRestServlet(hs).register(http_server)
\ No newline at end of file
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index f24154f146..7e60dc3951 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -95,11 +95,36 @@ class RegistrationStore(SQLBaseStore):
             "get_user_by_id", self.cursor_to_dict, query, user_id
         )
 
+    def user_set_password_hash(self, user_id, password_hash):
+        """
+        NB. This does *not* evict any cache because the one use for this
+            removes most of the entries subsequently anyway so it would be
+            pointless. Use flush_user separately.
+        """
+        return self._simple_update_one('users', {
+            'name': user_id
+        }, {
+            'password_hash': password_hash
+        })
+
+    def user_delete_access_tokens_apart_from(self, user_id, token_id):
+        return self._execute(
+            "delete_access_tokens_apart_from", None,
+            "DELETE FROM access_tokens WHERE user_id = ? AND id != ?",
+            user_id, token_id
+        )
+
+    @defer.inlineCallbacks
+    def flush_user(self, user_id):
+        rows = yield self._execute(
+            'user_delete_access_tokens_apart_from', None,
+            "SELECT token FROM access_tokens WHERE user_id = ?",
+            user_id
+        )
+        for r in rows:
+            self.get_user_by_token.invalidate(r)
+
     @cached()
-    # TODO(paul): Currently there's no code to invalidate this cache. That
-    #   means if/when we ever add internal ways to invalidate access tokens or
-    #   change whether a user is a server admin, those will need to invoke
-    #      store.get_user_by_token.invalidate(token)
     def get_user_by_token(self, token):
         """Get a user from the given access token.
 
-- 
cgit 1.5.1


From 78adccfaf497dcb75451adfc5d366d5ff26cad52 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 23 Mar 2015 14:23:51 +0000
Subject: pep8 / pyflakes

---
 synapse/handlers/login.py                | 6 +-----
 synapse/rest/client/v2_alpha/password.py | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 19b560d91e..7aff2e69e6 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -16,13 +16,9 @@
 from twisted.internet import defer
 
 from ._base import BaseHandler
-from synapse.api.errors import LoginError, Codes, CodeMessageException
-from synapse.http.client import SimpleHttpClient
-from synapse.util.emailutils import EmailException
-import synapse.util.emailutils as emailutils
+from synapse.api.errors import LoginError, Codes
 
 import bcrypt
-import json
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py
index 3663781c95..1277532110 100644
--- a/synapse/rest/client/v2_alpha/password.py
+++ b/synapse/rest/client/v2_alpha/password.py
@@ -21,7 +21,6 @@ from synapse.http.servlet import RestServlet
 
 from ._base import client_v2_pattern, parse_json_dict_from_request
 
-import simplejson as json
 import logging
 
 
@@ -72,5 +71,6 @@ class PasswordRestServlet(RestServlet):
 
         defer.returnValue((200, {}))
 
+
 def register_servlets(hs, http_server):
-    PasswordRestServlet(hs).register(http_server)
\ No newline at end of file
+    PasswordRestServlet(hs).register(http_server)
-- 
cgit 1.5.1


From 9a7f4962982e309877acb777eb358178579cacc9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 23 Mar 2015 15:29:04 +0000
Subject: Sanitize RoomMemberStore

---
 synapse/handlers/room.py      |  19 -------
 synapse/storage/roommember.py | 124 +++++++++++++++++++++---------------------
 2 files changed, 61 insertions(+), 82 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 823affc380..bc7f1c2402 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -310,25 +310,6 @@ class RoomMemberHandler(BaseHandler):
         # paginating
         defer.returnValue(chunk_data)
 
-    @defer.inlineCallbacks
-    def get_room_member(self, room_id, member_user_id, auth_user_id):
-        """Retrieve a room member from a room.
-
-        Args:
-            room_id : The room the member is in.
-            member_user_id : The member's user ID
-            auth_user_id : The user ID of the user making this request.
-        Returns:
-            The room member, or None if this member does not exist.
-        Raises:
-            SynapseError if something goes wrong.
-        """
-        yield self.auth.check_joined_room(room_id, auth_user_id)
-
-        member = yield self.store.get_room_member(user_id=member_user_id,
-                                                  room_id=room_id)
-        defer.returnValue(member)
-
     @defer.inlineCallbacks
     def change_membership(self, event, context, do_auth=True):
         """ Change the membership status of a user in a room.
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 17ee4bb9ec..a229505208 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -74,23 +74,10 @@ class RoomMemberStore(SQLBaseStore):
             txn.execute(sql, (event.room_id, domain))
         elif event.membership != Membership.INVITE:
             # Check if this was the last person to have left.
-            member_events = self._get_members_query_txn(
-                txn,
-                where_clause=("c.room_id = ? AND m.membership = ?"
-                              " AND m.user_id != ?"),
-                where_values=(event.room_id, Membership.JOIN, target_user_id,)
+            joined_domains = self._get_joined_hosts_for_room_txn(
+                txn, event.room_id
             )
 
-            joined_domains = set()
-            for e in member_events:
-                try:
-                    joined_domains.add(
-                        UserID.from_string(e.state_key).domain
-                    )
-                except:
-                    # FIXME: How do we deal with invalid user ids in the db?
-                    logger.exception("Invalid user_id: %s", event.state_key)
-
             if domain not in joined_domains:
                 sql = (
                     "DELETE FROM room_hosts WHERE room_id = ? AND host = ?"
@@ -100,7 +87,6 @@ class RoomMemberStore(SQLBaseStore):
 
         self.get_rooms_for_user.invalidate(target_user_id)
 
-    @defer.inlineCallbacks
     def get_room_member(self, user_id, room_id):
         """Retrieve the current state of a room member.
 
@@ -110,41 +96,27 @@ class RoomMemberStore(SQLBaseStore):
         Returns:
             Deferred: Results in a MembershipEvent or None.
         """
-        rows = yield self._get_members_by_dict({
-            "e.room_id": room_id,
-            "m.user_id": user_id,
-        })
+        def f(txn):
+            events = self._get_members_events_txn(
+                txn,
+                room_id,
+                user_id=user_id,
+            )
 
-        defer.returnValue(rows[0] if rows else None)
+            return events[0] if events else None
 
-    def _get_room_member(self, txn, user_id, room_id):
-        sql = (
-            "SELECT e.* FROM events as e"
-            " INNER JOIN room_memberships as m"
-            " ON e.event_id = m.event_id"
-            " INNER JOIN current_state_events as c"
-            " ON m.event_id = c.event_id"
-            " WHERE m.user_id = ? and e.room_id = ?"
-            " LIMIT 1"
-        )
-        txn.execute(sql, (user_id, room_id))
-        rows = self.cursor_to_dict(txn)
-        if rows:
-            return self._parse_events_txn(txn, rows)[0]
-        else:
-            return None
+        return self.runInteraction("get_room_member", f)
 
     def get_users_in_room(self, room_id):
         def f(txn):
-            sql = (
-                "SELECT m.user_id FROM room_memberships as m"
-                " INNER JOIN current_state_events as c"
-                " ON m.event_id = c.event_id"
-                " WHERE m.membership = ? AND m.room_id = ?"
+
+            rows = self._get_members_rows_txn(
+                txn,
+                room_id=room_id,
+                membership=Membership.JOIN,
             )
 
-            txn.execute(sql, (Membership.JOIN, room_id))
-            return [r[0] for r in txn.fetchall()]
+            return [r["user_id"] for r in rows]
         return self.runInteraction("get_users_in_room", f)
 
     def get_room_members(self, room_id, membership=None):
@@ -159,11 +131,14 @@ class RoomMemberStore(SQLBaseStore):
             list of namedtuples representing the members in this room.
         """
 
-        where = {"m.room_id": room_id}
-        if membership:
-            where["m.membership"] = membership
+        def f(txn):
+            return self._get_members_events_txn(
+                txn,
+                room_id,
+                membership=membership,
+            )
 
-        return self._get_members_by_dict(where)
+        return self.runInteraction("get_room_members", f)
 
     def get_rooms_for_user_where_membership_is(self, user_id, membership_list):
         """ Get all the rooms for this user where the membership for this user
@@ -209,28 +184,52 @@ class RoomMemberStore(SQLBaseStore):
         ]
 
     def get_joined_hosts_for_room(self, room_id):
-        return self._simple_select_onecol(
-            "room_hosts",
-            {"room_id": room_id},
-            "host",
-            desc="get_joined_hosts_for_room",
+        return self.runInteraction(
+            "get_joined_hosts_for_room",
+            self._get_joined_hosts_for_room_txn,
+            room_id,
+        )
+
+    def _get_joined_hosts_for_room_txn(self, txn, room_id):
+        rows = self._get_members_rows_txn(
+            txn,
+            room_id, membership=Membership.JOIN
+        )
+
+        joined_domains = set(
+            UserID.from_string(r["user_id"]).domain
+            for r in rows
         )
 
-    def _get_members_by_dict(self, where_dict):
-        clause = " AND ".join("%s = ?" % k for k in where_dict.keys())
-        vals = where_dict.values()
-        return self._get_members_query(clause, vals)
+        return joined_domains
 
     def _get_members_query(self, where_clause, where_values):
         return self.runInteraction(
-            "get_members_query", self._get_members_query_txn,
+            "get_members_query", self._get_members_events_txn,
             where_clause, where_values
         )
 
-    def _get_members_query_txn(self, txn, where_clause, where_values):
+    def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None):
+        rows = self._get_members_rows_txn(
+            txn,
+            room_id, membership, user_id,
+        )
+        return self._get_events_txn(txn, [r["event_id"] for r in rows])
+
+    def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None):
+        where_clause = "c.room_id = ?"
+        where_values = [room_id]
+
+        if membership:
+            where_clause += " AND m.membership = ?"
+            where_values.append(membership)
+
+        if user_id:
+            where_clause += " AND m.user_id = ?"
+            where_values.append(user_id)
+
         sql = (
-            "SELECT e.* FROM events as e "
-            "INNER JOIN room_memberships as m "
+            "SELECT m.* FROM room_memberships as m "
             "ON e.event_id = m.event_id "
             "INNER JOIN current_state_events as c "
             "ON m.event_id = c.event_id "
@@ -242,8 +241,7 @@ class RoomMemberStore(SQLBaseStore):
         txn.execute(sql, where_values)
         rows = self.cursor_to_dict(txn)
 
-        results = self._parse_events_txn(txn, rows)
-        return results
+        return rows
 
     @cached()
     def get_rooms_for_user(self, user_id):
-- 
cgit 1.5.1


From 0e8f5095c7e7075b249ad53a9f60a4d2fdeeaaed Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Mar 2015 17:15:20 +0000
Subject: Fix unicode database support

---
 synapse/app/homeserver.py                          | 47 +++++++++++++---------
 synapse/handlers/login.py                          |  2 +-
 synapse/rest/client/v1/profile.py                  |  7 +++-
 synapse/storage/__init__.py                        | 12 ++++--
 synapse/storage/_base.py                           |  4 ++
 synapse/storage/events.py                          |  8 ++--
 synapse/storage/keys.py                            |  4 +-
 synapse/storage/profile.py                         | 12 +++++-
 synapse/storage/registration.py                    | 18 +++++++--
 synapse/storage/room.py                            |  1 +
 .../schema/full_schemas/11/media_repository.sql    |  2 +-
 .../storage/schema/full_schemas/11/profiles.sql    |  2 +-
 .../schema/full_schemas/11/transactions.sql        |  1 -
 synapse/storage/signatures.py                      | 10 ++---
 synapse/storage/transactions.py                    |  2 +-
 15 files changed, 88 insertions(+), 44 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 394e93e6c2..beab6ffc7a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -110,14 +110,12 @@ class SynapseHomeServer(HomeServer):
             return None
 
     def build_db_pool(self):
-        name = self.db_config.pop("name", None)
-        if name == "MySQLdb":
-            return adbapi.ConnectionPool(
-                name,
-                **self.db_config
-            )
+        name = self.db_config["name"]
 
-        raise RuntimeError("Unsupported database type")
+        return adbapi.ConnectionPool(
+            name,
+            **self.db_config.get("args", {})
+        )
 
     def create_resource_tree(self, redirect_root_to_web_client):
         """Create the resource tree for this Home Server.
@@ -323,7 +321,7 @@ def change_resource_limit(soft_file_no):
         resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard))
 
         logger.info("Set file limit to: %d", soft_file_no)
-    except (ValueError, resource.error) as e:
+    except (   ValueError, resource.error) as e:
         logger.warn("Failed to set file limit: %s", e)
 
 
@@ -363,20 +361,33 @@ def setup(config_options):
     if config.database_config:
         with open(config.database_config, 'r') as f:
             db_config = yaml.safe_load(f)
-
-        name = db_config.get("name", None)
-        if name == "MySQLdb":
-            db_config.update({
-                "sql_mode": "TRADITIONAL",
-                "charset": "utf8",
-                "use_unicode": True,
-            })
     else:
         db_config = {
             "name": "sqlite3",
             "database": config.database_path,
         }
 
+    db_config = {
+        k: v for k, v in db_config.items()
+        if not k.startswith("cp_")
+    }
+
+    name = db_config.get("name", None)
+    if name in ["MySQLdb", "mysql.connector"]:
+        db_config.setdefault("args", {}).update({
+            "sql_mode": "TRADITIONAL",
+            "charset": "utf8",
+            "use_unicode": True,
+        })
+    elif name == "sqlite3":
+        db_config.setdefault("args", {}).update({
+            "cp_min": 1,
+            "cp_max": 1,
+            "cp_openfun": prepare_database,
+        })
+    else:
+        raise RuntimeError("Unsupported database type '%s'" % (name,))
+
     hs = SynapseHomeServer(
         config.server_name,
         domain_with_port=domain_with_port,
@@ -401,8 +412,8 @@ def setup(config_options):
         # with sqlite3.connect(db_name) as db_conn:
         #     prepare_sqlite3_database(db_conn)
         #     prepare_database(db_conn)
-        import MySQLdb
-        db_conn = MySQLdb.connect(**db_config)
+        import mysql.connector
+        db_conn = mysql.connector.connect(**db_config.get("args", {}))
         prepare_database(db_conn)
     except UpgradeDatabaseException:
         sys.stderr.write(
diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 7447800460..76647c7941 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -57,7 +57,7 @@ class LoginHandler(BaseHandler):
             logger.warn("Attempted to login as %s but they do not exist", user)
             raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
-        stored_hash = user_info[0]["password_hash"]
+        stored_hash = user_info["password_hash"]
         if bcrypt.checkpw(password, stored_hash):
             # generate an access token and store it.
             token = self.reg_handler._generate_token(user)
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index 1e77eb49cf..7387b4adb9 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -19,9 +19,13 @@ from twisted.internet import defer
 from .base import ClientV1RestServlet, client_path_pattern
 from synapse.types import UserID
 
+import logging
 import simplejson as json
 
 
+logger = logging.getLogger(__name__)
+
+
 class ProfileDisplaynameRestServlet(ClientV1RestServlet):
     PATTERN = client_path_pattern("/profile/(?P<user_id>[^/]*)/displayname")
 
@@ -47,7 +51,8 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
             defer.returnValue((400, "Unable to parse name"))
 
         yield self.handlers.profile_handler.set_displayname(
-            user, auth_user, new_name)
+            user, auth_user, new_name
+        )
 
         defer.returnValue((200, {}))
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index e03d55b00d..abde7d0df5 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -410,10 +410,14 @@ def executescript(txn, schema_path):
 
 
 def _get_or_create_schema_state(txn):
-    schema_path = os.path.join(
-        dir_path, "schema", "schema_version.sql",
-    )
-    executescript(txn, schema_path)
+    try:
+        # Bluntly try creating the schema_version tables.
+        schema_path = os.path.join(
+            dir_path, "schema", "schema_version.sql",
+        )
+        executescript(txn, schema_path)
+    except:
+        pass
 
     txn.execute("SELECT version, upgraded FROM schema_version")
     row = txn.fetchone()
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 1ea39bc0ad..76ec3ee93f 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -755,6 +755,8 @@ class SQLBaseStore(object):
             return None
 
         internal_metadata, js, redacted, rejected_reason = res
+        js = js.decode("utf8")
+        internal_metadata = internal_metadata.decode("utf8")
 
         start_time = update_counter("select_event", start_time)
 
@@ -779,9 +781,11 @@ class SQLBaseStore(object):
             sql_getevents_timer.inc_by(curr_time - last_time, desc)
             return curr_time
 
+        logger.debug("Got js: %r", js)
         d = json.loads(js)
         start_time = update_counter("decode_json", start_time)
 
+        logger.debug("Got internal_metadata: %r", internal_metadata)
         internal_metadata = json.loads(internal_metadata)
         start_time = update_counter("decode_internal", start_time)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 4d636d3f46..69f598967e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -294,15 +294,17 @@ class EventsStore(SQLBaseStore):
             )
 
             if is_new_state and not context.rejected:
-                self._simple_insert_txn(
+                self._simple_upsert_txn(
                     txn,
                     "current_state_events",
-                    {
-                        "event_id": event.event_id,
+                    keyvalues={
                         "room_id": event.room_id,
                         "type": event.type,
                         "state_key": event.state_key,
                     },
+                    values={
+                        "event_id": event.event_id,
+                    }
                 )
 
             for e_id, h in event.prev_state:
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 25fef79434..e6975a945b 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore):
                 "fingerprint": fingerprint,
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
-                "tls_certificate": buffer(tls_certificate_bytes),
+                "tls_certificate": tls_certificate_bytes,
             },
         )
 
@@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore):
                 "key_id": "%s:%s" % (verify_key.alg, verify_key.version),
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
-                "verify_key": buffer(verify_key.encode()),
+                "verify_key": verify_key.encode(),
             },
         )
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index a6e52cb248..09778045bf 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from twisted.internet import defer
+
 from ._base import SQLBaseStore
 
 
@@ -24,19 +26,25 @@ class ProfileStore(SQLBaseStore):
             desc="create_profile",
         )
 
+    @defer.inlineCallbacks
     def get_profile_displayname(self, user_localpart):
-        return self._simple_select_one_onecol(
+        name = yield self._simple_select_one_onecol(
             table="profiles",
             keyvalues={"user_id": user_localpart},
             retcol="displayname",
             desc="get_profile_displayname",
         )
 
+        if name:
+            name = name.decode("utf8")
+
+        defer.returnValue(name)
+
     def set_profile_displayname(self, user_localpart, new_displayname):
         return self._simple_update_one(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"displayname": new_displayname},
+            updatevalues={"displayname": new_displayname.encode("utf8")},
             desc="set_profile_displayname",
         )
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index fe26d6d62f..7258f7b2a5 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -81,13 +81,23 @@ class RegistrationStore(SQLBaseStore):
         txn.execute("INSERT INTO access_tokens(user_id, token) " +
                     "VALUES (?,?)", [user_id, token])
 
+    @defer.inlineCallbacks
     def get_user_by_id(self, user_id):
-        query = ("SELECT users.name, users.password_hash FROM users"
-                 " WHERE users.name = ?")
-        return self._execute(
-            "get_user_by_id", self.cursor_to_dict, query, user_id
+        user_info = yield self._simple_select_one(
+            table="users",
+            keyvalues={
+                "name": user_id,
+            },
+            retcols=["name", "password_hash"],
+            allow_none=True,
         )
 
+        if user_info:
+            user_info["password_hash"] = user_info["password_hash"].decode("utf8")
+
+        defer.returnValue(user_info)
+
+
     @cached()
     # TODO(paul): Currently there's no code to invalidate this cache. That
     #   means if/when we ever add internal ways to invalidate access tokens or
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 501e947ad7..a1a76280fe 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -72,6 +72,7 @@ class RoomStore(SQLBaseStore):
             keyvalues={"room_id": room_id},
             retcols=RoomsTable.fields,
             desc="get_room",
+            allow_none=True,
         )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 8bc84dc24d..d9559f5902 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -65,4 +65,4 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
 ) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
-    ON local_media_repository_thumbnails (media_id);
+    ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 32defe2f79..552645c56f 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -14,7 +14,7 @@
  */
 CREATE TABLE IF NOT EXISTS profiles(
     user_id VARCHAR(255) NOT NULL,
-    displayname VARCHAR(255),
+    displayname VARBINARY(255),
     avatar_url VARCHAR(255),
     UNIQUE(user_id)
 ) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 0570bf95d9..bd13bba8c2 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -38,7 +38,6 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
 ) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination);
 CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
 -- So that we can do an efficient look up of all transactions that have yet to be successfully
 -- sent.
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 13ce335101..35bba854f9 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore):
             {
                 "event_id": event_id,
                 "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
+                "hash": hash_bytes,
             },
         )
 
@@ -99,7 +99,7 @@ class SignatureStore(SQLBaseStore):
             " WHERE event_id = ?"
         )
         txn.execute(query, (event_id, ))
-        return dict(txn.fetchall())
+        return {k: v for k, v in txn.fetchall()}
 
     def _store_event_reference_hash_txn(self, txn, event_id, algorithm,
                                         hash_bytes):
@@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore):
             {
                 "event_id": event_id,
                 "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
+                "hash": hash_bytes,
             },
         )
 
@@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore):
                 "event_id": event_id,
                 "signature_name": signature_name,
                 "key_id": key_id,
-                "signature": buffer(signature_bytes),
+                "signature": signature_bytes,
             },
         )
 
@@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore):
                 "event_id": event_id,
                 "prev_event_id": prev_event_id,
                 "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
+                "hash": hash_bytes,
             },
         )
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 03e1e3b808..e3e484fb2d 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -282,7 +282,7 @@ class TransactionStore(SQLBaseStore):
         query = (
             "UPDATE destinations"
             " SET retry_last_ts = ?, retry_interval = ?"
-            " WHERE destinations = ?"
+            " WHERE destination = ?"
         )
 
         txn.execute(
-- 
cgit 1.5.1


From a32e876ef43df22cec37aad748c32c0cda30428a Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 26 Mar 2015 13:40:16 +0000
Subject: Delete pushers when changing password

---
 synapse/handlers/login.py  |  3 +++
 synapse/push/pusherpool.py | 20 ++++++++++++++++++--
 synapse/storage/pusher.py  | 45 ++++++++++-----------------------------------
 3 files changed, 31 insertions(+), 37 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 7aff2e69e6..04f6dbb95e 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -70,4 +70,7 @@ class LoginHandler(BaseHandler):
 
         yield self.store.user_set_password_hash(user_id, password_hash)
         yield self.store.user_delete_access_tokens_apart_from(user_id, token_id)
+        yield self.hs.get_pusherpool().remove_pushers_by_user_access_token(
+            user_id, token_id
+        )
         yield self.store.flush_user(user_id)
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 46444157c9..0fdd7ea786 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -71,7 +71,7 @@ class PusherPool:
             "app_display_name": app_display_name,
             "device_display_name": device_display_name,
             "pushkey": pushkey,
-            "pushkey_ts": self.hs.get_clock().time_msec(),
+            "ts": self.hs.get_clock().time_msec(),
             "lang": lang,
             "data": data,
             "last_token": None,
@@ -98,6 +98,22 @@ class PusherPool:
                 )
                 self.remove_pusher(p['app_id'], p['pushkey'], p['user_name'])
 
+    @defer.inlineCallbacks
+    def remove_pushers_by_user_access_token(self, user_id, not_access_token_id):
+        all = yield self.store.get_all_pushers()
+        logger.info(
+            "Removing all pushers for user %s except access token %s",
+            user_id, not_access_token_id
+        )
+        for p in all:
+            if (p['user_name'] == user_id and
+                        p['access_token'] != not_access_token_id):
+                logger.info(
+                    "Removing pusher for app id %s, pushkey %s, user %s",
+                    p['app_id'], p['pushkey'], p['user_name']
+                )
+                self.remove_pusher(p['app_id'], p['pushkey'], p['user_name'])
+
     @defer.inlineCallbacks
     def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind,
                              app_id, app_display_name, device_display_name,
@@ -127,7 +143,7 @@ class PusherPool:
                 app_display_name=pusherdict['app_display_name'],
                 device_display_name=pusherdict['device_display_name'],
                 pushkey=pusherdict['pushkey'],
-                pushkey_ts=pusherdict['pushkey_ts'],
+                pushkey_ts=pusherdict['ts'],
                 data=pusherdict['data'],
                 last_token=pusherdict['last_token'],
                 last_success=pusherdict['last_success'],
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 423878c6a0..1c657beddb 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -28,11 +28,9 @@ logger = logging.getLogger(__name__)
 class PusherStore(SQLBaseStore):
     @defer.inlineCallbacks
     def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey):
+        cols = ",".join(PushersTable.fields)
         sql = (
-            "SELECT id, user_name, kind, profile_tag, app_id,"
-            "app_display_name, device_display_name, pushkey, ts, data, "
-            "last_token, last_success, failing_since "
-            "FROM pushers "
+            "SELECT "+cols+" FROM pushers "
             "WHERE app_id = ? AND pushkey = ?"
         )
 
@@ -43,51 +41,26 @@ class PusherStore(SQLBaseStore):
 
         ret = [
             {
-                "id": r[0],
-                "user_name": r[1],
-                "kind": r[2],
-                "profile_tag": r[3],
-                "app_id": r[4],
-                "app_display_name": r[5],
-                "device_display_name": r[6],
-                "pushkey": r[7],
-                "pushkey_ts": r[8],
-                "data": r[9],
-                "last_token": r[10],
-                "last_success": r[11],
-                "failing_since": r[12]
+                k: r[i] for i, k in enumerate(PushersTable.fields)
             }
             for r in rows
         ]
+        print ret
 
         defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def get_all_pushers(self):
+        cols = ",".join(PushersTable.fields)
         sql = (
-            "SELECT id, user_name, kind, profile_tag, app_id,"
-            "app_display_name, device_display_name, pushkey, ts, data, "
-            "last_token, last_success, failing_since "
-            "FROM pushers"
+            "SELECT "+cols+" FROM pushers"
         )
 
         rows = yield self._execute("get_all_pushers", None, sql)
 
         ret = [
             {
-                "id": r[0],
-                "user_name": r[1],
-                "kind": r[2],
-                "profile_tag": r[3],
-                "app_id": r[4],
-                "app_display_name": r[5],
-                "device_display_name": r[6],
-                "pushkey": r[7],
-                "pushkey_ts": r[8],
-                "data": r[9],
-                "last_token": r[10],
-                "last_success": r[11],
-                "failing_since": r[12]
+                k: r[i] for i, k in enumerate(PushersTable.fields)
             }
             for r in rows
         ]
@@ -166,13 +139,15 @@ class PushersTable(Table):
     fields = [
         "id",
         "user_name",
+        "access_token",
         "kind",
         "profile_tag",
         "app_id",
         "app_display_name",
         "device_display_name",
         "pushkey",
-        "pushkey_ts",
+        "ts",
+        "lang",
         "data",
         "last_token",
         "last_success",
-- 
cgit 1.5.1


From 59bf16eddcb793705ee6bc243a2158824f7e05c8 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 30 Mar 2015 18:13:10 +0100
Subject: New registration for C/S API v2. Only ReCAPTCHA working currently.

---
 synapse/api/constants.py                 |  2 +
 synapse/config/captcha.py                |  7 ++-
 synapse/handlers/auth.py                 | 90 +++++++++++++++++++++++++++-----
 synapse/handlers/register.py             | 11 +++-
 synapse/http/client.py                   |  2 +
 synapse/rest/client/v2_alpha/__init__.py |  4 +-
 synapse/rest/client/v2_alpha/_base.py    |  6 +++
 synapse/rest/client/v2_alpha/register.py | 86 ++++++++++++++++++++++++++++++
 8 files changed, 192 insertions(+), 16 deletions(-)
 create mode 100644 synapse/rest/client/v2_alpha/register.py

(limited to 'synapse/handlers')

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index b16bf4247d..3e0ce170a4 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -62,6 +62,8 @@ class LoginType(object):
     APPLICATION_SERVICE = u"m.login.application_service"
     SHARED_SECRET = u"org.matrix.login.shared_secret"
 
+    HIDDEN_TYPES = [APPLICATION_SERVICE, SHARED_SECRET]
+
 
 class EventTypes(object):
     Member = "m.room.member"
diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py
index 7e21c7414d..07fbfadc0f 100644
--- a/synapse/config/captcha.py
+++ b/synapse/config/captcha.py
@@ -20,6 +20,7 @@ class CaptchaConfig(Config):
     def __init__(self, args):
         super(CaptchaConfig, self).__init__(args)
         self.recaptcha_private_key = args.recaptcha_private_key
+        self.recaptcha_public_key = args.recaptcha_public_key
         self.enable_registration_captcha = args.enable_registration_captcha
         self.captcha_ip_origin_is_x_forwarded = (
             args.captcha_ip_origin_is_x_forwarded
@@ -30,9 +31,13 @@ class CaptchaConfig(Config):
     def add_arguments(cls, parser):
         super(CaptchaConfig, cls).add_arguments(parser)
         group = parser.add_argument_group("recaptcha")
+        group.add_argument(
+            "--recaptcha-public-key", type=str, default="YOUR_PUBLIC_KEY",
+            help="This Home Server's ReCAPTCHA public key."
+        )
         group.add_argument(
             "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY",
-            help="The matching private key for the web client's public key."
+            help="This Home Server's ReCAPTCHA private key."
         )
         group.add_argument(
             "--enable-registration-captcha", type=bool, default=False,
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index e4a73da9a7..ec625f4ea8 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -19,9 +19,12 @@ from ._base import BaseHandler
 from synapse.api.constants import LoginType
 from synapse.types import UserID
 from synapse.api.errors import LoginError, Codes
+from synapse.http.client import SimpleHttpClient
+from twisted.web.client import PartialDownloadError
 
 import logging
 import bcrypt
+import simplejson
 
 
 logger = logging.getLogger(__name__)
@@ -33,7 +36,7 @@ class AuthHandler(BaseHandler):
         super(AuthHandler, self).__init__(hs)
 
     @defer.inlineCallbacks
-    def check_auth(self, flows, clientdict):
+    def check_auth(self, flows, clientdict, clientip=None):
         """
         Takes a dictionary sent by the client in the login / registration
         protocol and handles the login flow.
@@ -50,11 +53,12 @@ class AuthHandler(BaseHandler):
             login request and should be passed back to the client.
         """
         types = {
-            LoginType.PASSWORD: self.check_password_auth
+            LoginType.PASSWORD: self.check_password_auth,
+            LoginType.RECAPTCHA: self.check_recaptcha,
         }
 
-        if 'auth' not in clientdict:
-            defer.returnValue((False, auth_dict_for_flows(flows)))
+        if not clientdict or 'auth' not in clientdict:
+            defer.returnValue((False, self.auth_dict_for_flows(flows)))
 
         authdict = clientdict['auth']
 
@@ -67,7 +71,7 @@ class AuthHandler(BaseHandler):
             raise LoginError(400, "", Codes.MISSING_PARAM)
         if authdict['type'] not in types:
             raise LoginError(400, "", Codes.UNRECOGNIZED)
-        result = yield types[authdict['type']](authdict)
+        result = yield types[authdict['type']](authdict, clientip)
         if result:
             creds[authdict['type']] = result
 
@@ -76,12 +80,12 @@ class AuthHandler(BaseHandler):
                 logger.info("Auth completed with creds: %r", creds)
                 defer.returnValue((True, creds))
 
-        ret = auth_dict_for_flows(flows)
+        ret = self.auth_dict_for_flows(flows)
         ret['completed'] = creds.keys()
         defer.returnValue((False, ret))
 
     @defer.inlineCallbacks
-    def check_password_auth(self, authdict):
+    def check_password_auth(self, authdict, _):
         if "user" not in authdict or "password" not in authdict:
             raise LoginError(400, "", Codes.MISSING_PARAM)
 
@@ -93,17 +97,77 @@ class AuthHandler(BaseHandler):
         user_info = yield self.store.get_user_by_id(user_id=user)
         if not user_info:
             logger.warn("Attempted to login as %s but they do not exist", user)
-            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+            raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
         stored_hash = user_info[0]["password_hash"]
         if bcrypt.checkpw(password, stored_hash):
             defer.returnValue(user)
         else:
             logger.warn("Failed password login for user %s", user)
-            raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+            raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
+    @defer.inlineCallbacks
+    def check_recaptcha(self, authdict, clientip):
+        try:
+            user_response = authdict["response"]
+        except KeyError:
+            # Client tried to provide captcha but didn't give the parameter:
+            # bad request.
+            raise LoginError(
+                400, "Captcha response is required",
+                errcode=Codes.CAPTCHA_NEEDED
+            )
+
+        logger.info(
+            "Submitting recaptcha response %s with remoteip %s",
+            user_response, clientip
+        )
+
+        # TODO: get this from the homeserver rather than creating a new one for
+        # each request
+        try:
+            client = SimpleHttpClient(self.hs)
+            data = yield client.post_urlencoded_get_json(
+                "https://www.google.com/recaptcha/api/siteverify",
+                args={
+                    'secret': self.hs.config.recaptcha_private_key,
+                    'response': user_response,
+                    'remoteip': clientip,
+                }
+            )
+        except PartialDownloadError as pde:
+            # Twisted is silly
+            data = pde.response
+        resp_body = simplejson.loads(data)
+        if 'success' in resp_body and resp_body['success']:
+            defer.returnValue(True)
+        raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
+
+    def get_params_recaptcha(self):
+        return {"public_key": self.hs.config.recaptcha_public_key}
+
+    def auth_dict_for_flows(self, flows):
+        public_flows = []
+        for f in flows:
+            hidden = False
+            for stagetype in f:
+                if stagetype in LoginType.HIDDEN_TYPES:
+                    hidden = True
+            if not hidden:
+                public_flows.append(f)
+
+        get_params = {
+            LoginType.RECAPTCHA: self.get_params_recaptcha,
+        }
+
+        params = {}
+
+        for f in public_flows:
+            for stage in f:
+                if stage in get_params and stage not in params:
+                    params[stage] = get_params[stage]()
 
-def auth_dict_for_flows(flows):
-    return {
-        "flows": {"stages": f for f in flows}
-    }
+        return {
+            "flows": [{"stages": f} for f in public_flows],
+            "params": params
+        }
\ No newline at end of file
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index c25e321099..542759a827 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -157,7 +157,11 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def check_recaptcha(self, ip, private_key, challenge, response):
-        """Checks a recaptcha is correct."""
+        """
+        Checks a recaptcha is correct.
+
+        Used only by c/s api v1
+        """
 
         captcha_response = yield self._validate_captcha(
             ip,
@@ -282,6 +286,8 @@ class RegistrationHandler(BaseHandler):
     def _validate_captcha(self, ip_addr, private_key, challenge, response):
         """Validates the captcha provided.
 
+        Used only by c/s api v1
+
         Returns:
             dict: Containing 'valid'(bool) and 'error_url'(str) if invalid.
 
@@ -299,6 +305,9 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _submit_captcha(self, ip_addr, private_key, challenge, response):
+        """
+        Used only by c/s api v1
+        """
         # TODO: get this from the homeserver rather than creating a new one for
         # each request
         client = CaptchaServerHttpClient(self.hs)
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 2ae1c4d3a4..e8a5dedab4 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -200,6 +200,8 @@ class CaptchaServerHttpClient(SimpleHttpClient):
     """
     Separate HTTP client for talking to google's captcha servers
     Only slightly special because accepts partial download responses
+
+    used only by c/s api v1
     """
 
     @defer.inlineCallbacks
diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py
index 041f538e20..98189ead26 100644
--- a/synapse/rest/client/v2_alpha/__init__.py
+++ b/synapse/rest/client/v2_alpha/__init__.py
@@ -16,7 +16,8 @@
 from . import (
     sync,
     filter,
-    password
+    password,
+    register
 )
 
 from synapse.http.server import JsonResource
@@ -34,3 +35,4 @@ class ClientV2AlphaRestResource(JsonResource):
         sync.register_servlets(hs, client_resource)
         filter.register_servlets(hs, client_resource)
         password.register_servlets(hs, client_resource)
+        register.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index c772cc986f..db2c9b244a 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -40,6 +40,12 @@ def client_v2_pattern(path_regex):
     return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex)
 
 
+def parse_request_allow_empty(request):
+    content = request.content.read()
+    if content == None or content == '':
+        return None
+    return simplejson.loads(content)
+
 def parse_json_dict_from_request(request):
     try:
         content = simplejson.loads(request.content.read())
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
new file mode 100644
index 0000000000..84da010c29
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.constants import LoginType
+from synapse.api.errors import LoginError, SynapseError, Codes
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_pattern, parse_request_allow_empty
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class RegisterRestServlet(RestServlet):
+    PATTERN = client_v2_pattern("/register")
+
+    def __init__(self, hs):
+        super(RegisterRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.auth_handler = hs.get_handlers().auth_handler
+        self.registration_handler = hs.get_handlers().registration_handler
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        body = parse_request_allow_empty(request)
+
+        authed, result = yield self.auth_handler.check_auth([
+            [LoginType.RECAPTCHA],
+            [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
+            [LoginType.APPLICATION_SERVICE]
+        ], body)
+
+        if not authed:
+            defer.returnValue((401, result))
+
+        is_application_server = LoginType.APPLICATION_SERVICE in result
+        is_using_shared_secret = LoginType.SHARED_SECRET in result
+
+        can_register = (
+            not self.hs.config.disable_registration
+            or is_application_server
+            or is_using_shared_secret
+        )
+        if not can_register:
+            raise SynapseError(403, "Registration has been disabled")
+
+        if 'username' not in body or 'password' not in body:
+            raise SynapseError(400, "", Codes.MISSING_PARAM)
+        desired_username = body['username']
+        new_password = body['password']
+
+        (user_id, token) = yield self.registration_handler.register(
+            localpart=desired_username,
+            password=new_password
+        )
+        result = {
+            "user_id": user_id,
+            "access_token": token,
+            "home_server": self.hs.hostname,
+        }
+
+        defer.returnValue((200, result))
+
+    def on_OPTIONS(self, _):
+        return 200, {}
+
+
+def register_servlets(hs, http_server):
+    RegisterRestServlet(hs).register(http_server)
\ No newline at end of file
-- 
cgit 1.5.1


From 9f642a93ecab62fb56776ad4b7d7b062b869b66a Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 31 Mar 2015 09:50:44 +0100
Subject: pep8

---
 synapse/handlers/auth.py                 | 2 +-
 synapse/rest/client/v2_alpha/_base.py    | 3 ++-
 synapse/rest/client/v2_alpha/register.py | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index ec625f4ea8..26df9fcd86 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -170,4 +170,4 @@ class AuthHandler(BaseHandler):
         return {
             "flows": [{"stages": f} for f in public_flows],
             "params": params
-        }
\ No newline at end of file
+        }
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index db2c9b244a..8adcc9dd95 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -42,10 +42,11 @@ def client_v2_pattern(path_regex):
 
 def parse_request_allow_empty(request):
     content = request.content.read()
-    if content == None or content == '':
+    if content is None or content == '':
         return None
     return simplejson.loads(content)
 
+
 def parse_json_dict_from_request(request):
     try:
         content = simplejson.loads(request.content.read())
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 84da010c29..4a53e03743 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.api.constants import LoginType
-from synapse.api.errors import LoginError, SynapseError, Codes
+from synapse.api.errors import SynapseError, Codes
 from synapse.http.servlet import RestServlet
 
 from ._base import client_v2_pattern, parse_request_allow_empty
@@ -83,4 +83,4 @@ class RegisterRestServlet(RestServlet):
 
 
 def register_servlets(hs, http_server):
-    RegisterRestServlet(hs).register(http_server)
\ No newline at end of file
+    RegisterRestServlet(hs).register(http_server)
-- 
cgit 1.5.1


From e9c908ebc09ccc050bd09692c5413124a8c3c06e Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 1 Apr 2015 15:05:30 +0100
Subject: Completely replace fallback auth for C/S V2:

 * Now only the auth part goes to fallback, not the whole operation
 * Auth fallback is a normal API endpoint, not a static page
 * Params like the recaptcha pubkey can just live in the config
Involves a little engineering on JsonResource so its servlets aren't always forced to return JSON. I should document this more, in fact I'll do that now.
---
 static/client/register/style.css         |   6 +-
 synapse/handlers/auth.py                 |  98 ++++++++++++----
 synapse/http/server.py                   |   7 +-
 synapse/rest/client/v2_alpha/__init__.py |   4 +-
 synapse/rest/client/v2_alpha/auth.py     | 189 +++++++++++++++++++++++++++++++
 synapse/rest/client/v2_alpha/register.py |   2 +-
 6 files changed, 280 insertions(+), 26 deletions(-)
 create mode 100644 synapse/rest/client/v2_alpha/auth.py

(limited to 'synapse/handlers')

diff --git a/static/client/register/style.css b/static/client/register/style.css
index a3398852b9..5a7b6eebf2 100644
--- a/static/client/register/style.css
+++ b/static/client/register/style.css
@@ -37,9 +37,13 @@ textarea, input {
     margin: auto
 }
 
+.g-recaptcha div {
+    margin: auto;
+}
+
 #registrationForm {
     text-align: left;
-    padding: 1em;
+    padding: 5px;
     margin-bottom: 40px;
     display: inline-block;
     
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 26df9fcd86..3d2461dd7d 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -20,12 +20,15 @@ from synapse.api.constants import LoginType
 from synapse.types import UserID
 from synapse.api.errors import LoginError, Codes
 from synapse.http.client import SimpleHttpClient
+
 from twisted.web.client import PartialDownloadError
 
 import logging
 import bcrypt
 import simplejson
 
+import synapse.util.stringutils as stringutils
+
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +37,11 @@ class AuthHandler(BaseHandler):
 
     def __init__(self, hs):
         super(AuthHandler, self).__init__(hs)
+        self.checkers = {
+            LoginType.PASSWORD: self._check_password_auth,
+            LoginType.RECAPTCHA: self._check_recaptcha,
+        }
+        self.sessions = {}
 
     @defer.inlineCallbacks
     def check_auth(self, flows, clientdict, clientip=None):
@@ -52,40 +60,64 @@ class AuthHandler(BaseHandler):
             If authed is false, the dictionary is the server response to the
             login request and should be passed back to the client.
         """
-        types = {
-            LoginType.PASSWORD: self.check_password_auth,
-            LoginType.RECAPTCHA: self.check_recaptcha,
-        }
 
         if not clientdict or 'auth' not in clientdict:
-            defer.returnValue((False, self.auth_dict_for_flows(flows)))
+            sess = self._get_session_info(None)
+            defer.returnValue(
+                (False, self._auth_dict_for_flows(flows, sess))
+            )
 
         authdict = clientdict['auth']
 
-        # In future: support sessions & retrieve previously succeeded
-        # login types
-        creds = {}
+        sess = self._get_session_info(
+            authdict['session'] if 'session' in authdict else None
+        )
+        if 'creds' not in sess:
+            sess['creds'] = {}
+        creds = sess['creds']
 
         # check auth type currently being presented
-        if 'type' not in authdict:
-            raise LoginError(400, "", Codes.MISSING_PARAM)
-        if authdict['type'] not in types:
-            raise LoginError(400, "", Codes.UNRECOGNIZED)
-        result = yield types[authdict['type']](authdict, clientip)
-        if result:
-            creds[authdict['type']] = result
+        if 'type' in authdict:
+            if authdict['type'] not in self.checkers:
+                raise LoginError(400, "", Codes.UNRECOGNIZED)
+            result = yield self.checkers[authdict['type']](authdict, clientip)
+            if result:
+                creds[authdict['type']] = result
+                self._save_session(sess)
 
         for f in flows:
             if len(set(f) - set(creds.keys())) == 0:
                 logger.info("Auth completed with creds: %r", creds)
+                self._remove_session(sess)
                 defer.returnValue((True, creds))
 
-        ret = self.auth_dict_for_flows(flows)
+        ret = self._auth_dict_for_flows(flows, sess)
         ret['completed'] = creds.keys()
         defer.returnValue((False, ret))
 
     @defer.inlineCallbacks
-    def check_password_auth(self, authdict, _):
+    def add_oob_auth(self, stagetype, authdict, clientip):
+        if stagetype not in self.checkers:
+            raise LoginError(400, "", Codes.MISSING_PARAM)
+        if 'session' not in authdict:
+            raise LoginError(400, "", Codes.MISSING_PARAM)
+
+        sess = self._get_session_info(
+            authdict['session']
+        )
+        if 'creds' not in sess:
+            sess['creds'] = {}
+        creds = sess['creds']
+
+        result = yield self.checkers[stagetype](authdict, clientip)
+        if result:
+            creds[stagetype] = result
+            self._save_session(sess)
+            defer.returnValue(True)
+        defer.returnValue(False)
+
+    @defer.inlineCallbacks
+    def _check_password_auth(self, authdict, _):
         if "user" not in authdict or "password" not in authdict:
             raise LoginError(400, "", Codes.MISSING_PARAM)
 
@@ -107,7 +139,7 @@ class AuthHandler(BaseHandler):
             raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
     @defer.inlineCallbacks
-    def check_recaptcha(self, authdict, clientip):
+    def _check_recaptcha(self, authdict, clientip):
         try:
             user_response = authdict["response"]
         except KeyError:
@@ -143,10 +175,10 @@ class AuthHandler(BaseHandler):
             defer.returnValue(True)
         raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
-    def get_params_recaptcha(self):
+    def _get_params_recaptcha(self):
         return {"public_key": self.hs.config.recaptcha_public_key}
 
-    def auth_dict_for_flows(self, flows):
+    def _auth_dict_for_flows(self, flows, session):
         public_flows = []
         for f in flows:
             hidden = False
@@ -157,7 +189,7 @@ class AuthHandler(BaseHandler):
                 public_flows.append(f)
 
         get_params = {
-            LoginType.RECAPTCHA: self.get_params_recaptcha,
+            LoginType.RECAPTCHA: self._get_params_recaptcha,
         }
 
         params = {}
@@ -168,6 +200,30 @@ class AuthHandler(BaseHandler):
                     params[stage] = get_params[stage]()
 
         return {
+            "session": session['id'],
             "flows": [{"stages": f} for f in public_flows],
             "params": params
         }
+
+    def _get_session_info(self, session_id):
+        if session_id not in self.sessions:
+            session_id = None
+
+        if not session_id:
+            # create a new session
+            while session_id is None or session_id in self.sessions:
+                session_id = stringutils.random_string(24)
+            self.sessions[session_id] = {
+                "id": session_id,
+            }
+
+        return self.sessions[session_id]
+
+    def _save_session(self, session):
+        # TODO: Persistent storage
+        logger.debug("Saving session %s", session)
+        self.sessions[session["id"]] = session
+
+    def _remove_session(self, session):
+        logger.debug("Removing session %s", session)
+        del self.sessions[session["id"]]
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 30c3aa5cac..76c561d105 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -170,9 +170,12 @@ class JsonResource(HttpServer, resource.Resource):
                     request.method, request.path
                 )
 
-                code, response = yield callback(request, *args)
+                callback_return = yield callback(request, *args)
+                if callback_return is not None:
+                    code, response = callback_return
+
+                    self._send_response(request, code, response)
 
-                self._send_response(request, code, response)
                 response_timer.inc_by(
                     self.clock.time_msec() - start, request.method, servlet_classname
                 )
diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py
index 98189ead26..86e4bc729e 100644
--- a/synapse/rest/client/v2_alpha/__init__.py
+++ b/synapse/rest/client/v2_alpha/__init__.py
@@ -17,7 +17,8 @@ from . import (
     sync,
     filter,
     password,
-    register
+    register,
+    auth
 )
 
 from synapse.http.server import JsonResource
@@ -36,3 +37,4 @@ class ClientV2AlphaRestResource(JsonResource):
         filter.register_servlets(hs, client_resource)
         password.register_servlets(hs, client_resource)
         register.register_servlets(hs, client_resource)
+        auth.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py
new file mode 100644
index 0000000000..7a518e226f
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/auth.py
@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.api.constants import LoginType
+from synapse.api.errors import SynapseError
+from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_pattern
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+RECAPTCHA_TEMPLATE = """
+<html>
+<head>
+<title>Authentication</title>
+<meta name='viewport' content='width=device-width, initial-scale=1,
+    user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
+<script src="https://www.google.com/recaptcha/api.js"
+    async defer></script>
+<script src="//code.jquery.com/jquery-1.11.2.min.js"></script>
+<link rel="stylesheet" href="/_matrix/static/client/register/style.css">
+<script>
+function captchaDone() {
+    $('#registrationForm').submit();
+}
+</script>
+</head>
+<body>
+<form id="registrationForm" method="post" action="%(myurl)s">
+    <div>
+        <p>
+        Hello! We need to prevent computer programs and other automated
+        things from creating accounts on this server.
+        </p>
+        <p>
+        Please verify that you're not a robot.
+        </p>
+        <input type="hidden" name="session" value="%(session)s" />
+        <div class="g-recaptcha"
+            data-sitekey="%(sitekey)s"
+            data-callback="captchaDone">
+        </div>
+        <noscript>
+        <input type="submit" value="All Done" />
+        </noscript>
+        </div>
+    </div>
+</form>
+</body>
+</html>
+"""
+
+SUCCESS_TEMPLATE = """
+<html>
+<head>
+<title>Success!</title>
+<meta name='viewport' content='width=device-width, initial-scale=1,
+    user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
+<link rel="stylesheet" href="/_matrix/static/client/register/style.css">
+<script>
+if (window.onAuthDone != undefined) {
+    window.onAuthDone();
+}
+</script>
+</head>
+<body>
+    <div>
+        <p>Thank you</p>
+        <p>You may now close this window and return to the application</p>
+    </div>
+</body>
+</html>
+"""
+
+class AuthRestServlet(RestServlet):
+    """
+    Handles Client / Server API authentication in any situations where it
+    cannot be handled in the normal flow (with requests to the same endpoint).
+    Current use is for web fallback auth.
+    """
+    PATTERN = client_v2_pattern("/auth/(?P<stagetype>[\w\.]*)/fallback/web")
+
+    def __init__(self, hs):
+        super(AuthRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.auth_handler = hs.get_handlers().auth_handler
+        self.registration_handler = hs.get_handlers().registration_handler
+
+    @defer.inlineCallbacks
+    def on_GET(self, request, stagetype):
+        yield
+        if stagetype == LoginType.RECAPTCHA:
+            if ('session' not in request.args or
+                    len(request.args['session']) == 0):
+                raise SynapseError(400, "No session supplied")
+
+            session = request.args["session"][0]
+
+            html = RECAPTCHA_TEMPLATE % {
+                'session': session,
+                'myurl': "%s/auth/%s/fallback/web" % (
+                    CLIENT_V2_ALPHA_PREFIX, LoginType.RECAPTCHA
+                ),
+                'sitekey': self.hs.config.recaptcha_public_key,
+            }
+            html_bytes = html.encode("utf8")
+            request.setResponseCode(200)
+            request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+            request.setHeader(b"Server", self.hs.version_string)
+            request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
+
+            request.write(html_bytes)
+            request.finish()
+            defer.returnValue(None)
+        else:
+            raise SynapseError(404, "Unknown auth stage type")
+
+    @defer.inlineCallbacks
+    def on_POST(self, request, stagetype):
+        yield
+        if stagetype == "m.login.recaptcha":
+            if ('g-recaptcha-response' not in request.args or
+                    len(request.args['g-recaptcha-response'])) == 0:
+                raise SynapseError(400, "No captcha response supplied")
+            if ('session' not in request.args or
+                    len(request.args['session'])) == 0:
+                raise SynapseError(400, "No session supplied")
+
+            session = request.args['session'][0]
+
+            authdict = {
+                'response': request.args['g-recaptcha-response'][0],
+                'session': session,
+            }
+
+            success = yield self.auth_handler.add_oob_auth(
+                LoginType.RECAPTCHA,
+                authdict,
+                self.hs.get_ip_from_request(request)
+            )
+
+            if success:
+                html = SUCCESS_TEMPLATE
+            else:
+                html = RECAPTCHA_TEMPLATE % {
+                    'session': session,
+                    'myurl': "%s/auth/%s/fallback/web" % (
+                        CLIENT_V2_ALPHA_PREFIX, LoginType.RECAPTCHA
+                    ),
+                    'sitekey': self.hs.config.recaptcha_public_key,
+                }
+            html_bytes = html.encode("utf8")
+            request.setResponseCode(200)
+            request.setHeader(b"Content-Type", b"text/html; charset=utf-8")
+            request.setHeader(b"Server", self.hs.version_string)
+            request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),))
+
+            request.write(html_bytes)
+            request.finish()
+
+            defer.returnValue(None)
+        else:
+            raise SynapseError(404, "Unknown auth stage type")
+
+    def on_OPTIONS(self, _):
+        return 200, {}
+
+
+def register_servlets(hs, http_server):
+    AuthRestServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 4a53e03743..537918ea27 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -45,7 +45,7 @@ class RegisterRestServlet(RestServlet):
             [LoginType.RECAPTCHA],
             [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
             [LoginType.APPLICATION_SERVICE]
-        ], body)
+        ], body, self.hs.get_ip_from_request(request))
 
         if not authed:
             defer.returnValue((401, result))
-- 
cgit 1.5.1


From 1ec6fa98c93114f1799a26100a970273e97576bb Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 14:17:16 +0100
Subject: Parellelize initial sync

---
 synapse/handlers/message.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 7b9685be7f..9c2ef94290 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -274,7 +274,8 @@ class MessageHandler(BaseHandler):
         if limit is None:
             limit = 10
 
-        for event in room_list:
+        @defer.inlineCallbacks
+        def handle_room(event):
             d = {
                 "room_id": event.room_id,
                 "membership": event.membership,
@@ -290,7 +291,7 @@ class MessageHandler(BaseHandler):
             rooms_ret.append(d)
 
             if event.membership != Membership.JOIN:
-                continue
+                return
             try:
                 messages, token = yield self.store.get_recent_events_for_room(
                     event.room_id,
@@ -321,6 +322,11 @@ class MessageHandler(BaseHandler):
             except:
                 logger.exception("Failed to get snapshot")
 
+        yield defer.gatherResults(
+            [handle_room(e) for e in room_list],
+            consumeErrors=True
+        )
+
         ret = {
             "rooms": rooms_ret,
             "presence": presence,
-- 
cgit 1.5.1


From bc6cef823f20f3d14c25ee5409fdea44e518087e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 14:21:59 +0100
Subject: Do more parellelization for initialSync

---
 synapse/handlers/message.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 9c2ef94290..9667bb8674 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -293,10 +293,17 @@ class MessageHandler(BaseHandler):
             if event.membership != Membership.JOIN:
                 return
             try:
-                messages, token = yield self.store.get_recent_events_for_room(
-                    event.room_id,
-                    limit=limit,
-                    end_token=now_token.room_key,
+                (messages, token), current_state = yield defer.gatherResults(
+                    [
+                        self.store.get_recent_events_for_room(
+                            event.room_id,
+                            limit=limit,
+                            end_token=now_token.room_key,
+                        ),
+                        self.state_handler.get_current_state(
+                            event.room_id
+                        ),
+                    ]
                 )
 
                 start_token = now_token.copy_and_replace("room_key", token[0])
@@ -312,9 +319,6 @@ class MessageHandler(BaseHandler):
                     "end": end_token.to_string(),
                 }
 
-                current_state = yield self.state_handler.get_current_state(
-                    event.room_id
-                )
                 d["state"] = [
                     serialize_event(c, time_now, as_client_event)
                     for c in current_state.values()
-- 
cgit 1.5.1


From e845434028ff52f25c5b967c2d1dffcb716ed31e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 15:05:45 +0100
Subject: Remove run_on_reactor()s

---
 synapse/handlers/_base.py | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index 48816a242d..d0c027de3a 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -58,8 +58,6 @@ class BaseHandler(object):
 
     @defer.inlineCallbacks
     def _create_new_client_event(self, builder):
-        yield run_on_reactor()
-
         latest_ret = yield self.store.get_latest_events_in_room(
             builder.room_id,
         )
@@ -101,8 +99,6 @@ class BaseHandler(object):
     @defer.inlineCallbacks
     def handle_new_client_event(self, event, context, extra_destinations=[],
                                 extra_users=[], suppress_auth=False):
-        yield run_on_reactor()
-
         # We now need to go and hit out to wherever we need to hit out to.
 
         if not suppress_auth:
-- 
cgit 1.5.1


From d5ff9effcf59b55009eb8226d5f1bbac5a95bdff Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 15:05:57 +0100
Subject: Don't wait on federation_handler.handle_new_event

---
 synapse/handlers/_base.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index d0c027de3a..e4471dd9b6 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -139,7 +139,9 @@ class BaseHandler(object):
                 )
 
         # Don't block waiting on waking up all the listeners.
-        d = self.notifier.on_new_room_event(event, extra_users=extra_users)
+        notify_d = self.notifier.on_new_room_event(
+            event, extra_users=extra_users
+        )
 
         def log_failure(f):
             logger.warn(
@@ -147,8 +149,10 @@ class BaseHandler(object):
                 event.event_id, f.value
             )
 
-        d.addErrback(log_failure)
+        notify_d.addErrback(log_failure)
 
-        yield federation_handler.handle_new_event(
+        fed_d = federation_handler.handle_new_event(
             event, destinations=destinations,
         )
+
+        fed_d.addErrback(log_failure)
-- 
cgit 1.5.1


From a19b73990962ff3bfe8b2cae59446bbe7f93ec5c Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 15 Apr 2015 15:50:38 +0100
Subject: Regstration with email in v2

---
 synapse/handlers/__init__.py             |  2 +
 synapse/handlers/auth.py                 | 64 +++++++++++++++++++++----------
 synapse/handlers/identity.py             | 66 ++++++++++++++++++++++++++++++++
 synapse/handlers/register.py             |  6 ++-
 synapse/rest/client/v2_alpha/password.py |  6 +--
 synapse/rest/client/v2_alpha/register.py |  8 ++--
 6 files changed, 123 insertions(+), 29 deletions(-)
 create mode 100644 synapse/handlers/identity.py

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py
index 336ce15701..d1b0e032a3 100644
--- a/synapse/handlers/__init__.py
+++ b/synapse/handlers/__init__.py
@@ -30,6 +30,7 @@ from .admin import AdminHandler
 from .appservice import ApplicationServicesHandler
 from .sync import SyncHandler
 from .auth import AuthHandler
+from .identity import IdentityHandler
 
 
 class Handlers(object):
@@ -60,3 +61,4 @@ class Handlers(object):
         )
         self.sync_handler = SyncHandler(hs)
         self.auth_handler = AuthHandler(hs)
+        self.identity_handler = IdentityHandler(hs)
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 3d2461dd7d..2cc54707a2 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -20,6 +20,7 @@ from synapse.api.constants import LoginType
 from synapse.types import UserID
 from synapse.api.errors import LoginError, Codes
 from synapse.http.client import SimpleHttpClient
+from synapse.util.async import run_on_reactor
 
 from twisted.web.client import PartialDownloadError
 
@@ -40,6 +41,7 @@ class AuthHandler(BaseHandler):
         self.checkers = {
             LoginType.PASSWORD: self._check_password_auth,
             LoginType.RECAPTCHA: self._check_recaptcha,
+            LoginType.EMAIL_IDENTITY: self._check_email_identity,
         }
         self.sessions = {}
 
@@ -54,24 +56,37 @@ class AuthHandler(BaseHandler):
             authdict: The dictionary from the client root level, not the
                       'auth' key: this method prompts for auth if none is sent.
         Returns:
-            A tuple of authed, dict where authed is true if the client
-            has successfully completed an auth flow. If it is true, the dict
-            contains the authenticated credentials of each stage.
-            If authed is false, the dictionary is the server response to the
-            login request and should be passed back to the client.
+            A tuple of authed, dict, dict where authed is true if the client
+            has successfully completed an auth flow. If it is true, the first
+            dict contains the authenticated credentials of each stage.
+
+            If authed is false, the first dictionary is the server response to
+            the login request and should be passed back to the client.
+
+            In either case, the second dict contains the parameters for this
+            request (which may have been given only in a previous call).
         """
 
-        if not clientdict or 'auth' not in clientdict:
-            sess = self._get_session_info(None)
+        authdict = None
+        sid = None
+        if clientdict and 'auth' in clientdict:
+            authdict = clientdict['auth']
+            del clientdict['auth']
+            if 'session' in authdict:
+                sid = authdict['session']
+        sess = self._get_session_info(sid)
+
+        if len(clientdict) > 0:
+            sess['clientdict'] = clientdict
+            self._save_session(sess)
+        elif 'clientdict' in sess:
+            clientdict = sess['clientdict']
+
+        if not authdict:
             defer.returnValue(
-                (False, self._auth_dict_for_flows(flows, sess))
+                (False, self._auth_dict_for_flows(flows, sess), clientdict)
             )
 
-        authdict = clientdict['auth']
-
-        sess = self._get_session_info(
-            authdict['session'] if 'session' in authdict else None
-        )
         if 'creds' not in sess:
             sess['creds'] = {}
         creds = sess['creds']
@@ -89,11 +104,11 @@ class AuthHandler(BaseHandler):
             if len(set(f) - set(creds.keys())) == 0:
                 logger.info("Auth completed with creds: %r", creds)
                 self._remove_session(sess)
-                defer.returnValue((True, creds))
+                defer.returnValue((True, creds, clientdict))
 
         ret = self._auth_dict_for_flows(flows, sess)
         ret['completed'] = creds.keys()
-        defer.returnValue((False, ret))
+        defer.returnValue((False, ret, clientdict))
 
     @defer.inlineCallbacks
     def add_oob_auth(self, stagetype, authdict, clientip):
@@ -175,18 +190,25 @@ class AuthHandler(BaseHandler):
             defer.returnValue(True)
         raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
+    @defer.inlineCallbacks
+    def _check_email_identity(self, authdict, _):
+        yield run_on_reactor()
+
+        threepidCreds = authdict['threepidCreds']
+        identity_handler = self.hs.get_handlers().identity_handler
+
+        logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,))
+        threepid = yield identity_handler.threepid_from_creds(threepidCreds)
+
+        defer.returnValue(threepid)
+
     def _get_params_recaptcha(self):
         return {"public_key": self.hs.config.recaptcha_public_key}
 
     def _auth_dict_for_flows(self, flows, session):
         public_flows = []
         for f in flows:
-            hidden = False
-            for stagetype in f:
-                if stagetype in LoginType.HIDDEN_TYPES:
-                    hidden = True
-            if not hidden:
-                public_flows.append(f)
+            public_flows.append(f)
 
         get_params = {
             LoginType.RECAPTCHA: self._get_params_recaptcha,
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
new file mode 100644
index 0000000000..671d366e40
--- /dev/null
+++ b/synapse/handlers/identity.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for interacting with Identity Servers"""
+from twisted.internet import defer
+
+from synapse.api.errors import (
+    CodeMessageException
+)
+from ._base import BaseHandler
+from synapse.http.client import SimpleHttpClient
+from synapse.util.async import run_on_reactor
+
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class IdentityHandler(BaseHandler):
+
+    def __init__(self, hs):
+        super(IdentityHandler, self).__init__(hs)
+
+    @defer.inlineCallbacks
+    def threepid_from_creds(self, creds):
+        yield run_on_reactor()
+
+        # TODO: get this from the homeserver rather than creating a new one for
+        # each request
+        http_client = SimpleHttpClient(self.hs)
+        # XXX: make this configurable!
+        #trustedIdServers = ['matrix.org', 'localhost:8090']
+        trustedIdServers = ['matrix.org']
+        if not creds['idServer'] in trustedIdServers:
+            logger.warn('%s is not a trusted ID server: rejecting 3pid ' +
+                        'credentials', creds['idServer'])
+            defer.returnValue(None)
+
+        data = {}
+        try:
+            data = yield http_client.get_json(
+                "https://%s%s" % (
+                    creds['idServer'],
+                    "/_matrix/identity/api/v1/3pid/getValidated3pid"
+                ),
+                {'sid': creds['sid'], 'clientSecret': creds['clientSecret']}
+            )
+        except CodeMessageException as e:
+            data = json.loads(e.msg)
+
+        if 'medium' in data:
+            defer.returnValue(data)
+        defer.returnValue(None)
\ No newline at end of file
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 542759a827..6759a8c582 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -180,7 +180,11 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def register_email(self, threepidCreds):
-        """Registers emails with an identity server."""
+        """
+        Registers emails with an identity server.
+
+        Used only by c/s api v1
+        """
 
         for c in threepidCreds:
             logger.info("validating theeepidcred sid %s on id server %s",
diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py
index 85954c71cd..cb0c8cfb55 100644
--- a/synapse/rest/client/v2_alpha/password.py
+++ b/synapse/rest/client/v2_alpha/password.py
@@ -41,7 +41,7 @@ class PasswordRestServlet(RestServlet):
     def on_POST(self, request):
         body = parse_json_dict_from_request(request)
 
-        authed, result = yield self.auth_handler.check_auth([
+        authed, result, params = yield self.auth_handler.check_auth([
             [LoginType.PASSWORD]
         ], body)
 
@@ -61,9 +61,9 @@ class PasswordRestServlet(RestServlet):
 
         user_id = auth_user.to_string()
 
-        if 'new_password' not in body:
+        if 'new_password' not in params:
             raise SynapseError(400, "", Codes.MISSING_PARAM)
-        new_password = body['new_password']
+        new_password = params['new_password']
 
         yield self.login_handler.set_password(
             user_id, new_password, client.token_id
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 72319a3bb2..d7a20fc964 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -74,7 +74,7 @@ class RegisterRestServlet(RestServlet):
             )
             is_using_shared_secret = True
         else:
-            authed, result = yield self.auth_handler.check_auth([
+            authed, result, params = yield self.auth_handler.check_auth([
                 [LoginType.RECAPTCHA],
                 [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
             ], body, self.hs.get_ip_from_request(request))
@@ -90,10 +90,10 @@ class RegisterRestServlet(RestServlet):
         if not can_register:
             raise SynapseError(403, "Registration has been disabled")
 
-        if 'username' not in body or 'password' not in body:
+        if 'username' not in params or 'password' not in params:
             raise SynapseError(400, "", Codes.MISSING_PARAM)
-        desired_username = body['username']
-        new_password = body['password']
+        desired_username = params['username']
+        new_password = params['password']
 
         (user_id, token) = yield self.registration_handler.register(
             localpart=desired_username,
-- 
cgit 1.5.1


From 766bd8e88077cbeabffc353d9735a3af190abe61 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 15 Apr 2015 17:14:25 +0100
Subject: Dummy login so we can do the first POST request to get login flows
 without it just succeeding

---
 synapse/api/constants.py                 |  1 +
 synapse/handlers/auth.py                 |  6 ++++++
 synapse/handlers/identity.py             |  6 +++---
 synapse/rest/client/v2_alpha/register.py | 18 ++++++++++++++----
 4 files changed, 24 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index d29c2dde01..d8a18ee87b 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -59,6 +59,7 @@ class LoginType(object):
     EMAIL_URL = u"m.login.email.url"
     EMAIL_IDENTITY = u"m.login.email.identity"
     RECAPTCHA = u"m.login.recaptcha"
+    DUMMY = u"m.login.dummy"
 
     # Only for C/S API v1
     APPLICATION_SERVICE = u"m.login.application_service"
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 2cc54707a2..87866f298d 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -42,6 +42,7 @@ class AuthHandler(BaseHandler):
             LoginType.PASSWORD: self._check_password_auth,
             LoginType.RECAPTCHA: self._check_recaptcha,
             LoginType.EMAIL_IDENTITY: self._check_email_identity,
+            LoginType.DUMMY: self._check_dummy_auth,
         }
         self.sessions = {}
 
@@ -202,6 +203,11 @@ class AuthHandler(BaseHandler):
 
         defer.returnValue(threepid)
 
+    @defer.inlineCallbacks
+    def _check_dummy_auth(self, authdict, _):
+        yield run_on_reactor()
+        defer.returnValue(True)
+
     def _get_params_recaptcha(self):
         return {"public_key": self.hs.config.recaptcha_public_key}
 
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 671d366e40..19896ce90d 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -42,8 +42,8 @@ class IdentityHandler(BaseHandler):
         # each request
         http_client = SimpleHttpClient(self.hs)
         # XXX: make this configurable!
-        #trustedIdServers = ['matrix.org', 'localhost:8090']
-        trustedIdServers = ['matrix.org']
+        trustedIdServers = ['matrix.org', 'localhost:8090']
+        #trustedIdServers = ['matrix.org']
         if not creds['idServer'] in trustedIdServers:
             logger.warn('%s is not a trusted ID server: rejecting 3pid ' +
                         'credentials', creds['idServer'])
@@ -52,7 +52,7 @@ class IdentityHandler(BaseHandler):
         data = {}
         try:
             data = yield http_client.get_json(
-                "https://%s%s" % (
+                "http://%s%s" % (
                     creds['idServer'],
                     "/_matrix/identity/api/v1/3pid/getValidated3pid"
                 ),
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index d7a20fc964..ee99b74fd6 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -63,6 +63,17 @@ class RegisterRestServlet(RestServlet):
         if 'access_token' in request.args:
             service = yield self.auth.get_appservice_by_req(request)
 
+        if self.hs.config.enable_registration_captcha:
+            flows = [
+                [LoginType.RECAPTCHA],
+                [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]
+            ]
+        else:
+            flows = [
+                [LoginType.DUMMY],
+                [LoginType.EMAIL_IDENTITY]
+            ]
+
         if service:
             is_application_server = True
         elif 'mac' in body:
@@ -74,10 +85,9 @@ class RegisterRestServlet(RestServlet):
             )
             is_using_shared_secret = True
         else:
-            authed, result, params = yield self.auth_handler.check_auth([
-                [LoginType.RECAPTCHA],
-                [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
-            ], body, self.hs.get_ip_from_request(request))
+            authed, result, params = yield self.auth_handler.check_auth(
+                flows, body, self.hs.get_ip_from_request(request)
+            )
 
             if not authed:
                 defer.returnValue((401, result))
-- 
cgit 1.5.1


From e1c0970c116fe3700fc80401dc50aeb9d52c45a6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 16 Apr 2015 11:18:45 +0100
Subject: PEP8

---
 synapse/handlers/_base.py          | 1 -
 synapse/storage/engines/sqlite3.py | 2 --
 synapse/storage/pusher.py          | 3 ---
 3 files changed, 6 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index e4471dd9b6..dffb033fbd 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -16,7 +16,6 @@
 from twisted.internet import defer
 
 from synapse.api.errors import LimitExceededError, SynapseError
-from synapse.util.async import run_on_reactor
 from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.api.constants import Membership, EventTypes
 from synapse.types import UserID
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index dd0d8e0e0f..72c11df461 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -15,8 +15,6 @@
 
 from synapse.storage import prepare_database, prepare_sqlite3_database
 
-import types
-
 
 class Sqlite3Engine(object):
     def __init__(self, database_module):
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 5c1c3d32e6..a44bccdca6 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
-
 from ._base import SQLBaseStore, Table
 from twisted.internet import defer
 
@@ -167,4 +165,3 @@ class PusherStore(SQLBaseStore):
 
 class PushersTable(Table):
     table_name = "pushers"
-
-- 
cgit 1.5.1


From ea1776f556edaf6ca483bc5faed5e9d244aa1a15 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 16 Apr 2015 19:56:44 +0100
Subject: Return user ID in use error straight away

---
 synapse/handlers/auth.py                 |   2 +
 synapse/handlers/identity.py             |  25 +++++++-
 synapse/handlers/register.py             | 102 ++++++++++++-------------------
 synapse/rest/client/v2_alpha/register.py |  25 +++++++-
 4 files changed, 88 insertions(+), 66 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 87866f298d..1f927e67ad 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -201,6 +201,8 @@ class AuthHandler(BaseHandler):
         logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,))
         threepid = yield identity_handler.threepid_from_creds(threepidCreds)
 
+        threepid['threepidCreds'] = authdict['threepidCreds']
+
         defer.returnValue(threepid)
 
     @defer.inlineCallbacks
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 19896ce90d..cb5e1e80ac 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -63,4 +63,27 @@ class IdentityHandler(BaseHandler):
 
         if 'medium' in data:
             defer.returnValue(data)
-        defer.returnValue(None)
\ No newline at end of file
+        defer.returnValue(None)
+
+    @defer.inlineCallbacks
+    def bind_threepid(self, creds, mxid):
+        yield run_on_reactor()
+        logger.debug("binding threepid %r to %s", creds, mxid)
+        http_client = SimpleHttpClient(self.hs)
+        data = None
+        try:
+            data = yield http_client.post_urlencoded_get_json(
+                # XXX: Change when ID servers are all HTTPS
+                "http://%s%s" % (
+                    creds['idServer'], "/_matrix/identity/api/v1/3pid/bind"
+                ),
+                {
+                    'sid': creds['sid'],
+                    'clientSecret': creds['clientSecret'],
+                    'mxid': mxid,
+                }
+            )
+            logger.debug("bound threepid %r to %s", creds, mxid)
+        except CodeMessageException as e:
+            data = json.loads(e.msg)
+        defer.returnValue(data)
\ No newline at end of file
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 6759a8c582..541b1019da 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -44,6 +44,36 @@ class RegistrationHandler(BaseHandler):
         self.distributor = hs.get_distributor()
         self.distributor.declare("registered_user")
 
+    @defer.inlineCallbacks
+    def check_username(self, localpart):
+        yield run_on_reactor()
+
+        print "checking username %s" % (localpart)
+
+        if urllib.quote(localpart) != localpart:
+            raise SynapseError(
+                400,
+                "User ID must only contain characters which do not"
+                " require URL encoding."
+            )
+
+        user = UserID(localpart, self.hs.hostname)
+        user_id = user.to_string()
+
+        yield self.check_user_id_is_valid(user_id)
+
+        print "is valid"
+
+        u = yield self.store.get_user_by_id(user_id)
+        print "user is: "
+        print u
+        if u:
+            raise SynapseError(
+                400,
+                "User ID already taken.",
+                errcode=Codes.USER_IN_USE,
+            )
+
     @defer.inlineCallbacks
     def register(self, localpart=None, password=None):
         """Registers a new client on the server.
@@ -64,18 +94,11 @@ class RegistrationHandler(BaseHandler):
             password_hash = bcrypt.hashpw(password, bcrypt.gensalt())
 
         if localpart:
-            if localpart and urllib.quote(localpart) != localpart:
-                raise SynapseError(
-                    400,
-                    "User ID must only contain characters which do not"
-                    " require URL encoding."
-                )
+            self.check_username(localpart)
 
             user = UserID(localpart, self.hs.hostname)
             user_id = user.to_string()
 
-            yield self.check_user_id_is_valid(user_id)
-
             token = self._generate_token(user_id)
             yield self.store.register(
                 user_id=user_id,
@@ -190,7 +213,8 @@ class RegistrationHandler(BaseHandler):
             logger.info("validating theeepidcred sid %s on id server %s",
                         c['sid'], c['idServer'])
             try:
-                threepid = yield self._threepid_from_creds(c)
+                identity_handler = self.hs.get_handlers().identity_handler
+                threepid = yield identity_handler.threepid_from_creds(c)
             except:
                 logger.exception("Couldn't validate 3pid")
                 raise RegistrationError(400, "Couldn't validate 3pid")
@@ -202,12 +226,16 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def bind_emails(self, user_id, threepidCreds):
-        """Links emails with a user ID and informs an identity server."""
+        """Links emails with a user ID and informs an identity server.
+
+        Used only by c/s api v1
+        """
 
         # Now we have a matrix ID, bind it to the threepids we were given
         for c in threepidCreds:
+            identity_handler = self.hs.get_handlers().identity_handler
             # XXX: This should be a deferred list, shouldn't it?
-            yield self._bind_threepid(c, user_id)
+            yield identity_handler.bind_threepid(c, user_id)
 
     @defer.inlineCallbacks
     def check_user_id_is_valid(self, user_id):
@@ -234,58 +262,6 @@ class RegistrationHandler(BaseHandler):
     def _generate_user_id(self):
         return "-" + stringutils.random_string(18)
 
-    @defer.inlineCallbacks
-    def _threepid_from_creds(self, creds):
-        # TODO: get this from the homeserver rather than creating a new one for
-        # each request
-        http_client = SimpleHttpClient(self.hs)
-        # XXX: make this configurable!
-        trustedIdServers = ['matrix.org:8090', 'matrix.org']
-        if not creds['idServer'] in trustedIdServers:
-            logger.warn('%s is not a trusted ID server: rejecting 3pid ' +
-                        'credentials', creds['idServer'])
-            defer.returnValue(None)
-
-        data = {}
-        try:
-            data = yield http_client.get_json(
-                # XXX: This should be HTTPS
-                "http://%s%s" % (
-                    creds['idServer'],
-                    "/_matrix/identity/api/v1/3pid/getValidated3pid"
-                ),
-                {'sid': creds['sid'], 'clientSecret': creds['clientSecret']}
-            )
-        except CodeMessageException as e:
-            data = json.loads(e.msg)
-
-        if 'medium' in data:
-            defer.returnValue(data)
-        defer.returnValue(None)
-
-    @defer.inlineCallbacks
-    def _bind_threepid(self, creds, mxid):
-        yield
-        logger.debug("binding threepid")
-        http_client = SimpleHttpClient(self.hs)
-        data = None
-        try:
-            data = yield http_client.post_urlencoded_get_json(
-                # XXX: Change when ID servers are all HTTPS
-                "http://%s%s" % (
-                    creds['idServer'], "/_matrix/identity/api/v1/3pid/bind"
-                ),
-                {
-                    'sid': creds['sid'],
-                    'clientSecret': creds['clientSecret'],
-                    'mxid': mxid,
-                }
-            )
-            logger.debug("bound threepid")
-        except CodeMessageException as e:
-            data = json.loads(e.msg)
-        defer.returnValue(data)
-
     @defer.inlineCallbacks
     def _validate_captcha(self, ip_addr, private_key, challenge, response):
         """Validates the captcha provided.
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index ee99b74fd6..a5fec45dce 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -49,12 +49,20 @@ class RegisterRestServlet(RestServlet):
         self.auth = hs.get_auth()
         self.auth_handler = hs.get_handlers().auth_handler
         self.registration_handler = hs.get_handlers().registration_handler
+        self.identity_handler = hs.get_handlers().identity_handler
 
     @defer.inlineCallbacks
     def on_POST(self, request):
         yield run_on_reactor()
 
         body = parse_request_allow_empty(request)
+        if 'password' not in body:
+            raise SynapseError(400, "", Codes.MISSING_PARAM)
+
+        if 'username' in body:
+            desired_username = body['username']
+            print "username in body"
+            yield self.registration_handler.check_username(desired_username)
 
         is_using_shared_secret = False
         is_application_server = False
@@ -100,15 +108,28 @@ class RegisterRestServlet(RestServlet):
         if not can_register:
             raise SynapseError(403, "Registration has been disabled")
 
-        if 'username' not in params or 'password' not in params:
+        if 'password' not in params:
             raise SynapseError(400, "", Codes.MISSING_PARAM)
-        desired_username = params['username']
+        desired_username = params['username'] if 'username' in params else None
         new_password = params['password']
 
         (user_id, token) = yield self.registration_handler.register(
             localpart=desired_username,
             password=new_password
         )
+
+        if 'bind_email' in params and params['bind_email']:
+            logger.info("bind_email specified: binding")
+
+            emailThreepid = result[LoginType.EMAIL_IDENTITY]
+            threepidCreds = emailThreepid['threepidCreds']
+            logger.debug("Binding emails %s to %s" % (
+                emailThreepid, user_id
+            ))
+            yield self.identity_handler.bind_threepid(threepidCreds, user_id)
+        else:
+            logger.info("bind_email not specified: not binding email")
+
         result = {
             "user_id": user_id,
             "access_token": token,
-- 
cgit 1.5.1


From 4cd5fb13a31a4da1d7d8feb06d211a2f7842f5ad Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 16 Apr 2015 20:03:13 +0100
Subject: Oops, left debugging in.

---
 synapse/handlers/register.py | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 541b1019da..25b1db62ea 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -48,8 +48,6 @@ class RegistrationHandler(BaseHandler):
     def check_username(self, localpart):
         yield run_on_reactor()
 
-        print "checking username %s" % (localpart)
-
         if urllib.quote(localpart) != localpart:
             raise SynapseError(
                 400,
@@ -62,11 +60,7 @@ class RegistrationHandler(BaseHandler):
 
         yield self.check_user_id_is_valid(user_id)
 
-        print "is valid"
-
         u = yield self.store.get_user_by_id(user_id)
-        print "user is: "
-        print u
         if u:
             raise SynapseError(
                 400,
-- 
cgit 1.5.1


From 83b554437ec9810dd09de992c728c2a2f01aa0e1 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 17 Apr 2015 12:57:25 +0100
Subject: Need to yield the username check, otherwise very very weird things
 happen.

---
 synapse/handlers/register.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 25b1db62ea..d4483c3a1d 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -88,7 +88,7 @@ class RegistrationHandler(BaseHandler):
             password_hash = bcrypt.hashpw(password, bcrypt.gensalt())
 
         if localpart:
-            self.check_username(localpart)
+            yield self.check_username(localpart)
 
             user = UserID(localpart, self.hs.hostname)
             user_id = user.to_string()
-- 
cgit 1.5.1


From f96ab9d18dcebf995700f096792101a490b3a9b8 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 17 Apr 2015 16:44:49 +0100
Subject: make add3pid servlet work

---
 synapse/handlers/login.py                |  7 ++++++
 synapse/rest/client/v2_alpha/account.py  | 38 ++++++++++++++++++++++++++++++++
 synapse/rest/client/v2_alpha/register.py | 38 ++++++++++++++++++++++----------
 synapse/storage/registration.py          | 11 +++++++++
 4 files changed, 82 insertions(+), 12 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 04f6dbb95e..5c39356d71 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -74,3 +74,10 @@ class LoginHandler(BaseHandler):
             user_id, token_id
         )
         yield self.store.flush_user(user_id)
+
+    @defer.inlineCallbacks
+    def add_threepid(self, user_id, medium, address, validated_at):
+        yield self.store.user_add_threepid(
+            user_id, medium, address, validated_at,
+            self.hs.get_clock().time_msec()
+        )
\ No newline at end of file
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 750d826f91..6045b016ef 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -18,6 +18,7 @@ from twisted.internet import defer
 from synapse.api.constants import LoginType
 from synapse.api.errors import LoginError, SynapseError, Codes
 from synapse.http.servlet import RestServlet
+from synapse.util.async import run_on_reactor
 
 from ._base import client_v2_pattern, parse_json_dict_from_request
 
@@ -39,6 +40,8 @@ class PasswordRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
+        yield run_on_reactor()
+
         body = parse_json_dict_from_request(request)
 
         authed, result, params = yield self.auth_handler.check_auth([
@@ -78,16 +81,51 @@ class PasswordRestServlet(RestServlet):
 class ThreepidRestServlet(RestServlet):
     PATTERN = client_v2_pattern("/account/3pid")
 
+    def __init__(self, hs):
+        super(ThreepidRestServlet, self).__init__()
+        self.hs = hs
+        self.login_handler = hs.get_handlers().login_handler
+        self.identity_handler = hs.get_handlers().identity_handler
+        self.auth = hs.get_auth()
+
     @defer.inlineCallbacks
     def on_POST(self, request):
+        yield run_on_reactor()
+
         body = parse_json_dict_from_request(request)
 
         if 'threePidCreds' not in body:
             raise SynapseError(400, "Missing param", Codes.MISSING_PARAM)
+        threePidCreds = body['threePidCreds']
 
         auth_user, client = yield self.auth.get_user_by_req(request)
 
+        threepid = yield self.identity_handler.threepid_from_creds(threePidCreds)
 
+        if not threepid:
+            raise SynapseError(400, "Failed to auth 3pid")
+
+        for reqd in ['medium', 'address', 'validatedAt']:
+            if reqd not in threepid:
+                logger.warn("Couldn't add 3pid: invalid response from ID sevrer")
+                raise SynapseError(500, "Invalid response from ID Server")
+
+        yield self.login_handler.add_threepid(
+            auth_user.to_string(),
+            threepid['medium'],
+            threepid['address'],
+            threepid['validatedAt'],
+        )
+
+        if 'bind' in body and body['bind']:
+            logger.debug("Binding emails %s to %s" % (
+                threepid, auth_user.to_string()
+            ))
+            yield self.identity_handler.bind_threepid(
+                threePidCreds, auth_user.to_string()
+            )
+
+        defer.returnValue((200, {}))
 
 
 def register_servlets(hs, http_server):
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index a5fec45dce..e93897e285 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -50,6 +50,7 @@ class RegisterRestServlet(RestServlet):
         self.auth_handler = hs.get_handlers().auth_handler
         self.registration_handler = hs.get_handlers().registration_handler
         self.identity_handler = hs.get_handlers().identity_handler
+        self.login_handler = hs.get_handlers().login_handler
 
     @defer.inlineCallbacks
     def on_POST(self, request):
@@ -61,7 +62,6 @@ class RegisterRestServlet(RestServlet):
 
         if 'username' in body:
             desired_username = body['username']
-            print "username in body"
             yield self.registration_handler.check_username(desired_username)
 
         is_using_shared_secret = False
@@ -118,17 +118,31 @@ class RegisterRestServlet(RestServlet):
             password=new_password
         )
 
-        if 'bind_email' in params and params['bind_email']:
-            logger.info("bind_email specified: binding")
-
-            emailThreepid = result[LoginType.EMAIL_IDENTITY]
-            threepidCreds = emailThreepid['threepidCreds']
-            logger.debug("Binding emails %s to %s" % (
-                emailThreepid, user_id
-            ))
-            yield self.identity_handler.bind_threepid(threepidCreds, user_id)
-        else:
-            logger.info("bind_email not specified: not binding email")
+        if LoginType.EMAIL_IDENTITY in result:
+            threepid = result[LoginType.EMAIL_IDENTITY]
+
+            for reqd in ['medium', 'address', 'validatedAt']:
+                if reqd not in threepid:
+                    logger.info("Can't add incomplete 3pid")
+                else:
+                    yield self.login_handler.add_threepid(
+                        user_id,
+                        threepid['medium'],
+                        threepid['address'],
+                        threepid['validatedAt'],
+                    )
+
+            if 'bind_email' in params and params['bind_email']:
+                logger.info("bind_email specified: binding")
+
+                emailThreepid = result[LoginType.EMAIL_IDENTITY]
+                threepidCreds = emailThreepid['threepidCreds']
+                logger.debug("Binding emails %s to %s" % (
+                    emailThreepid, user_id
+                ))
+                yield self.identity_handler.bind_threepid(threepidCreds, user_id)
+            else:
+                logger.info("bind_email not specified: not binding email")
 
         result = {
             "user_id": user_id,
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index f61d8fdb6a..4bc01f3cc2 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -175,3 +175,14 @@ class RegistrationStore(SQLBaseStore):
             return rows[0]
 
         return None
+
+    @defer.inlineCallbacks
+    def user_add_threepid(self, user_id, medium, address, validated_at, added_at):
+        yield self._simple_upsert("user_threepids", {
+            "user": user_id,
+            "medium": medium,
+            "address": address,
+        }, {
+            "validated_at": validated_at,
+            "added_at": added_at,
+        })
\ No newline at end of file
-- 
cgit 1.5.1


From 4eea5cf6c2a301938466828b02557d8500197bb3 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 17 Apr 2015 16:46:45 +0100
Subject: pep8

---
 synapse/handlers/identity.py    | 6 +++---
 synapse/handlers/login.py       | 2 +-
 synapse/handlers/register.py    | 5 +----
 synapse/storage/registration.py | 2 +-
 4 files changed, 6 insertions(+), 9 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index cb5e1e80ac..5c72635915 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -42,8 +42,8 @@ class IdentityHandler(BaseHandler):
         # each request
         http_client = SimpleHttpClient(self.hs)
         # XXX: make this configurable!
-        trustedIdServers = ['matrix.org', 'localhost:8090']
-        #trustedIdServers = ['matrix.org']
+        # trustedIdServers = ['matrix.org', 'localhost:8090']
+        trustedIdServers = ['matrix.org']
         if not creds['idServer'] in trustedIdServers:
             logger.warn('%s is not a trusted ID server: rejecting 3pid ' +
                         'credentials', creds['idServer'])
@@ -86,4 +86,4 @@ class IdentityHandler(BaseHandler):
             logger.debug("bound threepid %r to %s", creds, mxid)
         except CodeMessageException as e:
             data = json.loads(e.msg)
-        defer.returnValue(data)
\ No newline at end of file
+        defer.returnValue(data)
diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 5c39356d71..f7f3698340 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -80,4 +80,4 @@ class LoginHandler(BaseHandler):
         yield self.store.user_add_threepid(
             user_id, medium, address, validated_at,
             self.hs.get_clock().time_msec()
-        )
\ No newline at end of file
+        )
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index d4483c3a1d..7b68585a17 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -18,18 +18,15 @@ from twisted.internet import defer
 
 from synapse.types import UserID
 from synapse.api.errors import (
-    AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError,
-    CodeMessageException
+    AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError
 )
 from ._base import BaseHandler
 import synapse.util.stringutils as stringutils
 from synapse.util.async import run_on_reactor
-from synapse.http.client import SimpleHttpClient
 from synapse.http.client import CaptchaServerHttpClient
 
 import base64
 import bcrypt
-import json
 import logging
 import urllib
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 4bc01f3cc2..8f62e5c6f2 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -185,4 +185,4 @@ class RegistrationStore(SQLBaseStore):
         }, {
             "validated_at": validated_at,
             "added_at": added_at,
-        })
\ No newline at end of file
+        })
-- 
cgit 1.5.1


From 8db6832db8a8ad1a68ff6781b90f3e2cb1a72fc0 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 17 Apr 2015 19:53:47 +0100
Subject: Password reset, finally.

---
 synapse/handlers/auth.py                |  8 +++++++-
 synapse/rest/client/v2_alpha/account.py | 21 ++++++++++++++++-----
 synapse/storage/registration.py         | 16 +++++++++++++++-
 3 files changed, 38 insertions(+), 7 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 1f927e67ad..7b0ab4829b 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -195,12 +195,18 @@ class AuthHandler(BaseHandler):
     def _check_email_identity(self, authdict, _):
         yield run_on_reactor()
 
+        if 'threepidCreds' not in authdict:
+            raise LoginError(400, "Missing threepidCreds", Codes.MISSING_PARAM)
+
         threepidCreds = authdict['threepidCreds']
         identity_handler = self.hs.get_handlers().identity_handler
 
-        logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,))
+        logger.info("Getting validated threepid. threepidcreds: %r" % (threepidCreds,))
         threepid = yield identity_handler.threepid_from_creds(threepidCreds)
 
+        if not threepid:
+            raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
+
         threepid['threepidCreds'] = authdict['threepidCreds']
 
         defer.returnValue(threepid)
diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py
index 5ac3ac0f71..e33607b799 100644
--- a/synapse/rest/client/v2_alpha/account.py
+++ b/synapse/rest/client/v2_alpha/account.py
@@ -45,31 +45,42 @@ class PasswordRestServlet(RestServlet):
         body = parse_json_dict_from_request(request)
 
         authed, result, params = yield self.auth_handler.check_auth([
-            [LoginType.PASSWORD]
+            [LoginType.PASSWORD],
+            [LoginType.EMAIL_IDENTITY]
         ], body)
 
         if not authed:
             defer.returnValue((401, result))
 
-        auth_user = None
+        user_id = None
 
         if LoginType.PASSWORD in result:
             # if using password, they should also be logged in
             auth_user, client = yield self.auth.get_user_by_req(request)
             if auth_user.to_string() != result[LoginType.PASSWORD]:
                 raise LoginError(400, "", Codes.UNKNOWN)
+            user_id = auth_user.to_string()
+        elif LoginType.EMAIL_IDENTITY in result:
+            threepid = result[LoginType.EMAIL_IDENTITY]
+            if 'medium' not in threepid or 'address' not in threepid:
+                raise SynapseError(500, "Malformed threepid")
+            # if using email, we must know about the email they're authing with!
+            threepid_user = yield self.hs.get_datastore().get_user_by_threepid(
+                threepid['medium'], threepid['address']
+            )
+            if not threepid_user:
+                raise SynapseError(404, "Email address not found", Codes.NOT_FOUND)
+            user_id = threepid_user
         else:
             logger.error("Auth succeeded but no known type!", result.keys())
             raise SynapseError(500, "", Codes.UNKNOWN)
 
-        user_id = auth_user.to_string()
-
         if 'new_password' not in params:
             raise SynapseError(400, "", Codes.MISSING_PARAM)
         new_password = params['new_password']
 
         yield self.login_handler.set_password(
-            user_id, new_password, client.token_id
+            user_id, new_password, None
         )
 
         defer.returnValue((200, {}))
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 08d60f0817..ab43856023 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -196,4 +196,18 @@ class RegistrationStore(SQLBaseStore):
             ['medium', 'address', 'validated_at', 'added_at'],
             'user_get_threepids'
         )
-        defer.returnValue(ret)
\ No newline at end of file
+        defer.returnValue(ret)
+
+    @defer.inlineCallbacks
+    def get_user_by_threepid(self, medium, address):
+        ret = yield self._simple_select_one(
+            "user_threepids",
+            {
+                "medium": medium,
+                "address": address
+            },
+            ['user'], True, 'get_user_by_threepid'
+        )
+        if ret:
+            defer.returnValue(ret['user'])
+        defer.returnValue(None)
\ No newline at end of file
-- 
cgit 1.5.1


From 48b6ee2b67ad4f8bce9774267f863e45192f67d4 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Tue, 21 Apr 2015 21:07:35 +0100
Subject: Create an 'invite' powerlevel when making new rooms

---
 synapse/handlers/room.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index f9fc4a9c98..1226b23bc7 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -213,7 +213,8 @@ class RoomCreationHandler(BaseHandler):
                 "state_default": 50,
                 "ban": 50,
                 "kick": 50,
-                "redact": 50
+                "redact": 50,
+                "invite": 0,
             },
         )
 
-- 
cgit 1.5.1


From 0eb61a3d16bffa83b0963418fa17a8cf6c760631 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Thu, 23 Apr 2015 14:44:12 +0100
Subject: Remove ultimately unused feature of saving params from the first call
 in the session: it's probably too open to abuse.

---
 synapse/handlers/auth.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 7b0ab4829b..ac07add2f7 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -78,8 +78,16 @@ class AuthHandler(BaseHandler):
         sess = self._get_session_info(sid)
 
         if len(clientdict) > 0:
-            sess['clientdict'] = clientdict
-            self._save_session(sess)
+            # This was designed to allow the client to omit the parameters
+            # and just supply the session in subsequent calls so it split
+            # auth between devices by just sharing the session, (eg. so you
+            # could continue registration from your phone having clicked the
+            # email auth link on there). It's probably too open to abuse
+            # because it lets unauthenticated clients store arbitrary objects
+            # on a home server.
+            #sess['clientdict'] = clientdict
+            #self._save_session(sess)
+            pass
         elif 'clientdict' in sess:
             clientdict = sess['clientdict']
 
-- 
cgit 1.5.1


From f7a79a37beb6bbb217b53a1d8d93a33cf577e6ac Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 24 Apr 2015 09:42:37 +0100
Subject: pep8

---
 synapse/handlers/auth.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index ac07add2f7..34d7080fab 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -85,8 +85,8 @@ class AuthHandler(BaseHandler):
             # email auth link on there). It's probably too open to abuse
             # because it lets unauthenticated clients store arbitrary objects
             # on a home server.
-            #sess['clientdict'] = clientdict
-            #self._save_session(sess)
+            # sess['clientdict'] = clientdict
+            # self._save_session(sess)
             pass
         elif 'clientdict' in sess:
             clientdict = sess['clientdict']
-- 
cgit 1.5.1


From a21861962608726a5fe443762421c80119517778 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 24 Apr 2015 11:27:38 +0100
Subject: Use underscores instead of camelcase for id server stuff

---
 synapse/handlers/auth.py                 | 12 ++++++------
 synapse/handlers/identity.py             | 12 ++++++------
 synapse/rest/client/v2_alpha/register.py |  4 ++--
 3 files changed, 14 insertions(+), 14 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 34d7080fab..ef3219b38e 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -203,19 +203,19 @@ class AuthHandler(BaseHandler):
     def _check_email_identity(self, authdict, _):
         yield run_on_reactor()
 
-        if 'threepidCreds' not in authdict:
-            raise LoginError(400, "Missing threepidCreds", Codes.MISSING_PARAM)
+        if 'threepid_creds' not in authdict:
+            raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM)
 
-        threepidCreds = authdict['threepidCreds']
+        threepid_creds = authdict['threepid_creds']
         identity_handler = self.hs.get_handlers().identity_handler
 
-        logger.info("Getting validated threepid. threepidcreds: %r" % (threepidCreds,))
-        threepid = yield identity_handler.threepid_from_creds(threepidCreds)
+        logger.info("Getting validated threepid. threepidcreds: %r" % (threepid_creds,))
+        threepid = yield identity_handler.threepid_from_creds(threepid_creds)
 
         if not threepid:
             raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
-        threepid['threepidCreds'] = authdict['threepidCreds']
+        threepid['threepid_creds'] = authdict['threepid_creds']
 
         defer.returnValue(threepid)
 
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 5c72635915..3ddd834c61 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -44,19 +44,19 @@ class IdentityHandler(BaseHandler):
         # XXX: make this configurable!
         # trustedIdServers = ['matrix.org', 'localhost:8090']
         trustedIdServers = ['matrix.org']
-        if not creds['idServer'] in trustedIdServers:
+        if not creds['id_server'] in trustedIdServers:
             logger.warn('%s is not a trusted ID server: rejecting 3pid ' +
-                        'credentials', creds['idServer'])
+                        'credentials', creds['id_server'])
             defer.returnValue(None)
 
         data = {}
         try:
             data = yield http_client.get_json(
                 "http://%s%s" % (
-                    creds['idServer'],
+                    creds['id_server'],
                     "/_matrix/identity/api/v1/3pid/getValidated3pid"
                 ),
-                {'sid': creds['sid'], 'clientSecret': creds['clientSecret']}
+                {'sid': creds['sid'], 'client_secret': creds['client_secret']}
             )
         except CodeMessageException as e:
             data = json.loads(e.msg)
@@ -75,11 +75,11 @@ class IdentityHandler(BaseHandler):
             data = yield http_client.post_urlencoded_get_json(
                 # XXX: Change when ID servers are all HTTPS
                 "http://%s%s" % (
-                    creds['idServer'], "/_matrix/identity/api/v1/3pid/bind"
+                    creds['id_server'], "/_matrix/identity/api/v1/3pid/bind"
                 ),
                 {
                     'sid': creds['sid'],
-                    'clientSecret': creds['clientSecret'],
+                    'client_secret': creds['client_secret'],
                     'mxid': mxid,
                 }
             )
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index e93897e285..dd176c7e77 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -136,11 +136,11 @@ class RegisterRestServlet(RestServlet):
                 logger.info("bind_email specified: binding")
 
                 emailThreepid = result[LoginType.EMAIL_IDENTITY]
-                threepidCreds = emailThreepid['threepidCreds']
+                threepid_creds = emailThreepid['threepid_creds']
                 logger.debug("Binding emails %s to %s" % (
                     emailThreepid, user_id
                 ))
-                yield self.identity_handler.bind_threepid(threepidCreds, user_id)
+                yield self.identity_handler.bind_threepid(threepid_creds, user_id)
             else:
                 logger.info("bind_email not specified: not binding email")
 
-- 
cgit 1.5.1


From 1bac74b9aea46f9e46152955ecf06d8cc7eacdd3 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Fri, 24 Apr 2015 14:48:49 +0100
Subject: Change to https for ID server communication

---
 synapse/handlers/identity.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 3ddd834c61..ad8246b58c 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -52,7 +52,7 @@ class IdentityHandler(BaseHandler):
         data = {}
         try:
             data = yield http_client.get_json(
-                "http://%s%s" % (
+                "https://%s%s" % (
                     creds['id_server'],
                     "/_matrix/identity/api/v1/3pid/getValidated3pid"
                 ),
@@ -73,8 +73,7 @@ class IdentityHandler(BaseHandler):
         data = None
         try:
             data = yield http_client.post_urlencoded_get_json(
-                # XXX: Change when ID servers are all HTTPS
-                "http://%s%s" % (
+                "https://%s%s" % (
                     creds['id_server'], "/_matrix/identity/api/v1/3pid/bind"
                 ),
                 {
-- 
cgit 1.5.1


From 412ece18e7edb87053a3684e49d5dd485f88a65d Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 27 Apr 2015 14:08:45 +0100
Subject: Add commentage.

---
 synapse/handlers/auth.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index ef3219b38e..2e8009d3c3 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -121,6 +121,10 @@ class AuthHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def add_oob_auth(self, stagetype, authdict, clientip):
+        """
+        Adds the result of out-of-band authentication into an existing auth
+        session. Currently used for adding the result of fallback auth.
+        """
         if stagetype not in self.checkers:
             raise LoginError(400, "", Codes.MISSING_PARAM)
         if 'session' not in authdict:
-- 
cgit 1.5.1


From d98edb548af8833a7c44eded610a528e3d0515c6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 28 Apr 2015 17:20:32 +0100
Subject: Ensure the serial returned by presence is always an integer

---
 synapse/handlers/presence.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 571eacd343..774df46aba 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -859,7 +859,7 @@ class PresenceEventSource(object):
         presence = self.hs.get_handlers().presence_handler
         cachemap = presence._user_cachemap
         clock = self.clock
-        latest_serial = None
+        latest_serial = 0
 
         updates = []
         # TODO(paul): use a DeferredList ? How to limit concurrency.
@@ -872,8 +872,7 @@ class PresenceEventSource(object):
             if not (yield self.is_visible(observer_user, observed_user)):
                 continue
 
-            if latest_serial is None or cached.serial > latest_serial:
-                latest_serial = cached.serial
+            latest_serial = max(cached.serial, latest_serial)
             updates.append(cached.make_event(user=observed_user, clock=clock))
 
         # TODO(paul): limit
@@ -882,6 +881,7 @@ class PresenceEventSource(object):
             if serial < from_key:
                 break
 
+            latest_serial = max(cached.serial, serial)
             for u in user_ids:
                 updates.append({
                     "type": "m.presence",
-- 
cgit 1.5.1


From 0126ef7f3c1304bd920260db369628d39b4badd3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 28 Apr 2015 17:23:53 +0100
Subject: Fix typo

---
 synapse/handlers/presence.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 774df46aba..47cfe62c82 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -881,7 +881,7 @@ class PresenceEventSource(object):
             if serial < from_key:
                 break
 
-            latest_serial = max(cached.serial, serial)
+            latest_serial = max(latest_serial, serial)
             for u in user_ids:
                 updates.append({
                     "type": "m.presence",
-- 
cgit 1.5.1


From 1783c7ca920cdacb22fa8536af03f42557af9d41 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 28 Apr 2015 17:24:24 +0100
Subject: Ensure we never miss any presence updates

---
 synapse/handlers/presence.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'synapse/handlers')

diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 47cfe62c82..42cd528908 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -858,6 +858,9 @@ class PresenceEventSource(object):
 
         presence = self.hs.get_handlers().presence_handler
         cachemap = presence._user_cachemap
+
+        max_serial = presence._user_cachemap_latest_serial
+
         clock = self.clock
         latest_serial = 0
 
@@ -866,7 +869,7 @@ class PresenceEventSource(object):
         for observed_user in cachemap.keys():
             cached = cachemap[observed_user]
 
-            if cached.serial <= from_key:
+            if cached.serial <= from_key or cached.serial > max_serial:
                 continue
 
             if not (yield self.is_visible(observer_user, observed_user)):
@@ -881,6 +884,9 @@ class PresenceEventSource(object):
             if serial < from_key:
                 break
 
+            if serial > max_serial:
+                continue
+
             latest_serial = max(latest_serial, serial)
             for u in user_ids:
                 updates.append({
-- 
cgit 1.5.1