Merge branch 'develop' of github.com:matrix-org/synapse into erikj/search

13 files changed, 393 insertions, 25 deletions

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index e3b8c3099a..5c83aafa7d 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -14,15 +14,19 @@
 # limitations under the License.
 
 """This module contains classes for authenticating the user."""
+from nacl.exceptions import BadSignatureError
 
 from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership, JoinRules
 from synapse.api.errors import AuthError, Codes, SynapseError
-from synapse.util.logutils import log_function
 from synapse.types import RoomID, UserID, EventID
+from synapse.util.logutils import log_function
+from synapse.util import third_party_invites
+from unpaddedbase64 import decode_base64
 
 import logging
+import nacl.signing
 import pymacaroons
 
 logger = logging.getLogger(__name__)
@@ -31,6 +35,7 @@ logger = logging.getLogger(__name__)
 AuthEventTypes = (
     EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels,
     EventTypes.JoinRules, EventTypes.RoomHistoryVisibility,
+    EventTypes.ThirdPartyInvite,
 )
 
 
@@ -341,7 +346,8 @@ class Auth(object):
                 pass
             elif join_rule == JoinRules.INVITE:
                 if not caller_in_room and not caller_invited:
-                    raise AuthError(403, "You are not invited to this room.")
+                    if not self._verify_third_party_invite(event, auth_events):
+                        raise AuthError(403, "You are not invited to this room.")
             else:
                 # TODO (erikj): may_join list
                 # TODO (erikj): private rooms
@@ -367,6 +373,61 @@ class Auth(object):
 
         return True
 
+    def _verify_third_party_invite(self, event, auth_events):
+        """
+        Validates that the join event is authorized by a previous third-party invite.
+
+        Checks that the public key, and keyserver, match those in the invite,
+        and that the join event has a signature issued using that public key.
+
+        Args:
+            event: The m.room.member join event being validated.
+            auth_events: All relevant previous context events which may be used
+                for authorization decisions.
+
+        Return:
+            True if the event fulfills the expectations of a previous third party
+            invite event.
+        """
+        if not third_party_invites.join_has_third_party_invite(event.content):
+            return False
+        join_third_party_invite = event.content["third_party_invite"]
+        token = join_third_party_invite["token"]
+        invite_event = auth_events.get(
+            (EventTypes.ThirdPartyInvite, token,)
+        )
+        if not invite_event:
+            logger.info("Failing 3pid invite because no invite found for token %s", token)
+            return False
+        try:
+            public_key = join_third_party_invite["public_key"]
+            key_validity_url = join_third_party_invite["key_validity_url"]
+            if invite_event.content["public_key"] != public_key:
+                logger.info(
+                    "Failing 3pid invite because public key invite: %s != join: %s",
+                    invite_event.content["public_key"],
+                    public_key
+                )
+                return False
+            if invite_event.content["key_validity_url"] != key_validity_url:
+                logger.info(
+                    "Failing 3pid invite because key_validity_url invite: %s != join: %s",
+                    invite_event.content["key_validity_url"],
+                    key_validity_url
+                )
+                return False
+            for _, signature_block in join_third_party_invite["signatures"].items():
+                for key_name, encoded_signature in signature_block.items():
+                    if not key_name.startswith("ed25519:"):
+                        return False
+                    verify_key = nacl.signing.VerifyKey(decode_base64(public_key))
+                    signature = decode_base64(encoded_signature)
+                    verify_key.verify(token, signature)
+                    return True
+            return False
+        except (KeyError, BadSignatureError,):
+            return False
+
     def _get_power_level_event(self, auth_events):
         key = (EventTypes.PowerLevels, "", )
         return auth_events.get(key)
@@ -646,6 +707,14 @@ class Auth(object):
             if e_type == Membership.JOIN:
                 if member_event and not is_public:
                     auth_ids.append(member_event.event_id)
+                if third_party_invites.join_has_third_party_invite(event.content):
+                    key = (
+                        EventTypes.ThirdPartyInvite,
+                        event.content["third_party_invite"]["token"]
+                    )
+                    invite = current_state.get(key)
+                    if invite:
+                        auth_ids.append(invite.event_id)
             else:
                 if member_event:
                     auth_ids.append(member_event.event_id)
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 008ee64727..41125e8719 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -63,6 +63,7 @@ class EventTypes(object):
     PowerLevels = "m.room.power_levels"
     Aliases = "m.room.aliases"
     Redaction = "m.room.redaction"
+    ThirdPartyInvite = "m.room.third_party_invite"
 
     RoomHistoryVisibility = "m.room.history_visibility"
     CanonicalAlias = "m.room.canonical_alias"
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index f5e346cdbc..f5b430e046 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,6 +25,7 @@ from synapse.api.errors import (
 from synapse.util import unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.logutils import log_function
+from synapse.util import third_party_invites
 from synapse.events import FrozenEvent
 import synapse.metrics
 
@@ -356,14 +357,19 @@ class FederationClient(FederationBase):
         defer.returnValue(signed_auth)
 
     @defer.inlineCallbacks
-    def make_join(self, destinations, room_id, user_id):
+    def make_join(self, destinations, room_id, user_id, content):
         for destination in destinations:
             if destination == self.server_name:
                 continue
 
+            args = {}
+            if third_party_invites.join_has_third_party_invite(content):
+                args = third_party_invites.extract_join_keys(
+                    content["third_party_invite"]
+                )
             try:
                 ret = yield self.transport_layer.make_join(
-                    destination, room_id, user_id
+                    destination, room_id, user_id, args
                 )
 
                 pdu_dict = ret["event"]
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 725c6f3fa5..7934f740e0 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -23,10 +23,12 @@ from synapse.util.logutils import log_function
 from synapse.events import FrozenEvent
 import synapse.metrics
 
-from synapse.api.errors import FederationError, SynapseError
+from synapse.api.errors import FederationError, SynapseError, Codes
 
 from synapse.crypto.event_signing import compute_event_signature
 
+from synapse.util import third_party_invites
+
 import simplejson as json
 import logging
 
@@ -228,8 +230,19 @@ class FederationServer(FederationBase):
             )
 
     @defer.inlineCallbacks
-    def on_make_join_request(self, room_id, user_id):
-        pdu = yield self.handler.on_make_join_request(room_id, user_id)
+    def on_make_join_request(self, room_id, user_id, query):
+        threepid_details = {}
+        if third_party_invites.has_join_keys(query):
+            for k in third_party_invites.JOIN_KEYS:
+                if not isinstance(query[k], list) or len(query[k]) != 1:
+                    raise FederationError(
+                        "FATAL",
+                        Codes.MISSING_PARAM,
+                        "key %s value %s" % (k, query[k],),
+                        None
+                    )
+                threepid_details[k] = query[k][0]
+        pdu = yield self.handler.on_make_join_request(room_id, user_id, threepid_details)
         time_now = self._clock.time_msec()
         defer.returnValue({"event": pdu.get_pdu_json(time_now)})
 
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index ced703364b..ae4195e83a 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -160,13 +160,14 @@ class TransportLayerClient(object):
 
     @defer.inlineCallbacks
     @log_function
-    def make_join(self, destination, room_id, user_id, retry_on_dns_fail=True):
+    def make_join(self, destination, room_id, user_id, args={}):
         path = PREFIX + "/make_join/%s/%s" % (room_id, user_id)
 
         content = yield self.client.get_json(
             destination=destination,
             path=path,
-            retry_on_dns_fail=retry_on_dns_fail,
+            args=args,
+            retry_on_dns_fail=True,
         )
 
         defer.returnValue(content)
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 36f250e1a3..6e394f039e 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -292,7 +292,7 @@ class FederationMakeJoinServlet(BaseFederationServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, origin, content, query, context, user_id):
-        content = yield self.handler.on_make_join_request(context, user_id)
+        content = yield self.handler.on_make_join_request(context, user_id, query)
         defer.returnValue((200, content))
 
 
diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py
index ee2d571329..6a26cb1879 100644
--- a/synapse/handlers/_base.py
+++ b/synapse/handlers/_base.py
@@ -21,6 +21,7 @@ from synapse.api.constants import Membership, EventTypes
 from synapse.types import UserID, RoomAlias
 
 from synapse.util.logcontext import PreserveLoggingContext
+from synapse.util import third_party_invites
 
 import logging
 
@@ -169,6 +170,16 @@ class BaseHandler(object):
                         )
                     )
 
+        if (
+            event.type == EventTypes.Member and
+            event.content["membership"] == Membership.JOIN and
+            third_party_invites.join_has_third_party_invite(event.content)
+        ):
+            yield third_party_invites.check_key_valid(
+                self.hs.get_simple_http_client(),
+                event
+            )
+
         federation_handler = self.hs.get_handlers().federation_handler
 
         if event.type == EventTypes.Member:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a710bdcfdb..946ff97c7d 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -39,7 +39,7 @@ from twisted.internet import defer
 
 import itertools
 import logging
-
+from synapse.util import third_party_invites
 
 logger = logging.getLogger(__name__)
 
@@ -584,7 +584,8 @@ class FederationHandler(BaseHandler):
         origin, pdu = yield self.replication_layer.make_join(
             target_hosts,
             room_id,
-            joinee
+            joinee,
+            content
         )
 
         logger.debug("Got response to make_join: %s", pdu)
@@ -697,14 +698,20 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     @log_function
-    def on_make_join_request(self, room_id, user_id):
+    def on_make_join_request(self, room_id, user_id, query):
         """ We've received a /make_join/ request, so we create a partial
         join event for the room and return that. We don *not* persist or
         process it until the other server has signed it and sent it back.
         """
+        event_content = {"membership": Membership.JOIN}
+        if third_party_invites.has_join_keys(query):
+            event_content["third_party_invite"] = (
+                third_party_invites.extract_join_keys(query)
+            )
+
         builder = self.event_builder_factory.new({
             "type": EventTypes.Member,
-            "content": {"membership": Membership.JOIN},
+            "content": event_content,
             "room_id": room_id,
             "sender": user_id,
             "state_key": user_id,
@@ -716,6 +723,9 @@ class FederationHandler(BaseHandler):
 
         self.auth.check(event, auth_events=context.current_state)
 
+        if third_party_invites.join_has_third_party_invite(event.content):
+            third_party_invites.check_key_valid(self.hs.get_simple_http_client(), event)
+
         defer.returnValue(event)
 
     @defer.inlineCallbacks
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 3364a5de14..3f0cde56f0 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -22,11 +22,16 @@ from synapse.types import UserID, RoomAlias, RoomID
 from synapse.api.constants import (
     EventTypes, Membership, JoinRules, RoomCreationPreset,
 )
-from synapse.api.errors import StoreError, SynapseError
+from synapse.api.errors import AuthError, StoreError, SynapseError
 from synapse.util import stringutils, unwrapFirstError
 from synapse.util.async import run_on_reactor
 
+from signedjson.sign import verify_signed_json
+from signedjson.key import decode_verify_key_bytes
+
 from collections import OrderedDict
+from unpaddedbase64 import decode_base64
+
 import logging
 import string
 
@@ -483,6 +488,13 @@ class RoomMemberHandler(BaseHandler):
 
                 should_do_dance = not self.hs.is_mine(inviter)
                 room_hosts = [inviter.domain]
+            elif "third_party_invite" in event.content:
+                if "sender" in event.content["third_party_invite"]:
+                    inviter = UserID.from_string(
+                        event.content["third_party_invite"]["sender"]
+                    )
+                    should_do_dance = not self.hs.is_mine(inviter)
+                    room_hosts = [inviter.domain]
             else:
                 # return the same error as join_room_alias does
                 raise SynapseError(404, "No known servers")
@@ -540,6 +552,160 @@ class RoomMemberHandler(BaseHandler):
             suppress_auth=(not do_auth),
         )
 
+    @defer.inlineCallbacks
+    def do_3pid_invite(
+            self,
+            room_id,
+            inviter,
+            medium,
+            address,
+            id_server,
+            display_name,
+            token_id,
+            txn_id
+    ):
+        invitee = yield self._lookup_3pid(
+            id_server, medium, address
+        )
+
+        if invitee:
+            # make sure it looks like a user ID; it'll throw if it's invalid.
+            UserID.from_string(invitee)
+            yield self.hs.get_handlers().message_handler.create_and_send_event(
+                {
+                    "type": EventTypes.Member,
+                    "content": {
+                        "membership": unicode("invite")
+                    },
+                    "room_id": room_id,
+                    "sender": inviter.to_string(),
+                    "state_key": invitee,
+                },
+                token_id=token_id,
+                txn_id=txn_id,
+            )
+        else:
+            yield self._make_and_store_3pid_invite(
+                id_server,
+                display_name,
+                medium,
+                address,
+                room_id,
+                inviter,
+                token_id,
+                txn_id=txn_id
+            )
+
+    @defer.inlineCallbacks
+    def _lookup_3pid(self, id_server, medium, address):
+        """Looks up a 3pid in the passed identity server.
+
+        Args:
+            id_server (str): The server name (including port, if required)
+                of the identity server to use.
+            medium (str): The type of the third party identifier (e.g. "email").
+            address (str): The third party identifier (e.g. "foo@example.com").
+
+        Returns:
+            (str) the matrix ID of the 3pid, or None if it is not recognized.
+        """
+        try:
+            data = yield self.hs.get_simple_http_client().get_json(
+                "https://%s/_matrix/identity/api/v1/lookup" % (id_server,),
+                {
+                    "medium": medium,
+                    "address": address,
+                }
+            )
+
+            if "mxid" in data:
+                if "signatures" not in data:
+                    raise AuthError(401, "No signatures on 3pid binding")
+                self.verify_any_signature(data, id_server)
+                defer.returnValue(data["mxid"])
+
+        except IOError as e:
+            logger.warn("Error from identity server lookup: %s" % (e,))
+            defer.returnValue(None)
+
+    @defer.inlineCallbacks
+    def verify_any_signature(self, data, server_hostname):
+        if server_hostname not in data["signatures"]:
+            raise AuthError(401, "No signature from server %s" % (server_hostname,))
+        for key_name, signature in data["signatures"][server_hostname].items():
+            key_data = yield self.hs.get_simple_http_client().get_json(
+                "https://%s/_matrix/identity/api/v1/pubkey/%s" %
+                (server_hostname, key_name,),
+            )
+            if "public_key" not in key_data:
+                raise AuthError(401, "No public key named %s from %s" %
+                                (key_name, server_hostname,))
+            verify_signed_json(
+                data,
+                server_hostname,
+                decode_verify_key_bytes(key_name, decode_base64(key_data["public_key"]))
+            )
+            return
+
+    @defer.inlineCallbacks
+    def _make_and_store_3pid_invite(
+            self,
+            id_server,
+            display_name,
+            medium,
+            address,
+            room_id,
+            user,
+            token_id,
+            txn_id
+    ):
+        token, public_key, key_validity_url = (
+            yield self._ask_id_server_for_third_party_invite(
+                id_server,
+                medium,
+                address,
+                room_id,
+                user.to_string()
+            )
+        )
+        msg_handler = self.hs.get_handlers().message_handler
+        yield msg_handler.create_and_send_event(
+            {
+                "type": EventTypes.ThirdPartyInvite,
+                "content": {
+                    "display_name": display_name,
+                    "key_validity_url": key_validity_url,
+                    "public_key": public_key,
+                },
+                "room_id": room_id,
+                "sender": user.to_string(),
+                "state_key": token,
+            },
+            token_id=token_id,
+            txn_id=txn_id,
+        )
+
+    @defer.inlineCallbacks
+    def _ask_id_server_for_third_party_invite(
+            self, id_server, medium, address, room_id, sender):
+        is_url = "https://%s/_matrix/identity/api/v1/store-invite" % (id_server,)
+        data = yield self.hs.get_simple_http_client().post_urlencoded_get_json(
+            is_url,
+            {
+                "medium": medium,
+                "address": address,
+                "room_id": room_id,
+                "sender": sender,
+            }
+        )
+        # TODO: Check for success
+        token = data["token"]
+        public_key = data["public_key"]
+        key_validity_url = "https://%s/_matrix/identity/api/v1/pubkey/isvalid" % (
+            id_server,
+        )
+        defer.returnValue((token, public_key, key_validity_url))
+
 
 class RoomListHandler(BaseHandler):
 
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 9a5869abee..27e5190224 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -24,7 +24,6 @@ from canonicaljson import encode_canonical_json
 from twisted.internet import defer, reactor, ssl
 from twisted.web.client import (
     Agent, readBody, FileBodyProducer, PartialDownloadError,
-    HTTPConnectionPool,
 )
 from twisted.web.http_headers import Headers
 
@@ -59,11 +58,8 @@ class SimpleHttpClient(object):
         # The default context factory in Twisted 14.0.0 (which we require) is
         # BrowserLikePolicyForHTTPS which will do regular cert validation
         # 'like a browser'
-        pool = HTTPConnectionPool(reactor)
-        pool.maxPersistentPerHost = 10
         self.agent = Agent(
             reactor,
-            pool=pool,
             connectTimeout=15,
             contextFactory=hs.get_http_client_context_factory()
         )
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 94adabca62..4cee1c1599 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -26,7 +26,7 @@ from synapse.events.utils import serialize_event
 import simplejson as json
 import logging
 import urllib
-
+from synapse.util import third_party_invites
 
 logger = logging.getLogger(__name__)
 
@@ -414,10 +414,26 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
 
         # target user is you unless it is an invite
         state_key = user.to_string()
-        if membership_action in ["invite", "ban", "kick"]:
-            if "user_id" not in content:
+
+        if membership_action == "invite" and third_party_invites.has_invite_keys(content):
+            yield self.handlers.room_member_handler.do_3pid_invite(
+                room_id,
+                user,
+                content["medium"],
+                content["address"],
+                content["id_server"],
+                content["display_name"],
+                token_id,
+                txn_id
+            )
+            defer.returnValue((200, {}))
+            return
+        elif membership_action in ["invite", "ban", "kick"]:
+            if "user_id" in content:
+                state_key = content["user_id"]
+            else:
                 raise SynapseError(400, "Missing user_id key.")
-            state_key = content["user_id"]
+
             # make sure it looks like a user ID; it'll throw if it's invalid.
             UserID.from_string(state_key)
 
@@ -425,10 +441,20 @@ class RoomMembershipRestServlet(ClientV1RestServlet):
                 membership_action = "leave"
 
         msg_handler = self.handlers.message_handler
+
+        event_content = {
+            "membership": unicode(membership_action),
+        }
+
+        if membership_action == "join" and third_party_invites.has_join_keys(content):
+            event_content["third_party_invite"] = (
+                third_party_invites.extract_join_keys(content)
+            )
+
         yield msg_handler.create_and_send_event(
             {
                 "type": EventTypes.Member,
-                "content": {"membership": unicode(membership_action)},
+                "content": event_content,
                 "room_id": room_id,
                 "sender": user.to_string(),
                 "state_key": state_key,
diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py
index 700da8d8c3..fcd43c7fdd 100644
--- a/synapse/storage/filtering.py
+++ b/synapse/storage/filtering.py
@@ -34,7 +34,7 @@ class FilteringStore(SQLBaseStore):
             desc="get_user_filter",
         )
 
-        defer.returnValue(json.loads(def_json.decode("utf-8")))
+        defer.returnValue(json.loads(str(def_json).decode("utf-8")))
 
     def add_user_filter(self, user_localpart, user_filter):
         def_json = json.dumps(user_filter).encode("utf-8")
diff --git a/synapse/util/third_party_invites.py b/synapse/util/third_party_invites.py
new file mode 100644
index 0000000000..792db5ba39
--- /dev/null
+++ b/synapse/util/third_party_invites.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+from synapse.api.errors import AuthError
+
+
+INVITE_KEYS = {"id_server", "medium", "address", "display_name"}
+
+JOIN_KEYS = {
+    "token",
+    "public_key",
+    "key_validity_url",
+    "signatures",
+    "sender",
+}
+
+
+def has_invite_keys(content):
+    for key in INVITE_KEYS:
+        if key not in content:
+            return False
+    return True
+
+
+def has_join_keys(content):
+    for key in JOIN_KEYS:
+        if key not in content:
+            return False
+    return True
+
+
+def join_has_third_party_invite(content):
+    if "third_party_invite" not in content:
+        return False
+    return has_join_keys(content["third_party_invite"])
+
+
+def extract_join_keys(src):
+    return {
+        key: value
+        for key, value in src.items()
+        if key in JOIN_KEYS
+    }
+
+
+@defer.inlineCallbacks
+def check_key_valid(http_client, event):
+    try:
+        response = yield http_client.get_json(
+            event.content["third_party_invite"]["key_validity_url"],
+            {"public_key": event.content["third_party_invite"]["public_key"]}
+        )
+    except Exception:
+        raise AuthError(502, "Third party certificate could not be checked")
+    if "valid" not in response or not response["valid"]:
+        raise AuthError(403, "Third party certificate was invalid")