From 2f9157b427efe243c306fc219accb1dba9807f10 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 20 Apr 2015 16:23:47 +0100
Subject: Implement v2 key lookup

---
 synapse/crypto/keyring.py | 268 +++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 251 insertions(+), 17 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index f4db7b8a05..5528d0a280 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -36,6 +36,8 @@ class Keyring(object):
     def __init__(self, hs):
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
+        self.client = hs.get_http_client()
+        self.perspective_servers = {}
         self.hs = hs
 
     @defer.inlineCallbacks
@@ -85,19 +87,26 @@ class Keyring(object):
     @defer.inlineCallbacks
     def get_server_verify_key(self, server_name, key_ids):
         """Finds a verification key for the server with one of the key ids.
+        Trys to fetch the key from a trusted perspective server first.
         Args:
-            server_name (str): The name of the server to fetch a key for.
+            server_name(str): The name of the server to fetch a key for.
             keys_ids (list of str): The key_ids to check for.
         """
-
-        # Check the datastore to see if we have one cached.
         cached = yield self.store.get_server_verify_keys(server_name, key_ids)
 
         if cached:
             defer.returnValue(cached[0])
             return
 
-        # Try to fetch the key from the remote server.
+        keys = None
+        for perspective_name, perspective_keys in self.perspective_servers.items():
+            try:
+                keys = yield self.get_server_verify_key_v2_indirect(
+                    server_name, key_ids, perspective_name, perspective_keys
+                )
+                break
+            except:
+                pass
 
         limiter = yield get_retry_limiter(
             server_name,
@@ -106,10 +115,221 @@ class Keyring(object):
         )
 
         with limiter:
-            (response, tls_certificate) = yield fetch_server_key(
-                server_name, self.hs.tls_context_factory
+            if keys is None:
+                try:
+                    keys = yield self.get_server_verify_key_v2_direct(
+                        server_name, key_ids
+                    )
+                except:
+                    pass
+
+            keys = yield self.get_server_verify_key_v1_direct(
+                server_name, key_ids
+            )
+
+
+        for key_id in key_ids:
+            if key_id in keys:
+                defer.returnValue(keys[key_id])
+                return
+        raise ValueError("No verification key found for given key ids")
+
+    @defer.inlineCallbacks
+    def get_server_verify_key_v2_indirect(self, server_name, key_ids,
+                                          perspective_name,
+                                          perspective_keys):
+        limiter = yield get_retry_limiter(
+            perspective_name, self.clock, self.store
+        )
+
+        responses = yield self.client.post_json(
+            destination=perspective_name,
+            path=b"/_matrix/key/v2/query",
+            data={u"server_keys": {server_name: list(key_ids)}},
+        )
+
+        keys = dict()
+
+        for response in responses:
+            if (u"signatures" not in response
+                or perspective_name not in response[u"signatures"]):
+                raise ValueError(
+                    "Key response not signed by perspective server"
+                    " %r" % (perspective_name,)
+                )
+
+            verified = False
+            for key_id in response[u"signatures"][perspective_name]:
+                if key_id in perspective_keys:
+                    verify_signed_json(
+                        response,
+                        perspective_name,
+                        perspective_keys[key_id]
+                    )
+                    verified = True
+
+            if not verified:
+                logging.info(
+                    "Response from perspective server %r not signed with a"
+                    " known key, signed with: %r, known keys: %r",
+                    perspective_name,
+                    list(response[u"signatures"][perspective_name]),
+                    list(perspective_keys)
+                )
+                raise ValueError(
+                    "Response not signed with a known key for perspective"
+                    " server %r" % (perspective_name,)
+                )
+
+            response_keys = process_v2_response(self, server_name, key_ids)
+
+            keys.update(response_keys)
+
+        yield self.store_keys(
+            server_name=server_name,
+            from_server=perspective_name,
+            verify_keys=keys,
+        )
+
+        defer.returnValue(keys)
+
+    @defer.inlineCallbacks
+    def get_server_verify_key_v2_direct(self, server_name, key_ids):
+
+        keys = {}
+
+        for requested_key_id in key_ids:
+            if requested_key_id in keys:
+                continue
+
+            (response_json, tls_certificate) = yield fetch_server_key(
+                server_name, self.hs.tls_context_factory,
+                path="/_matrix/key/v2/server/%s" % (
+                    urllib.quote(requested_key_id),
+                ),
+            )
+
+            if (u"signatures" not in response
+                or server_name not in response[u"signatures"]):
+                raise ValueError("Key response not signed by remote server")
+
+            if "tls_fingerprints" not in response:
+                raise ValueError("Key response missing TLS fingerprints")
+
+            certificate_bytes = crypto.dump_certificate(
+                crypto.FILETYPE_ASN1, tls_certificate
+            )
+            sha256_fingerprint = hashlib.sha256(certificate_bytes).digest()
+            sha256_fingerprint_b64 = encode_base64(sha256_fingerprint)
+
+            response_sha256_fingerprints = set()
+            for fingerprint in response_json[u"tls_fingerprints"]:
+                if u"sha256" in fingerprint:
+                    response_sha256_fingerprints.add(fingerprint[u"sha256"])
+
+            if sha256_fingerprint not in response_sha256_fingerprints:
+                raise ValueError("TLS certificate not allowed by fingerprints")
+
+            response_keys = yield self.process_v2_response(
+                server_name=server_name,
+                from_server=server_name,
+                response_json=response_json,
+            )
+
+            keys.update(response_keys)
+
+        yield self.store_keys(
+            server_name=server_name,
+            from_server=server_name,
+            verify_keys=keys,
+        )
+
+        for key_id in key_ids:
+            if key_id in verify_keys:
+                defer.returnValue(verify_keys[key_id])
+                return
+
+        raise ValueError("No verification key found for given key ids")
+
+    @defer.inlineCallbacks
+    def process_v2_response(self, server_name, from_server, json_response):
+        time_now_ms = clock.time_msec()
+        response_keys = {}
+        verify_keys = {}
+        for key_id, key_data in response["verify_keys"].items():
+            if is_signing_algorithm_supported(key_id):
+                key_base64 = key_data["key"]
+                key_bytes = decode_base64(key_base64)
+                verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                verify_keys[key_id] = verify_key
+
+        old_verify_keys = {}
+        for key_id, key_data in response["verify_keys"].items():
+            if is_signing_algorithm_supported(key_id):
+                key_base64 = key_data["key"]
+                key_bytes = decode_base64(key_base64)
+                verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                verify_key.expired = key_data["expired"]
+                verify_key.time_added = time_now_ms
+                old_verify_keys[key_id] = verify_key
+
+        for key_id in response["signatures"][server_name]:
+            if key_id not in response["verify_keys"]:
+                raise ValueError(
+                    "Key response must include verification keys for all"
+                    " signatures"
+                )
+            if key_id in verify_keys:
+                verify_signed_json(
+                    response,
+                    server_name,
+                    verify_keys[key_id]
+                )
+
+        signed_key_json = sign_json(
+            response,
+            self.config.server_name,
+            self.config.signing_key[0],
+        )
+
+        signed_key_json_bytes = encode_canonical_json(signed_key_json)
+        ts_valid_until_ms = signed_key_json[u"valid_until"]
+
+        updated_key_ids = set([requested_key_id])
+        updated_key_ids.update(verify_keys)
+        updated_key_ids.update(old_verify_keys)
+
+        response_keys.update(verify_keys)
+        response_keys.update(old_verify_keys)
+
+        for key_id in updated_key_ids:
+            yield self.store.store_server_keys_json(
+                server_name=server_name,
+                key_id=key_id,
+                from_server=server_name,
+                ts_now_ms=ts_now_ms,
+                ts_valid_until_ms=valid_until,
+                key_json_bytes=signed_key_json_bytes,
             )
 
+        defer.returnValue(response_keys)
+
+        raise ValueError("No verification key found for given key ids")
+
+    @defer.inlineCallbacks
+    def get_server_verify_key_v1_direct(self, server_name, key_ids):
+        """Finds a verification key for the server with one of the key ids.
+        Args:
+            server_name (str): The name of the server to fetch a key for.
+            keys_ids (list of str): The key_ids to check for.
+        """
+
+        # Try to fetch the key from the remote server.
+
+        (response, tls_certificate) = yield fetch_server_key(
+            server_name, self.hs.tls_context_factory
+        )
+
         # Check the response.
 
         x509_certificate_bytes = crypto.dump_certificate(
@@ -128,11 +348,16 @@ class Keyring(object):
         if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
             raise ValueError("TLS certificate doesn't match")
 
+        # Cache the result in the datastore.
+
+        time_now_ms = self.clock.time_msec()
+
         verify_keys = {}
         for key_id, key_base64 in response["verify_keys"].items():
             if is_signing_algorithm_supported(key_id):
                 key_bytes = decode_base64(key_base64)
                 verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                verify_key.time_added = time_now_ms
                 verify_keys[key_id] = verify_key
 
         for key_id in response["signatures"][server_name]:
@@ -148,9 +373,6 @@ class Keyring(object):
                     verify_keys[key_id]
                 )
 
-        # Cache the result in the datastore.
-
-        time_now_ms = self.clock.time_msec()
 
         yield self.store.store_server_certificate(
             server_name,
@@ -159,14 +381,26 @@ class Keyring(object):
             tls_certificate,
         )
 
+        yield self.store_keys(
+            server_name=server_name,
+            from_server=server_name,
+            verify_keys=verify_keys,
+        )
+
+        defer.returnValue(verify_keys)
+
+    @defer.inlineCallbacks
+    def store_keys(self, server_name, from_server, verify_keys):
+        """Store a collection of verify keys for a given server
+        Args:
+            server_name(str): The name of the server the keys are for.
+            from_server(str): The server the keys were downloaded from.
+            verify_keys(dict): A mapping of key_id to VerifyKey.
+        Returns:
+            A deferred that completes when the keys are stored.
+        """
         for key_id, key in verify_keys.items():
+            # TODO(markjh): Store whether the keys have expired.
             yield self.store.store_server_verify_key(
-                server_name, server_name, time_now_ms, key
+                server_name, server_name, key.time_added, key
             )
-
-        for key_id in key_ids:
-            if key_id in verify_keys:
-                defer.returnValue(verify_keys[key_id])
-                return
-
-        raise ValueError("No verification key found for given key ids")
-- 
cgit 1.4.1


From f30d47c87651f92b69be224e016bda2cd7285f04 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 22 Apr 2015 14:21:08 +0100
Subject: Implement remote key lookup api

---
 synapse/config/server.py                   |   4 +-
 synapse/crypto/keyclient.py                |   6 +-
 synapse/crypto/keyring.py                  |  75 +++++++------
 synapse/rest/key/v2/__init__.py            |  10 +-
 synapse/rest/key/v2/local_key_resource.py  |   9 +-
 synapse/rest/key/v2/remote_key_resource.py | 174 +++++++++++++++++++++++++++++
 synapse/storage/keys.py                    |  35 +++---
 7 files changed, 252 insertions(+), 61 deletions(-)
 create mode 100644 synapse/rest/key/v2/remote_key_resource.py

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/config/server.py b/synapse/config/server.py
index 050ab90403..a26fb115f2 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -62,7 +62,7 @@ class ServerConfig(Config):
         server_group.add_argument("--old-signing-key-path",
                                   help="The old signing keys")
         server_group.add_argument("--key-refresh-interval",
-                                  default=24 * 60 * 60 * 1000, # 1 Day
+                                  default=24 * 60 * 60 * 1000,  # 1 Day
                                   help="How long a key response is valid for."
                                        " Used to set the exipiry in /key/v2/."
                                        " Controls how frequently servers will"
@@ -156,5 +156,5 @@ class ServerConfig(Config):
             args.old_signing_key_path = base_key_name + ".old.signing.keys"
 
         if not os.path.exists(args.old_signing_key_path):
-            with open(args.old_signing_key_path, "w") as old_signing_key_file:
+            with open(args.old_signing_key_path, "w"):
                 pass
diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index 2452c7a26e..4911f0896b 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -26,7 +26,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 KEY_API_V1 = b"/_matrix/key/v1/"
-KEY_API_V2 = b"/_matrix/key/v2/local"
+
 
 @defer.inlineCallbacks
 def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1):
@@ -94,8 +94,8 @@ class SynapseKeyClientProtocol(HTTPClient):
         if status != b"200":
             # logger.info("Non-200 response from %s: %s %s",
             #            self.transport.getHost(), status, message)
-            error = SynapseKeyClientError("Non-200 response %r from %r" %
-                (status, self.host)
+            error = SynapseKeyClientError(
+                "Non-200 response %r from %r" % (status, self.host)
             )
             error.status = status
             self.errback(error)
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 5528d0a280..17ac66731c 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -15,7 +15,9 @@
 
 from synapse.crypto.keyclient import fetch_server_key
 from twisted.internet import defer
-from syutil.crypto.jsonsign import verify_signed_json, signature_ids
+from syutil.crypto.jsonsign import (
+    verify_signed_json, signature_ids, sign_json, encode_canonical_json
+)
 from syutil.crypto.signing_key import (
     is_signing_algorithm_supported, decode_verify_key_bytes
 )
@@ -26,6 +28,8 @@ from synapse.util.retryutils import get_retry_limiter
 
 from OpenSSL import crypto
 
+import urllib
+import hashlib
 import logging
 
 
@@ -37,6 +41,7 @@ class Keyring(object):
         self.store = hs.get_datastore()
         self.clock = hs.get_clock()
         self.client = hs.get_http_client()
+        self.config = hs.get_config()
         self.perspective_servers = {}
         self.hs = hs
 
@@ -127,7 +132,6 @@ class Keyring(object):
                 server_name, key_ids
             )
 
-
         for key_id in key_ids:
             if key_id in keys:
                 defer.returnValue(keys[key_id])
@@ -142,17 +146,18 @@ class Keyring(object):
             perspective_name, self.clock, self.store
         )
 
-        responses = yield self.client.post_json(
-            destination=perspective_name,
-            path=b"/_matrix/key/v2/query",
-            data={u"server_keys": {server_name: list(key_ids)}},
-        )
+        with limiter:
+            responses = yield self.client.post_json(
+                destination=perspective_name,
+                path=b"/_matrix/key/v2/query",
+                data={u"server_keys": {server_name: list(key_ids)}},
+            )
 
-        keys = dict()
+        keys = {}
 
         for response in responses:
             if (u"signatures" not in response
-                or perspective_name not in response[u"signatures"]):
+                    or perspective_name not in response[u"signatures"]):
                 raise ValueError(
                     "Key response not signed by perspective server"
                     " %r" % (perspective_name,)
@@ -181,7 +186,9 @@ class Keyring(object):
                     " server %r" % (perspective_name,)
                 )
 
-            response_keys = process_v2_response(self, server_name, key_ids)
+            response_keys = yield self.process_v2_response(
+                server_name, perspective_name, response
+            )
 
             keys.update(response_keys)
 
@@ -202,15 +209,15 @@ class Keyring(object):
             if requested_key_id in keys:
                 continue
 
-            (response_json, tls_certificate) = yield fetch_server_key(
+            (response, tls_certificate) = yield fetch_server_key(
                 server_name, self.hs.tls_context_factory,
-                path="/_matrix/key/v2/server/%s" % (
+                path=(b"/_matrix/key/v2/server/%s" % (
                     urllib.quote(requested_key_id),
-                ),
+                )).encode("ascii"),
             )
 
             if (u"signatures" not in response
-                or server_name not in response[u"signatures"]):
+                    or server_name not in response[u"signatures"]):
                 raise ValueError("Key response not signed by remote server")
 
             if "tls_fingerprints" not in response:
@@ -223,17 +230,18 @@ class Keyring(object):
             sha256_fingerprint_b64 = encode_base64(sha256_fingerprint)
 
             response_sha256_fingerprints = set()
-            for fingerprint in response_json[u"tls_fingerprints"]:
+            for fingerprint in response[u"tls_fingerprints"]:
                 if u"sha256" in fingerprint:
                     response_sha256_fingerprints.add(fingerprint[u"sha256"])
 
-            if sha256_fingerprint not in response_sha256_fingerprints:
+            if sha256_fingerprint_b64 not in response_sha256_fingerprints:
                 raise ValueError("TLS certificate not allowed by fingerprints")
 
             response_keys = yield self.process_v2_response(
                 server_name=server_name,
                 from_server=server_name,
-                response_json=response_json,
+                requested_id=requested_key_id,
+                response_json=response,
             )
 
             keys.update(response_keys)
@@ -244,19 +252,15 @@ class Keyring(object):
             verify_keys=keys,
         )
 
-        for key_id in key_ids:
-            if key_id in verify_keys:
-                defer.returnValue(verify_keys[key_id])
-                return
-
-        raise ValueError("No verification key found for given key ids")
+        defer.returnValue(keys)
 
     @defer.inlineCallbacks
-    def process_v2_response(self, server_name, from_server, json_response):
-        time_now_ms = clock.time_msec()
+    def process_v2_response(self, server_name, from_server, response_json,
+                            requested_id=None):
+        time_now_ms = self.clock.time_msec()
         response_keys = {}
         verify_keys = {}
-        for key_id, key_data in response["verify_keys"].items():
+        for key_id, key_data in response_json["verify_keys"].items():
             if is_signing_algorithm_supported(key_id):
                 key_base64 = key_data["key"]
                 key_bytes = decode_base64(key_base64)
@@ -264,7 +268,7 @@ class Keyring(object):
                 verify_keys[key_id] = verify_key
 
         old_verify_keys = {}
-        for key_id, key_data in response["verify_keys"].items():
+        for key_id, key_data in response_json["old_verify_keys"].items():
             if is_signing_algorithm_supported(key_id):
                 key_base64 = key_data["key"]
                 key_bytes = decode_base64(key_base64)
@@ -273,21 +277,21 @@ class Keyring(object):
                 verify_key.time_added = time_now_ms
                 old_verify_keys[key_id] = verify_key
 
-        for key_id in response["signatures"][server_name]:
-            if key_id not in response["verify_keys"]:
+        for key_id in response_json["signatures"][server_name]:
+            if key_id not in response_json["verify_keys"]:
                 raise ValueError(
                     "Key response must include verification keys for all"
                     " signatures"
                 )
             if key_id in verify_keys:
                 verify_signed_json(
-                    response,
+                    response_json,
                     server_name,
                     verify_keys[key_id]
                 )
 
         signed_key_json = sign_json(
-            response,
+            response_json,
             self.config.server_name,
             self.config.signing_key[0],
         )
@@ -295,7 +299,9 @@ class Keyring(object):
         signed_key_json_bytes = encode_canonical_json(signed_key_json)
         ts_valid_until_ms = signed_key_json[u"valid_until"]
 
-        updated_key_ids = set([requested_key_id])
+        updated_key_ids = set()
+        if requested_id is not None:
+            updated_key_ids.add(requested_id)
         updated_key_ids.update(verify_keys)
         updated_key_ids.update(old_verify_keys)
 
@@ -307,8 +313,8 @@ class Keyring(object):
                 server_name=server_name,
                 key_id=key_id,
                 from_server=server_name,
-                ts_now_ms=ts_now_ms,
-                ts_valid_until_ms=valid_until,
+                ts_now_ms=time_now_ms,
+                ts_expires_ms=ts_valid_until_ms,
                 key_json_bytes=signed_key_json_bytes,
             )
 
@@ -373,7 +379,6 @@ class Keyring(object):
                     verify_keys[key_id]
                 )
 
-
         yield self.store.store_server_certificate(
             server_name,
             server_name,
diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py
index b79ed02590..1c14791b09 100644
--- a/synapse/rest/key/v2/__init__.py
+++ b/synapse/rest/key/v2/__init__.py
@@ -13,7 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from twisted.web.resource import Resource
 from .local_key_resource import LocalKey
+from .remote_key_resource import RemoteKey
 
-class KeyApiV2Resource(LocalKey):
-    pass
+
+class KeyApiV2Resource(Resource):
+    def __init__(self, hs):
+        Resource.__init__(self)
+        self.putChild("server", LocalKey(hs))
+        self.putChild("query", RemoteKey(hs))
diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py
index 1c0e0717c1..982a460962 100644
--- a/synapse/rest/key/v2/local_key_resource.py
+++ b/synapse/rest/key/v2/local_key_resource.py
@@ -31,7 +31,7 @@ class LocalKey(Resource):
     """HTTP resource containing encoding the TLS X.509 certificate and NACL
     signature verification keys for this server::
 
-        GET /_matrix/key/v2/ HTTP/1.1
+        GET /_matrix/key/v2/server/a.key.id HTTP/1.1
 
         HTTP/1.1 200 OK
         Content-Type: application/json
@@ -56,6 +56,8 @@ class LocalKey(Resource):
         }
     """
 
+    isLeaf = True
+
     def __init__(self, hs):
         self.version_string = hs.version_string
         self.config = hs.config
@@ -68,7 +70,6 @@ class LocalKey(Resource):
         self.expires = int(time_now_msec + refresh_interval)
         self.response_body = encode_canonical_json(self.response_json_object())
 
-
     def response_json_object(self):
         verify_keys = {}
         for key in self.config.signing_key:
@@ -120,7 +121,3 @@ class LocalKey(Resource):
             request, 200, self.response_body,
             version_string=self.version_string
         )
-
-    def getChild(self, name, request):
-        if name == '':
-            return self
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
new file mode 100644
index 0000000000..cf6f2c2e73
--- /dev/null
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -0,0 +1,174 @@
+from synapse.http.server import request_handler, respond_with_json_bytes
+from synapse.api.errors import SynapseError, Codes
+
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET
+from twisted.internet import defer
+
+
+from io import BytesIO
+import json
+import logging
+logger = logging.getLogger(__name__)
+
+
+class RemoteKey(Resource):
+    """HTTP resource for retreiving the TLS certificate and NACL signature
+    verification keys for a collection of servers. Checks that the reported
+    X.509 TLS certificate matches the one used in the HTTPS connection. Checks
+    that the NACL signature for the remote server is valid. Returns a dict of
+    JSON signed by both the remote server and by this server.
+
+    Supports individual GET APIs and a bulk query POST API.
+
+    Requsts:
+
+    GET /_matrix/key/v2/query/remote.server.example.com HTTP/1.1
+
+    GET /_matrix/key/v2/query/remote.server.example.com/a.key.id HTTP/1.1
+
+    POST /_matrix/v2/query HTTP/1.1
+    Content-Type: application/json
+    {
+        "server_keys": { "remote.server.example.com": ["a.key.id"] }
+    }
+
+    Response:
+
+    HTTP/1.1 200 OK
+    Content-Type: application/json
+    {
+        "server_keys": [
+            {
+                "server_name": "remote.server.example.com"
+                "valid_until": # posix timestamp
+                "verify_keys": {
+                    "a.key.id": { # The identifier for a key.
+                        key: "" # base64 encoded verification key.
+                    }
+                }
+                "old_verify_keys": {
+                    "an.old.key.id": { # The identifier for an old key.
+                        key: "", # base64 encoded key
+                        expired: 0, # when th e
+                    }
+                }
+                "tls_fingerprints": [
+                    { "sha256": # fingerprint }
+                ]
+                "signatures": {
+                    "remote.server.example.com": {...}
+                    "this.server.example.com": {...}
+                }
+            }
+        ]
+    }
+    """
+
+    isLeaf = True
+
+    def __init__(self, hs):
+        self.keyring = hs.get_keyring()
+        self.store = hs.get_datastore()
+        self.version_string = hs.version_string
+        self.clock = hs.get_clock()
+
+    def render_GET(self, request):
+        self.async_render_GET(request)
+        return NOT_DONE_YET
+
+    @request_handler
+    @defer.inlineCallbacks
+    def async_render_GET(self, request):
+        if len(request.postpath) == 1:
+            server, = request.postpath
+            query = {server: [None]}
+        elif len(request.postpath) == 2:
+            server, key_id = request.postpath
+            query = {server: [key_id]}
+        else:
+            raise SynapseError(
+                404, "Not found %r" % request.postpath, Codes.NOT_FOUND
+            )
+        yield self.query_keys(request, query, query_remote_on_cache_miss=True)
+
+    def render_POST(self, request):
+        self.async_render_POST(request)
+        return NOT_DONE_YET
+
+    @request_handler
+    @defer.inlineCallbacks
+    def async_render_POST(self, request):
+        try:
+            content = json.loads(request.content.read())
+            if type(content) != dict:
+                raise ValueError()
+        except ValueError:
+            raise SynapseError(
+                400, "Content must be JSON object.", errcode=Codes.NOT_JSON
+            )
+
+        query = content["server_keys"]
+
+        yield self.query_keys(request, query, query_remote_on_cache_miss=True)
+
+    @defer.inlineCallbacks
+    def query_keys(self, request, query, query_remote_on_cache_miss=False):
+        store_queries = []
+        for server_name, key_ids in query.items():
+            for key_id in key_ids:
+                store_queries.append((server_name, key_id, None))
+
+        cached = yield self.store.get_server_keys_json(store_queries)
+
+        json_results = []
+
+        time_now_ms = self.clock.time_msec()
+
+        cache_misses = dict()
+        for (server_name, key_id, from_server), results in cached.items():
+            results = [
+                (result["ts_added_ms"], result) for result in results
+                if result["ts_valid_until_ms"] > time_now_ms
+            ]
+
+            if not results:
+                if key_id is not None:
+                    cache_misses.setdefault(server_name, set()).add(key_id)
+                continue
+
+            if key_id is not None:
+                most_recent_result = max(results)
+                json_results.append(most_recent_result[-1]["key_json"])
+            else:
+                for result in results:
+                    json_results.append(result[-1]["key_json"])
+
+        if cache_misses and query_remote_on_cache_miss:
+            for server_name, key_ids in cache_misses.items():
+                try:
+                    yield self.keyring.get_server_verify_key_v2_direct(
+                        server_name, key_ids
+                    )
+                except:
+                    logger.exception("Failed to get key for %r", server_name)
+                    pass
+            yield self.query_keys(
+                request, query, query_remote_on_cache_miss=False
+            )
+        else:
+            result_io = BytesIO()
+            result_io.write(b"{\"server_keys\":")
+            sep = b"["
+            for json_bytes in json_results:
+                result_io.write(sep)
+                result_io.write(json_bytes)
+                sep = b","
+            if sep == b"[":
+                result_io.write(sep)
+            result_io.write(b"]}")
+
+            respond_with_json_bytes(
+                request, 200, result_io.getvalue(),
+                version_string=self.version_string
+            )
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 8b08d42859..22b158d71e 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -140,8 +140,8 @@ class KeyStore(SQLBaseStore):
                 "key_id": key_id,
                 "from_server": from_server,
                 "ts_added_ms": ts_now_ms,
-                "ts_valid_until_ms": ts_valid_until_ms,
-                "key_json": key_json_bytes,
+                "ts_valid_until_ms": ts_expires_ms,
+                "key_json": buffer(key_json_bytes),
             },
             or_replace=True,
         )
@@ -149,9 +149,9 @@ class KeyStore(SQLBaseStore):
     def get_server_keys_json(self, server_keys):
         """Retrive the key json for a list of server_keys and key ids.
         If no keys are found for a given server, key_id and source then
-        that server, key_id, and source triplet will be missing from the
-        returned dictionary. The JSON is returned as a byte array so that it
-        can be efficiently used in an HTTP response.
+        that server, key_id, and source triplet entry will be an empty list.
+        The JSON is returned as a byte array so that it can be efficiently
+        used in an HTTP response.
         Args:
             server_keys (list): List of (server_name, key_id, source) triplets.
         Returns:
@@ -161,16 +161,25 @@ class KeyStore(SQLBaseStore):
         def _get_server_keys_json_txn(txn):
             results = {}
             for server_name, key_id, from_server in server_keys:
-                rows = _simple_select_list_txn(
-                    keyvalues={
-                        "server_name": server_name,
-                        "key_id": key_id,
-                        "from_server": from_server,
-                    },
-                    retcols=("ts_valid_until_ms", "key_json"),
+                keyvalues = {"server_name": server_name}
+                if key_id is not None:
+                    keyvalues["key_id"] = key_id
+                if from_server is not None:
+                    keyvalues["from_server"] = from_server
+                rows = self._simple_select_list_txn(
+                    txn,
+                    "server_keys_json",
+                    keyvalues=keyvalues,
+                    retcols=(
+                        "key_id",
+                        "from_server",
+                        "ts_added_ms",
+                        "ts_valid_until_ms",
+                        "key_json",
+                    ),
                 )
                 results[(server_name, key_id, from_server)] = rows
             return results
-        return runInteraction(
+        return self.runInteraction(
             "get_server_keys_json", _get_server_keys_json_txn
         )
-- 
cgit 1.4.1


From 4bbf7156efdc493fc1c9e3177bef90c45f25f0d3 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 23 Apr 2015 16:39:13 +0100
Subject: Update to match the specification for key/v2

---
 synapse/crypto/keyring.py                  |  4 ++--
 synapse/rest/key/v2/local_key_resource.py  | 18 ++++++++++--------
 synapse/rest/key/v2/remote_key_resource.py | 25 ++++++++++++++-----------
 3 files changed, 26 insertions(+), 21 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 17ac66731c..d248776bc1 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -273,7 +273,7 @@ class Keyring(object):
                 key_base64 = key_data["key"]
                 key_bytes = decode_base64(key_base64)
                 verify_key = decode_verify_key_bytes(key_id, key_bytes)
-                verify_key.expired = key_data["expired"]
+                verify_key.expired = key_data["expired_ts"]
                 verify_key.time_added = time_now_ms
                 old_verify_keys[key_id] = verify_key
 
@@ -297,7 +297,7 @@ class Keyring(object):
         )
 
         signed_key_json_bytes = encode_canonical_json(signed_key_json)
-        ts_valid_until_ms = signed_key_json[u"valid_until"]
+        ts_valid_until_ms = signed_key_json[u"valid_until_ts"]
 
         updated_key_ids = set()
         if requested_id is not None:
diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py
index 982a460962..33cbd7cf8e 100644
--- a/synapse/rest/key/v2/local_key_resource.py
+++ b/synapse/rest/key/v2/local_key_resource.py
@@ -36,14 +36,16 @@ class LocalKey(Resource):
         HTTP/1.1 200 OK
         Content-Type: application/json
         {
-            "expires": # integer posix timestamp when this result expires.
+            "valid_until_ts": # integer posix timestamp when this result expires.
             "server_name": "this.server.example.com"
             "verify_keys": {
-                "algorithm:version": # base64 encoded NACL verification key.
+                "algorithm:version": {
+                    "key": # base64 encoded NACL verification key.
+                }
             },
             "old_verify_keys": {
                 "algorithm:version": {
-                    "expired": # integer posix timestamp when the key expired.
+                    "expired_ts": # integer posix timestamp when the key expired.
                     "key": # base64 encoded NACL verification key.
                 }
             }
@@ -67,7 +69,7 @@ class LocalKey(Resource):
 
     def update_response_body(self, time_now_msec):
         refresh_interval = self.config.key_refresh_interval
-        self.expires = int(time_now_msec + refresh_interval)
+        self.valid_until_ts = int(time_now_msec + refresh_interval)
         self.response_body = encode_canonical_json(self.response_json_object())
 
     def response_json_object(self):
@@ -85,7 +87,7 @@ class LocalKey(Resource):
             verify_key_bytes = key.encode()
             old_verify_keys[key_id] = {
                 u"key": encode_base64(verify_key_bytes),
-                u"expired": key.expired,
+                u"expired_ts": key.expired,
             }
 
         x509_certificate_bytes = crypto.dump_certificate(
@@ -96,7 +98,7 @@ class LocalKey(Resource):
         sha256_fingerprint = sha256(x509_certificate_bytes).digest()
 
         json_object = {
-            u"valid_until": self.expires,
+            u"valid_until_ts": self.valid_until_ts,
             u"server_name": self.config.server_name,
             u"verify_keys": verify_keys,
             u"old_verify_keys": old_verify_keys,
@@ -115,8 +117,8 @@ class LocalKey(Resource):
     def render_GET(self, request):
         time_now = self.clock.time_msec()
         # Update the expiry time if less than half the interval remains.
-        if time_now + self.config.key_refresh_interval / 2 > self.expires:
-            self.update_response_body()
+        if time_now + self.config.key_refresh_interval / 2 > self.valid_until_ts:
+            self.update_response_body(time_now)
         return respond_with_json_bytes(
             request, 200, self.response_body,
             version_string=self.version_string
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index cf6f2c2e73..724ca00397 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -41,7 +41,7 @@ class RemoteKey(Resource):
         "server_keys": [
             {
                 "server_name": "remote.server.example.com"
-                "valid_until": # posix timestamp
+                "valid_until_ts": # posix timestamp
                 "verify_keys": {
                     "a.key.id": { # The identifier for a key.
                         key: "" # base64 encoded verification key.
@@ -50,7 +50,7 @@ class RemoteKey(Resource):
                 "old_verify_keys": {
                     "an.old.key.id": { # The identifier for an old key.
                         key: "", # base64 encoded key
-                        expired: 0, # when th e
+                        "expired_ts": 0, # when the key stop being used.
                     }
                 }
                 "tls_fingerprints": [
@@ -121,7 +121,7 @@ class RemoteKey(Resource):
 
         cached = yield self.store.get_server_keys_json(store_queries)
 
-        json_results = []
+        json_results = set()
 
         time_now_ms = self.clock.time_msec()
 
@@ -129,20 +129,23 @@ class RemoteKey(Resource):
         for (server_name, key_id, from_server), results in cached.items():
             results = [
                 (result["ts_added_ms"], result) for result in results
-                if result["ts_valid_until_ms"] > time_now_ms
             ]
 
-            if not results:
-                if key_id is not None:
-                    cache_misses.setdefault(server_name, set()).add(key_id)
+            if not results and key_id is not None:
+                cache_misses.setdefault(server_name, set()).add(key_id)
                 continue
 
             if key_id is not None:
-                most_recent_result = max(results)
-                json_results.append(most_recent_result[-1]["key_json"])
+                ts_added_ms, most_recent_result = max(results)
+                ts_valid_until_ms = most_recent_result["ts_valid_until_ms"]
+                if (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms:
+                    # We more than half way through the lifetime of the
+                    # response. We should fetch a fresh copy.
+                    cache_misses.setdefault(server_name, set()).add(key_id)
+                json_results.add(bytes(most_recent_result["key_json"]))
             else:
-                for result in results:
-                    json_results.append(result[-1]["key_json"])
+                for ts_added, result in results:
+                    json_results.add(bytes(result["key_json"]))
 
         if cache_misses and query_remote_on_cache_miss:
             for server_name, key_ids in cache_misses.items():
-- 
cgit 1.4.1


From 288702170d6fc8b44926856b37e4a0e1bb5b2ac4 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 24 Apr 2015 17:01:34 +0100
Subject: Add config for setting the perspective servers

---
 synapse/config/_base.py   |  4 ++--
 synapse/config/key.py     | 22 ++++++++++++++++++++--
 synapse/crypto/keyring.py |  6 +++++-
 3 files changed, 27 insertions(+), 5 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index f07ea4cc46..6fd086a471 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -83,9 +83,9 @@ class Config(object):
         with open(file_path) as file_stream:
             try:
                 return yaml.load(file_stream)
-            except Exception as e:
+            except:
                 raise ConfigError(
-                    "Error parsing yaml in file %r: " % (file_path,), e
+                    "Error parsing yaml in file %r" % (file_path,)
                 )
 
     @staticmethod
diff --git a/synapse/config/key.py b/synapse/config/key.py
index de4e33a7f3..a2de6d5c17 100644
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -16,6 +16,10 @@
 import os
 from ._base import Config, ConfigError
 import syutil.crypto.signing_key
+from syutil.crypto.signing_key import (
+    is_signing_algorithm_supported, decode_verify_key_bytes
+)
+from syutil.base64util import decode_base64
 
 
 class KeyConfig(Config):
@@ -53,9 +57,17 @@ class KeyConfig(Config):
                                     " keys from")
 
     def read_perspectives(self, perspectives_config_path):
-        servers = self.read_yaml_file(
+        config = self.read_yaml_file(
             perspectives_config_path, "perspectives_config_path"
         )
+        servers = {}
+        for server_name, server_config in config["servers"].items():
+            for key_id, key_data in server_config["verify_keys"].items():
+                if is_signing_algorithm_supported(key_id):
+                    key_base64 = key_data["key"]
+                    key_bytes = decode_base64(key_base64)
+                    verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                    servers.setdefault(server_name, {})[key_id] = verify_key
         return servers
 
     def read_signing_key(self, signing_key_path):
@@ -126,4 +138,10 @@ class KeyConfig(Config):
 
         if not os.path.exists(args.perspectives_config_path):
             with open(args.perspectives_config_path, "w") as perspectives_file:
-                perspectives_file.write("@@@")
+                perspectives_file.write(
+                    'servers:\n'
+                    '  matrix.org:\n'
+                    '    verify_keys:\n'
+                    '      "ed25519:auto":\n'
+                    '         key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"\n'
+                )
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index d248776bc1..f7ae227916 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -42,7 +42,7 @@ class Keyring(object):
         self.clock = hs.get_clock()
         self.client = hs.get_http_client()
         self.config = hs.get_config()
-        self.perspective_servers = {}
+        self.perspective_servers = self.config.perspectives
         self.hs = hs
 
     @defer.inlineCallbacks
@@ -111,6 +111,10 @@ class Keyring(object):
                 )
                 break
             except:
+                logging.info(
+                    "Unable to getting key %r for %r from %r",
+                    key_ids, server_name, perspective_name,
+                )
                 pass
 
         limiter = yield get_retry_limiter(
-- 
cgit 1.4.1


From e701aec2d1e9a565d29bc27d2bde61032cba5fd1 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 27 Apr 2015 14:20:26 +0100
Subject: Implement locks using create_observer for fetching media and server
 keys

---
 synapse/crypto/keyring.py              | 138 +++++++++++++++++++--------------
 synapse/rest/media/v1/base_resource.py |   4 +-
 2 files changed, 82 insertions(+), 60 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index f4db7b8a05..d98341f5c2 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -24,6 +24,8 @@ from synapse.api.errors import SynapseError, Codes
 
 from synapse.util.retryutils import get_retry_limiter
 
+from synapse.util.async import create_observer
+
 from OpenSSL import crypto
 
 import logging
@@ -38,6 +40,8 @@ class Keyring(object):
         self.clock = hs.get_clock()
         self.hs = hs
 
+        self.key_downloads = {}
+
     @defer.inlineCallbacks
     def verify_json_for_server(self, server_name, json_object):
         logger.debug("Verifying for %s", server_name)
@@ -97,76 +101,92 @@ class Keyring(object):
             defer.returnValue(cached[0])
             return
 
-        # Try to fetch the key from the remote server.
-
-        limiter = yield get_retry_limiter(
-            server_name,
-            self.clock,
-            self.store,
-        )
+        @defer.inlineCallbacks
+        def fetch_keys():
+            # Try to fetch the key from the remote server.
 
-        with limiter:
-            (response, tls_certificate) = yield fetch_server_key(
-                server_name, self.hs.tls_context_factory
+            limiter = yield get_retry_limiter(
+                server_name,
+                self.clock,
+                self.store,
             )
 
-        # Check the response.
-
-        x509_certificate_bytes = crypto.dump_certificate(
-            crypto.FILETYPE_ASN1, tls_certificate
-        )
-
-        if ("signatures" not in response
-                or server_name not in response["signatures"]):
-            raise ValueError("Key response not signed by remote server")
-
-        if "tls_certificate" not in response:
-            raise ValueError("Key response missing TLS certificate")
+            with limiter:
+                (response, tls_certificate) = yield fetch_server_key(
+                    server_name, self.hs.tls_context_factory
+                )
 
-        tls_certificate_b64 = response["tls_certificate"]
+            # Check the response.
 
-        if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
-            raise ValueError("TLS certificate doesn't match")
+            x509_certificate_bytes = crypto.dump_certificate(
+                crypto.FILETYPE_ASN1, tls_certificate
+            )
 
-        verify_keys = {}
-        for key_id, key_base64 in response["verify_keys"].items():
-            if is_signing_algorithm_supported(key_id):
-                key_bytes = decode_base64(key_base64)
-                verify_key = decode_verify_key_bytes(key_id, key_bytes)
-                verify_keys[key_id] = verify_key
+            if ("signatures" not in response
+                    or server_name not in response["signatures"]):
+                raise ValueError("Key response not signed by remote server")
+
+            if "tls_certificate" not in response:
+                raise ValueError("Key response missing TLS certificate")
+
+            tls_certificate_b64 = response["tls_certificate"]
+
+            if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
+                raise ValueError("TLS certificate doesn't match")
+
+            verify_keys = {}
+            for key_id, key_base64 in response["verify_keys"].items():
+                if is_signing_algorithm_supported(key_id):
+                    key_bytes = decode_base64(key_base64)
+                    verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                    verify_keys[key_id] = verify_key
+
+            for key_id in response["signatures"][server_name]:
+                if key_id not in response["verify_keys"]:
+                    raise ValueError(
+                        "Key response must include verification keys for all"
+                        " signatures"
+                    )
+                if key_id in verify_keys:
+                    verify_signed_json(
+                        response,
+                        server_name,
+                        verify_keys[key_id]
+                    )
+
+            # Cache the result in the datastore.
+
+            time_now_ms = self.clock.time_msec()
+
+            yield self.store.store_server_certificate(
+                server_name,
+                server_name,
+                time_now_ms,
+                tls_certificate,
+            )
 
-        for key_id in response["signatures"][server_name]:
-            if key_id not in response["verify_keys"]:
-                raise ValueError(
-                    "Key response must include verification keys for all"
-                    " signatures"
-                )
-            if key_id in verify_keys:
-                verify_signed_json(
-                    response,
-                    server_name,
-                    verify_keys[key_id]
+            for key_id, key in verify_keys.items():
+                yield self.store.store_server_verify_key(
+                    server_name, server_name, time_now_ms, key
                 )
 
-        # Cache the result in the datastore.
+            for key_id in key_ids:
+                if key_id in verify_keys:
+                    defer.returnValue(verify_keys[key_id])
+                    return
 
-        time_now_ms = self.clock.time_msec()
+            raise ValueError("No verification key found for given key ids")
 
-        yield self.store.store_server_certificate(
-            server_name,
-            server_name,
-            time_now_ms,
-            tls_certificate,
-        )
+        download = self.key_downloads.get(server_name)
 
-        for key_id, key in verify_keys.items():
-            yield self.store.store_server_verify_key(
-                server_name, server_name, time_now_ms, key
-            )
+        if download is None:
+            download = fetch_keys()
+            self.key_downloads[server_name] = download
 
-        for key_id in key_ids:
-            if key_id in verify_keys:
-                defer.returnValue(verify_keys[key_id])
-                return
+            @download.addBoth
+            def callback(ret):
+                del self.key_downloads[server_name]
+                return ret
 
-        raise ValueError("No verification key found for given key ids")
+        r = yield create_observer(download)
+        defer.returnValue(r)
diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index edd4f78024..08c8d75af4 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -25,6 +25,8 @@ from twisted.internet import defer
 from twisted.web.resource import Resource
 from twisted.protocols.basic import FileSender
 
+from synapse.util.async import create_observer
+
 import os
 
 import logging
@@ -87,7 +89,7 @@ class BaseMediaResource(Resource):
             def callback(media_info):
                 del self.downloads[key]
                 return media_info
-        return download
+        return create_observer(download)
 
     @defer.inlineCallbacks
     def _get_remote_media_impl(self, server_name, media_id):
-- 
cgit 1.4.1


From 0a016b0525c918927c8134d5cb11d9be520a9efc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 27 Apr 2015 14:37:24 +0100
Subject: Pull inner function out.

---
 synapse/crypto/keyring.py | 153 +++++++++++++++++++++++-----------------------
 1 file changed, 77 insertions(+), 76 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index d98341f5c2..14f8f536e4 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -101,92 +101,93 @@ class Keyring(object):
             defer.returnValue(cached[0])
             return
 
-        @defer.inlineCallbacks
-        def fetch_keys():
-            # Try to fetch the key from the remote server.
-
-            limiter = yield get_retry_limiter(
-                server_name,
-                self.clock,
-                self.store,
-            )
+        download = self.key_downloads.get(server_name)
 
-            with limiter:
-                (response, tls_certificate) = yield fetch_server_key(
-                    server_name, self.hs.tls_context_factory
-                )
+        if download is None:
+            download = self._get_server_verify_key_impl(server_name, key_ids)
+            self.key_downloads[server_name] = download
+
+            @download.addBoth
+            def callback(ret):
+                del self.key_downloads[server_name]
+                return ret
+
+        r = yield create_observer(download)
+        defer.returnValue(r)
 
-            # Check the response.
 
-            x509_certificate_bytes = crypto.dump_certificate(
-                crypto.FILETYPE_ASN1, tls_certificate
+    @defer.inlineCallbacks
+    def _get_server_verify_key_impl(self, server_name, key_ids):
+        # Try to fetch the key from the remote server.
+
+        limiter = yield get_retry_limiter(
+            server_name,
+            self.clock,
+            self.store,
+        )
+
+        with limiter:
+            (response, tls_certificate) = yield fetch_server_key(
+                server_name, self.hs.tls_context_factory
             )
 
-            if ("signatures" not in response
-                    or server_name not in response["signatures"]):
-                raise ValueError("Key response not signed by remote server")
-
-            if "tls_certificate" not in response:
-                raise ValueError("Key response missing TLS certificate")
-
-            tls_certificate_b64 = response["tls_certificate"]
-
-            if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
-                raise ValueError("TLS certificate doesn't match")
-
-            verify_keys = {}
-            for key_id, key_base64 in response["verify_keys"].items():
-                if is_signing_algorithm_supported(key_id):
-                    key_bytes = decode_base64(key_base64)
-                    verify_key = decode_verify_key_bytes(key_id, key_bytes)
-                    verify_keys[key_id] = verify_key
-
-            for key_id in response["signatures"][server_name]:
-                if key_id not in response["verify_keys"]:
-                    raise ValueError(
-                        "Key response must include verification keys for all"
-                        " signatures"
-                    )
-                if key_id in verify_keys:
-                    verify_signed_json(
-                        response,
-                        server_name,
-                        verify_keys[key_id]
-                    )
-
-            # Cache the result in the datastore.
-
-            time_now_ms = self.clock.time_msec()
-
-            yield self.store.store_server_certificate(
-                server_name,
-                server_name,
-                time_now_ms,
-                tls_certificate,
-            )
+        # Check the response.
+
+        x509_certificate_bytes = crypto.dump_certificate(
+            crypto.FILETYPE_ASN1, tls_certificate
+        )
 
-            for key_id, key in verify_keys.items():
-                yield self.store.store_server_verify_key(
-                    server_name, server_name, time_now_ms, key
+        if ("signatures" not in response
+                or server_name not in response["signatures"]):
+            raise ValueError("Key response not signed by remote server")
+
+        if "tls_certificate" not in response:
+            raise ValueError("Key response missing TLS certificate")
+
+        tls_certificate_b64 = response["tls_certificate"]
+
+        if encode_base64(x509_certificate_bytes) != tls_certificate_b64:
+            raise ValueError("TLS certificate doesn't match")
+
+        verify_keys = {}
+        for key_id, key_base64 in response["verify_keys"].items():
+            if is_signing_algorithm_supported(key_id):
+                key_bytes = decode_base64(key_base64)
+                verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                verify_keys[key_id] = verify_key
+
+        for key_id in response["signatures"][server_name]:
+            if key_id not in response["verify_keys"]:
+                raise ValueError(
+                    "Key response must include verification keys for all"
+                    " signatures"
+                )
+            if key_id in verify_keys:
+                verify_signed_json(
+                    response,
+                    server_name,
+                    verify_keys[key_id]
                 )
 
-            for key_id in key_ids:
-                if key_id in verify_keys:
-                    defer.returnValue(verify_keys[key_id])
-                    return
+        # Cache the result in the datastore.
 
-            raise ValueError("No verification key found for given key ids")
+        time_now_ms = self.clock.time_msec()
 
-        download = self.key_downloads.get(server_name)
+        yield self.store.store_server_certificate(
+            server_name,
+            server_name,
+            time_now_ms,
+            tls_certificate,
+        )
 
-        if download is None:
-            download = fetch_keys()
-            self.key_downloads[server_name] = download
+        for key_id, key in verify_keys.items():
+            yield self.store.store_server_verify_key(
+                server_name, server_name, time_now_ms, key
+            )
 
-            @download.addBoth
-            def callback(ret):
-                del self.key_downloads[server_name]
-                return ret
+        for key_id in key_ids:
+            if key_id in verify_keys:
+                defer.returnValue(verify_keys[key_id])
+                return
 
-        r = yield create_observer(download)
-        defer.returnValue(r)
+        raise ValueError("No verification key found for given key ids")
\ No newline at end of file
-- 
cgit 1.4.1


From 2c70849dc32a52157217d75298c99c4cfccce639 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 27 Apr 2015 14:38:29 +0100
Subject: Fix newlines

---
 synapse/crypto/keyring.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 14f8f536e4..2b4faee4c1 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -115,7 +115,6 @@ class Keyring(object):
         r = yield create_observer(download)
         defer.returnValue(r)
 
-
     @defer.inlineCallbacks
     def _get_server_verify_key_impl(self, server_name, key_ids):
         # Try to fetch the key from the remote server.
@@ -190,4 +189,4 @@ class Keyring(object):
                 defer.returnValue(verify_keys[key_id])
                 return
 
-        raise ValueError("No verification key found for given key ids")
\ No newline at end of file
+        raise ValueError("No verification key found for given key ids")
-- 
cgit 1.4.1


From 46d200a3a15352e45e167f8cecaca6631c03eea1 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 29 Apr 2015 11:57:26 +0100
Subject: Implement minimum_valid_until_ts in the remote key resource

---
 synapse/crypto/keyring.py                  |  1 +
 synapse/rest/key/v2/remote_key_resource.py | 59 ++++++++++++++++++++++++++++--
 2 files changed, 56 insertions(+), 4 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 0d24aa7ac2..bfe6e61602 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -289,6 +289,7 @@ class Keyring(object):
                 key_base64 = key_data["key"]
                 key_bytes = decode_base64(key_base64)
                 verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                verify_key.time_added = time_now_ms
                 verify_keys[key_id] = verify_key
 
         old_verify_keys = {}
diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py
index 69bc15ba75..e434847b45 100644
--- a/synapse/rest/key/v2/remote_key_resource.py
+++ b/synapse/rest/key/v2/remote_key_resource.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from synapse.http.server import request_handler, respond_with_json_bytes
+from synapse.http.servlet import parse_integer
 from synapse.api.errors import SynapseError, Codes
 
 from twisted.web.resource import Resource
@@ -44,7 +45,13 @@ class RemoteKey(Resource):
     POST /_matrix/v2/query HTTP/1.1
     Content-Type: application/json
     {
-        "server_keys": { "remote.server.example.com": ["a.key.id"] }
+        "server_keys": {
+            "remote.server.example.com": {
+                "a.key.id": {
+                    "minimum_valid_until_ts": 1234567890123
+                }
+            }
+        }
     }
 
     Response:
@@ -96,10 +103,16 @@ class RemoteKey(Resource):
     def async_render_GET(self, request):
         if len(request.postpath) == 1:
             server, = request.postpath
-            query = {server: [None]}
+            query = {server: {}}
         elif len(request.postpath) == 2:
             server, key_id = request.postpath
-            query = {server: [key_id]}
+            minimum_valid_until_ts = parse_integer(
+                request, "minimum_valid_until_ts"
+            )
+            arguments = {}
+            if minimum_valid_until_ts is not None:
+                arguments["minimum_valid_until_ts"] = minimum_valid_until_ts
+            query = {server: {key_id: arguments}}
         else:
             raise SynapseError(
                 404, "Not found %r" % request.postpath, Codes.NOT_FOUND
@@ -128,8 +141,11 @@ class RemoteKey(Resource):
 
     @defer.inlineCallbacks
     def query_keys(self, request, query, query_remote_on_cache_miss=False):
+        logger.info("Handling query for keys %r", query)
         store_queries = []
         for server_name, key_ids in query.items():
+            if not key_ids:
+                key_ids = (None,)
             for key_id in key_ids:
                 store_queries.append((server_name, key_id, None))
 
@@ -152,9 +168,44 @@ class RemoteKey(Resource):
             if key_id is not None:
                 ts_added_ms, most_recent_result = max(results)
                 ts_valid_until_ms = most_recent_result["ts_valid_until_ms"]
-                if (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms:
+                req_key = query.get(server_name, {}).get(key_id, {})
+                req_valid_until = req_key.get("minimum_valid_until_ts")
+                miss = False
+                if req_valid_until is not None:
+                    if ts_valid_until_ms < req_valid_until:
+                        logger.debug(
+                            "Cached response for %r/%r is older than requested"
+                            ": valid_until (%r) < minimum_valid_until (%r)",
+                            server_name, key_id,
+                            ts_valid_until_ms, req_valid_until
+                        )
+                        miss = True
+                    else:
+                        logger.debug(
+                            "Cached response for %r/%r is newer than requested"
+                            ": valid_until (%r) >= minimum_valid_until (%r)",
+                            server_name, key_id,
+                            ts_valid_until_ms, req_valid_until
+                        )
+                elif (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms:
+                    logger.debug(
+                        "Cached response for %r/%r is too old"
+                        ": (added (%r) + valid_until (%r)) / 2 < now (%r)",
+                        server_name, key_id,
+                        ts_added_ms, ts_valid_until_ms, time_now_ms
+                    )
                     # We more than half way through the lifetime of the
                     # response. We should fetch a fresh copy.
+                    miss = True
+                else:
+                    logger.debug(
+                        "Cached response for %r/%r is still valid"
+                        ": (added (%r) + valid_until (%r)) / 2 < now (%r)",
+                        server_name, key_id,
+                        ts_added_ms, ts_valid_until_ms, time_now_ms
+                    )
+
+                if miss:
                     cache_misses.setdefault(server_name, set()).add(key_id)
                 json_results.add(bytes(most_recent_result["key_json"]))
             else:
-- 
cgit 1.4.1


From 74874ffda7dd4c72cf723d1f5bce757a852bfcb6 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 29 Apr 2015 12:14:08 +0100
Subject: Update the query format used by keyring to match current key v2 spec

---
 synapse/crypto/keyring.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index bfe6e61602..078361fa85 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -171,10 +171,21 @@ class Keyring(object):
         )
 
         with limiter:
+            # TODO(mark): Set the minimum_valid_until_ts to that needed by
+            # the events being validated or the current time if validating
+            # an incoming request.
             responses = yield self.client.post_json(
                 destination=perspective_name,
                 path=b"/_matrix/key/v2/query",
-                data={u"server_keys": {server_name: list(key_ids)}},
+                data={
+                    u"server_keys": {
+                        server_name: {
+                            key_id: {
+                                u"minimum_valid_until_ts": 0
+                            } for key_id in key_ids
+                        }
+                    }
+                },
             )
 
         keys = {}
-- 
cgit 1.4.1


From 1319905d7af955e7790eb6072dbf4222674be89e Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 29 Apr 2015 13:31:14 +0100
Subject: Use a defer.gatherResults to collect results from the perspective
 servers

---
 synapse/crypto/keyring.py | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

(limited to 'synapse/crypto/keyring.py')

diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index 078361fa85..8709394b97 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -124,18 +124,28 @@ class Keyring(object):
     @defer.inlineCallbacks
     def _get_server_verify_key_impl(self, server_name, key_ids):
         keys = None
+
+        perspective_results = []
         for perspective_name, perspective_keys in self.perspective_servers.items():
-            try:
-                keys = yield self.get_server_verify_key_v2_indirect(
-                    server_name, key_ids, perspective_name, perspective_keys
-                )
-                break
-            except:
-                logging.info(
-                    "Unable to getting key %r for %r from %r",
-                    key_ids, server_name, perspective_name,
-                )
-                pass
+            @defer.inlineCallbacks
+            def get_key():
+                try:
+                    result = yield self.get_server_verify_key_v2_indirect(
+                        server_name, key_ids, perspective_name, perspective_keys
+                    )
+                    defer.returnValue(result)
+                except:
+                    logging.info(
+                        "Unable to getting key %r for %r from %r",
+                        key_ids, server_name, perspective_name,
+                    )
+            perspective_results.append(get_key())
+
+        perspective_results = yield defer.gatherResults(perspective_results)
+
+        for results in perspective_results:
+            if results is not None:
+                keys = results
 
         limiter = yield get_retry_limiter(
             server_name,
-- 
cgit 1.4.1