From 3dac27a8a9846a892284971f71e05c2440225484 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 14 Oct 2014 14:54:26 +0100
Subject: Storage for pdu signatures

---
 synapse/storage/schema/signatures.sql | 36 ++++++++++++++
 synapse/storage/signatures.py         | 90 +++++++++++++++++++++++++++++++++++
 2 files changed, 126 insertions(+)
 create mode 100644 synapse/storage/schema/signatures.sql
 create mode 100644 synapse/storage/signatures.py

(limited to 'synapse/storage')

diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql
new file mode 100644
index 0000000000..ba3bbb5471
--- /dev/null
+++ b/synapse/storage/schema/signatures.sql
@@ -0,0 +1,36 @@
+/* Copyright 2014 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS pdu_hashes (
+  pdu_id TEXT,
+  origin TEXT,
+  algorithm TEXT,
+  hash BLOB,
+  CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS pdu_hashes_id ON pdu_hashes (pdu_id, origin);
+
+CREATE TABLE IF NOT EXISTS pdu_origin_signatures (
+  pdu_id TEXT,
+  origin TEXT,
+  key_id TEXT,
+  signature BLOB,
+  CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures (
+    pdu_id, origin,
+);
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
new file mode 100644
index 0000000000..bb860f09f0
--- /dev/null
+++ b/synapse/storage/signatures.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from _base import SQLBaseStore
+
+from twisted.internet import defer
+
+
+class SignatureStore(SQLBaseStore):
+    """Persistence for PDU signatures and hashes"""
+
+    def _get_pdu_hashes_txn(self, txn, pdu_id, origin):
+        """Get all the hashes for a given PDU.
+        Args:
+            txn (cursor):
+            pdu_id (str): Id for the PDU.
+            origin (str): origin of the PDU.
+        Returns:
+            A dict of algorithm -> hash.
+        """
+        query = (
+            "SELECT algorithm, hash"
+            " FROM pdu_hashes"
+            " WHERE pdu_id = ? and origin = ?"
+        )
+        txn.execute(query, (pdu_id, origin))
+        return dict(txn.fetchall())
+
+    def _store_pdu_hash_txn(self, txn, pdu_id, origin, algorithm, hash_bytes):
+        """Store a hash for a PDU
+        Args:
+            txn (cursor):
+            pdu_id (str): Id for the PDU.
+            origin (str): origin of the PDU.
+            algorithm (str): Hashing algorithm.
+            hash_bytes (bytes): Hash function output bytes.
+        """
+        self._simple_insert_txn(self, txn, "pdu_hashes", {
+            "pdu_id": pdu_id,
+            "origin": origin,
+            "algorithm": algorithm,
+            "hash": buffer(hash_bytes),
+        })
+
+    def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin):
+        """Get all the signatures for a given PDU.
+        Args:
+            txn (cursor):
+            pdu_id (str): Id for the PDU.
+            origin (str): origin of the PDU.
+        Returns:
+            A dict of key_id -> signature_bytes.
+        """
+        query = (
+            "SELECT key_id, signature"
+            " FROM pdu_origin_signatures"
+            " WHERE WHERE pdu_id = ? and origin = ?"
+        )
+        txn.execute(query, (pdu_id, origin))
+        return dict(txn.fetchall())
+
+    def _store_pdu_origin_signature_txn(self, txn, pdu_id, origin, key_id,
+                                        signature_bytes):
+        """Store a signature from the origin server for a PDU.
+        Args:
+            txn (cursor):
+            pdu_id (str): Id for the PDU.
+            origin (str): origin of the PDU.
+            key_id (str): Id for the signing key.
+            signature (bytes): The signature.
+        """
+        self._simple_insert_txn(self, txn, "pdu_origin_signatures", {
+            "pdu_id": pdu_id,
+            "origin": origin,
+            "key_id": key_id,
+            "signature": buffer(signature_bytes),
+        })
+
-- 
cgit 1.4.1


From 1c445f88f64beabf0bd9bec3950a4a4c0d529e8a Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 15 Oct 2014 17:09:04 +0100
Subject: persist hashes and origin signatures for PDUs

---
 synapse/api/events/utils.py           | 23 ++++++++----
 synapse/crypto/event_signing.py       | 70 +++++++++++++++++++++++++++++++++++
 synapse/federation/units.py           | 17 ++++++++-
 synapse/storage/__init__.py           | 21 ++++++++++-
 synapse/storage/pdu.py                | 11 +++++-
 synapse/storage/schema/signatures.sql |  4 +-
 tests/federation/test_federation.py   |  4 +-
 7 files changed, 135 insertions(+), 15 deletions(-)
 create mode 100644 synapse/crypto/event_signing.py

(limited to 'synapse/storage')

diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py
index c3a32be8c1..7fdf45a264 100644
--- a/synapse/api/events/utils.py
+++ b/synapse/api/events/utils.py
@@ -27,7 +27,14 @@ def prune_event(event):
     the user has specified, but we do want to keep necessary information like
     type, state_key etc.
     """
+    return _prune_event_or_pdu(event.type, event)
 
+def prune_pdu(pdu):
+    """Removes keys that contain unrestricted and non-essential data from a PDU
+    """
+    return _prune_event_or_pdu(pdu.pdu_type, pdu)
+
+def _prune_event_or_pdu(event_type, event):
     # Remove all extraneous fields.
     event.unrecognized_keys = {}
 
@@ -38,25 +45,25 @@ def prune_event(event):
             if field in event.content:
                 new_content[field] = event.content[field]
 
-    if event.type == RoomMemberEvent.TYPE:
+    if event_type == RoomMemberEvent.TYPE:
         add_fields("membership")
-    elif event.type == RoomCreateEvent.TYPE:
+    elif event_type == RoomCreateEvent.TYPE:
         add_fields("creator")
-    elif event.type == RoomJoinRulesEvent.TYPE:
+    elif event_type == RoomJoinRulesEvent.TYPE:
         add_fields("join_rule")
-    elif event.type == RoomPowerLevelsEvent.TYPE:
+    elif event_type == RoomPowerLevelsEvent.TYPE:
         # TODO: Actually check these are valid user_ids etc.
         add_fields("default")
         for k, v in event.content.items():
             if k.startswith("@") and isinstance(v, (int, long)):
                 new_content[k] = v
-    elif event.type == RoomAddStateLevelEvent.TYPE:
+    elif event_type == RoomAddStateLevelEvent.TYPE:
         add_fields("level")
-    elif event.type == RoomSendEventLevelEvent.TYPE:
+    elif event_type == RoomSendEventLevelEvent.TYPE:
         add_fields("level")
-    elif event.type == RoomOpsPowerLevelsEvent.TYPE:
+    elif event_type == RoomOpsPowerLevelsEvent.TYPE:
         add_fields("kick_level", "ban_level", "redact_level")
-    elif event.type == RoomAliasesEvent.TYPE:
+    elif event_type == RoomAliasesEvent.TYPE:
         add_fields("aliases")
 
     event.content = new_content
diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py
new file mode 100644
index 0000000000..6557727e06
--- /dev/null
+++ b/synapse/crypto/event_signing.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from synapse.api.events.utils import prune_pdu
+from syutil.jsonutil import encode_canonical_json
+from syutil.base64util import encode_base64, decode_base64
+from syutil.crypto.jsonsign import sign_json, verify_signed_json
+
+import hashlib
+
+
+def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256):
+    hashed = _compute_hash(pdu, hash_algortithm)
+    hashes[hashed.name] = encode_base64(hashed.digest())
+    pdu.hashes = hashes
+    return pdu
+
+
+def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256):
+    """Check whether the hash for this PDU matches the contents"""
+    computed_hash = _compute_hash(pdu, hash_algortithm)
+    if computed_hash.name not in pdu.hashes:
+        raise Exception("Algorithm %s not in hashes %s" % (
+            computed_hash.name, list(pdu.hashes)
+        ))
+    message_hash_base64 = hashes[computed_hash.name]
+    try:
+        message_hash_bytes = decode_base64(message_hash_base64)
+    except:
+        raise Exception("Invalid base64: %s" % (message_hash_base64,))
+    return message_hash_bytes == computed_hash.digest()
+
+
+def _compute_hash(pdu, hash_algorithm):
+    pdu_json = pdu.get_dict()
+    pdu_json.pop("meta", None)
+    pdu_json.pop("signatures", None)
+    hashes = pdu_json.pop("hashes", {})
+    pdu_json_bytes = encode_canonical_json(pdu_json)
+    return hash_algorithm(pdu_json_bytes)
+
+
+def sign_event_pdu(pdu, signature_name, signing_key):
+    tmp_pdu = Pdu(**pdu.get_dict())
+    tmp_pdu = prune_pdu(tmp_pdu)
+    pdu_json = tmp_pdu.get_dict()
+    pdu_jdon = sign_json(pdu_json, signature_name, signing_key)
+    pdu.signatures = pdu_json["signatures"]
+    return pdu
+
+
+def verify_signed_event_pdu(pdu, signature_name, verify_key):
+    tmp_pdu = Pdu(**pdu.get_dict())
+    tmp_pdu = prune_pdu(tmp_pdu)
+    pdu_json = tmp_pdu.get_dict()
+    verify_signed_json(pdu_json, signature_name, verify_key)
diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index d97aeb698e..3518efb215 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -18,6 +18,7 @@ server protocol.
 """
 
 from synapse.util.jsonobject import JsonEncodedObject
+from syutil.base64util import encode_base64
 
 import logging
 import json
@@ -63,6 +64,8 @@ class Pdu(JsonEncodedObject):
         "depth",
         "content",
         "outlier",
+        "hashes",
+        "signatures",
         "is_state",  # Below this are keys valid only for State Pdus.
         "state_key",
         "power_level",
@@ -91,7 +94,7 @@ class Pdu(JsonEncodedObject):
     # just leaving it as a dict. (OR DO WE?!)
 
     def __init__(self, destinations=[], is_state=False, prev_pdus=[],
-                 outlier=False, **kwargs):
+                 outlier=False, hashes={}, signatures={}, **kwargs):
         if is_state:
             for required_key in ["state_key"]:
                 if required_key not in kwargs:
@@ -102,6 +105,8 @@ class Pdu(JsonEncodedObject):
             is_state=is_state,
             prev_pdus=prev_pdus,
             outlier=outlier,
+            hashes=hashes,
+            signatures=signatures,
             **kwargs
         )
 
@@ -126,6 +131,16 @@ class Pdu(JsonEncodedObject):
             if "unrecognized_keys" in d and d["unrecognized_keys"]:
                 args.update(json.loads(d["unrecognized_keys"]))
 
+            hashes = {
+                alg: encode_base64(hsh)
+                for alg, hsh in pdu_tuple.hashes.items()
+            }
+
+            signatures = {
+                kid: encode_base64(sig)
+                for kid, sig in pdu_tuple.signatures.items()
+            }
+
             return Pdu(
                 prev_pdus=pdu_tuple.prev_pdu_list,
                 **args
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 6dadeb8cce..bfeab7d1e8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -40,6 +40,8 @@ from .stream import StreamStore
 from .pdu import StatePduStore, PduStore, PdusTable
 from .transactions import TransactionStore
 from .keys import KeyStore
+from .signatures import SignatureStore
+
 
 import json
 import logging
@@ -59,6 +61,7 @@ SCHEMAS = [
     "room_aliases",
     "keys",
     "redactions",
+    "signatures",
 ]
 
 
@@ -76,7 +79,7 @@ class _RollbackButIsFineException(Exception):
 class DataStore(RoomMemberStore, RoomStore,
                 RegistrationStore, StreamStore, ProfileStore, FeedbackStore,
                 PresenceStore, PduStore, StatePduStore, TransactionStore,
-                DirectoryStore, KeyStore):
+                DirectoryStore, KeyStore, SignatureStore):
 
     def __init__(self, hs):
         super(DataStore, self).__init__(hs)
@@ -144,6 +147,8 @@ class DataStore(RoomMemberStore, RoomStore,
     def _persist_event_pdu_txn(self, txn, pdu):
         cols = dict(pdu.__dict__)
         unrec_keys = dict(pdu.unrecognized_keys)
+        del cols["hashes"]
+        del cols["signatures"]
         del cols["content"]
         del cols["prev_pdus"]
         cols["content_json"] = json.dumps(pdu.content)
@@ -157,6 +162,20 @@ class DataStore(RoomMemberStore, RoomStore,
 
         logger.debug("Persisting: %s", repr(cols))
 
+        for hash_alg, hash_base64 in pdu.hashes.items():
+            hash_bytes = decode_base64(hash_base64)
+            self._store_pdu_hash_txn(
+                txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes,
+            )
+
+        signatures = pdu.sigatures.get(pdu.orgin, {})
+
+        for key_id, signature_base64 in signatures:
+            signature_bytes = decode_base64(signature_base64)
+            self.store_pdu_origin_signatures_txn(
+                txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes,
+            )
+
         if pdu.is_state:
             self._persist_state_txn(txn, pdu.prev_pdus, cols)
         else:
diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py
index d70467dcd6..9d624429b7 100644
--- a/synapse/storage/pdu.py
+++ b/synapse/storage/pdu.py
@@ -64,6 +64,11 @@ class PduStore(SQLBaseStore):
                 for r in PduEdgesTable.decode_results(txn.fetchall())
             ]
 
+            hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin)
+            signatures = self._get_pdu_origin_signatures_txn(
+                txn, pdu_id, origin
+            )
+
             query = (
                 "SELECT %(fields)s FROM %(pdus)s as p "
                 "LEFT JOIN %(state)s as s "
@@ -80,7 +85,9 @@ class PduStore(SQLBaseStore):
 
             row = txn.fetchone()
             if row:
-                results.append(PduTuple(PduEntry(*row), edges))
+                results.append(PduTuple(
+                    PduEntry(*row), edges, hashes, signatures
+                ))
 
         return results
 
@@ -908,7 +915,7 @@ This does not include a prev_pdus key.
 
 PduTuple = namedtuple(
     "PduTuple",
-    ("pdu_entry", "prev_pdu_list")
+    ("pdu_entry", "prev_pdu_list", "hashes", "signatures")
 )
 """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent
 the `prev_pdus` key of a PDU.
diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql
index ba3bbb5471..86ee0f2377 100644
--- a/synapse/storage/schema/signatures.sql
+++ b/synapse/storage/schema/signatures.sql
@@ -28,9 +28,9 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures (
   origin TEXT,
   key_id TEXT,
   signature BLOB,
-  CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm)
+  CONSTRAINT uniqueness UNIQUE (pdu_id, origin, key_id)
 );
 
 CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures (
-    pdu_id, origin,
+    pdu_id, origin
 );
diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py
index d86ce83b28..03b2167cf7 100644
--- a/tests/federation/test_federation.py
+++ b/tests/federation/test_federation.py
@@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs):
     }
     pdu_fields.update(kwargs)
 
-    return PduTuple(PduEntry(**pdu_fields), prev_pdus)
+    return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {})
 
 
 class FederationTestCase(unittest.TestCase):
@@ -183,6 +183,8 @@ class FederationTestCase(unittest.TestCase):
                             "is_state": False,
                             "content": {"testing": "content here"},
                             "depth": 1,
+                            "hashes": {},
+                            "signatures": {},
                         },
                     ]
                 },
-- 
cgit 1.4.1


From 66104da10c4191aa1e048f2379190574755109e6 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 16 Oct 2014 00:09:48 +0100
Subject: Sign outgoing PDUs.

---
 synapse/crypto/event_signing.py    |  4 ++--
 synapse/federation/pdu_codec.py    |  6 +++++-
 synapse/storage/__init__.py        |  7 ++++---
 synapse/storage/signatures.py      |  6 +++---
 tests/federation/test_pdu_codec.py | 13 ++++++++++---
 tests/rest/test_events.py          |  7 +++++--
 tests/rest/test_profile.py         |  8 ++++++--
 tests/rest/test_rooms.py           | 32 +++++++++++++++++++++++++-------
 tests/utils.py                     |  3 ++-
 9 files changed, 62 insertions(+), 24 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py
index 6557727e06..a115967c0a 100644
--- a/synapse/crypto/event_signing.py
+++ b/synapse/crypto/event_signing.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 
+from synapse.federation.units import Pdu
 from synapse.api.events.utils import prune_pdu
 from syutil.jsonutil import encode_canonical_json
 from syutil.base64util import encode_base64, decode_base64
@@ -25,8 +26,7 @@ import hashlib
 
 def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256):
     hashed = _compute_hash(pdu, hash_algortithm)
-    hashes[hashed.name] = encode_base64(hashed.digest())
-    pdu.hashes = hashes
+    pdu.hashes[hashed.name] = encode_base64(hashed.digest())
     return pdu
 
 
diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py
index cef61108dd..bcac5f9ae8 100644
--- a/synapse/federation/pdu_codec.py
+++ b/synapse/federation/pdu_codec.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from .units import Pdu
+from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu
 
 import copy
 
@@ -33,6 +34,7 @@ def encode_event_id(pdu_id, origin):
 class PduCodec(object):
 
     def __init__(self, hs):
+        self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
         self.event_factory = hs.get_event_factory()
         self.clock = hs.get_clock()
@@ -99,4 +101,6 @@ class PduCodec(object):
         if "ts" not in kwargs:
             kwargs["ts"] = int(self.clock.time_msec())
 
-        return Pdu(**kwargs)
+        pdu = Pdu(**kwargs)
+        pdu = hash_event_pdu(pdu)
+        return sign_event_pdu(pdu, self.server_name, self.signing_key)
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index bfeab7d1e8..b2a3f0b56c 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -42,6 +42,7 @@ from .transactions import TransactionStore
 from .keys import KeyStore
 from .signatures import SignatureStore
 
+from syutil.base64util import decode_base64
 
 import json
 import logging
@@ -168,11 +169,11 @@ class DataStore(RoomMemberStore, RoomStore,
                 txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes,
             )
 
-        signatures = pdu.sigatures.get(pdu.orgin, {})
+        signatures = pdu.signatures.get(pdu.origin, {})
 
-        for key_id, signature_base64 in signatures:
+        for key_id, signature_base64 in signatures.items():
             signature_bytes = decode_base64(signature_base64)
-            self.store_pdu_origin_signatures_txn(
+            self._store_pdu_origin_signature_txn(
                 txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes,
             )
 
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index bb860f09f0..1f0a680500 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -47,7 +47,7 @@ class SignatureStore(SQLBaseStore):
             algorithm (str): Hashing algorithm.
             hash_bytes (bytes): Hash function output bytes.
         """
-        self._simple_insert_txn(self, txn, "pdu_hashes", {
+        self._simple_insert_txn(txn, "pdu_hashes", {
             "pdu_id": pdu_id,
             "origin": origin,
             "algorithm": algorithm,
@@ -66,7 +66,7 @@ class SignatureStore(SQLBaseStore):
         query = (
             "SELECT key_id, signature"
             " FROM pdu_origin_signatures"
-            " WHERE WHERE pdu_id = ? and origin = ?"
+            " WHERE pdu_id = ? and origin = ?"
         )
         txn.execute(query, (pdu_id, origin))
         return dict(txn.fetchall())
@@ -81,7 +81,7 @@ class SignatureStore(SQLBaseStore):
             key_id (str): Id for the signing key.
             signature (bytes): The signature.
         """
-        self._simple_insert_txn(self, txn, "pdu_origin_signatures", {
+        self._simple_insert_txn(txn, "pdu_origin_signatures", {
             "pdu_id": pdu_id,
             "origin": origin,
             "key_id": key_id,
diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py
index 344e1baf60..80851a4258 100644
--- a/tests/federation/test_pdu_codec.py
+++ b/tests/federation/test_pdu_codec.py
@@ -23,14 +23,21 @@ from synapse.federation.units import Pdu
 
 from synapse.server import HomeServer
 
-from mock import Mock
+from mock import Mock, NonCallableMock
+
+from ..utils import MockKey
 
 
 class PduCodecTestCase(unittest.TestCase):
     def setUp(self):
-        self.hs = HomeServer("blargle.net")
-        self.event_factory = self.hs.get_event_factory()
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
+        self.hs = HomeServer(
+            "blargle.net",
+            config=self.mock_config,
+        )
+        self.event_factory = self.hs.get_event_factory()
         self.codec = PduCodec(self.hs)
 
     def test_decode_event_id(self):
diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py
index 79b371c04d..362c7bc01c 100644
--- a/tests/rest/test_events.py
+++ b/tests/rest/test_events.py
@@ -28,7 +28,7 @@ from synapse.server import HomeServer
 # python imports
 import json
 
-from ..utils import MockHttpResource, MemoryDataStore
+from ..utils import MockHttpResource, MemoryDataStore, MockKey
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -122,6 +122,9 @@ class EventStreamPermissionsTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "test",
             db_pool=None,
@@ -139,7 +142,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py
index b0f48e7fd8..3a0d1e700a 100644
--- a/tests/rest/test_profile.py
+++ b/tests/rest/test_profile.py
@@ -18,9 +18,9 @@
 from tests import unittest
 from twisted.internet import defer
 
-from mock import Mock
+from mock import Mock, NonCallableMock
 
-from ..utils import MockHttpResource
+from ..utils import MockHttpResource, MockKey
 
 from synapse.api.errors import SynapseError, AuthError
 from synapse.server import HomeServer
@@ -41,6 +41,9 @@ class ProfileTestCase(unittest.TestCase):
             "set_avatar_url",
         ])
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer("test",
             db_pool=None,
             http_client=None,
@@ -48,6 +51,7 @@ class ProfileTestCase(unittest.TestCase):
             federation=Mock(),
             replication_layer=Mock(),
             datastore=None,
+            config=self.mock_config,
         )
 
         def _get_user_by_req(request=None):
diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py
index 1ce9b8a83d..7170193051 100644
--- a/tests/rest/test_rooms.py
+++ b/tests/rest/test_rooms.py
@@ -27,7 +27,7 @@ from synapse.server import HomeServer
 import json
 import urllib
 
-from ..utils import MockHttpResource, MemoryDataStore
+from ..utils import MockHttpResource, MemoryDataStore, MockKey
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -50,6 +50,9 @@ class RoomPermissionsTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "red",
             db_pool=None,
@@ -61,7 +64,7 @@ class RoomPermissionsTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -408,6 +411,9 @@ class RoomsMemberListTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "red",
             db_pool=None,
@@ -419,7 +425,7 @@ class RoomsMemberListTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -497,6 +503,9 @@ class RoomsCreateTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "red",
             db_pool=None,
@@ -508,7 +517,7 @@ class RoomsCreateTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -598,6 +607,9 @@ class RoomTopicTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "red",
             db_pool=None,
@@ -609,7 +621,7 @@ class RoomTopicTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -712,6 +724,9 @@ class RoomMemberStateTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "red",
             db_pool=None,
@@ -723,7 +738,7 @@ class RoomMemberStateTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -853,6 +868,9 @@ class RoomMessagesTestCase(RestTestCase):
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             "red",
             db_pool=None,
@@ -864,7 +882,7 @@ class RoomMessagesTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
diff --git a/tests/utils.py b/tests/utils.py
index 60fd6085ac..d8be73dba8 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -118,13 +118,14 @@ class MockHttpResource(HttpServer):
 class MockKey(object):
     alg = "mock_alg"
     version = "mock_version"
+    signature = b"\x9a\x87$"
 
     @property
     def verify_key(self):
         return self
 
     def sign(self, message):
-        return b"\x9a\x87$"
+        return self
 
     def verify(self, message, sig):
         assert sig == b"\x9a\x87$"
-- 
cgit 1.4.1


From bb04447c44036ebf3ae5dde7a4cc7a7909d50ef6 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 16 Oct 2014 23:25:12 +0100
Subject: Include hashes of previous pdus when referencing them

---
 synapse/api/events/__init__.py        |  2 +-
 synapse/federation/pdu_codec.py       | 13 ++++---------
 synapse/federation/replication.py     |  2 +-
 synapse/federation/units.py           | 10 +++++++++-
 synapse/state.py                      |  4 ----
 synapse/storage/__init__.py           | 20 ++++++++++++++------
 synapse/storage/pdu.py                | 22 ++++++++++++++++------
 synapse/storage/schema/signatures.sql | 16 ++++++++++++++++
 synapse/storage/signatures.py         | 31 +++++++++++++++++++++++++++++++
 tests/federation/test_federation.py   |  2 +-
 tests/federation/test_pdu_codec.py    |  4 ++--
 11 files changed, 95 insertions(+), 31 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py
index f66fea2904..a5a55742e0 100644
--- a/synapse/api/events/__init__.py
+++ b/synapse/api/events/__init__.py
@@ -65,13 +65,13 @@ class SynapseEvent(JsonEncodedObject):
 
     internal_keys = [
         "is_state",
-        "prev_events",
         "depth",
         "destinations",
         "origin",
         "outlier",
         "power_level",
         "redacted",
+        "prev_pdus",
     ]
 
     required_keys = [
diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py
index bcac5f9ae8..11fd7264b3 100644
--- a/synapse/federation/pdu_codec.py
+++ b/synapse/federation/pdu_codec.py
@@ -45,9 +45,7 @@ class PduCodec(object):
         kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin)
         kwargs["room_id"] = pdu.context
         kwargs["etype"] = pdu.pdu_type
-        kwargs["prev_events"] = [
-            encode_event_id(p[0], p[1]) for p in pdu.prev_pdus
-        ]
+        kwargs["prev_pdus"] = pdu.prev_pdus
 
         if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"):
             kwargs["prev_state"] = encode_event_id(
@@ -78,11 +76,8 @@ class PduCodec(object):
         d["context"] = event.room_id
         d["pdu_type"] = event.type
 
-        if hasattr(event, "prev_events"):
-            d["prev_pdus"] = [
-                decode_event_id(e, self.server_name)
-                for e in event.prev_events
-            ]
+        if hasattr(event, "prev_pdus"):
+            d["prev_pdus"] = event.prev_pdus
 
         if hasattr(event, "prev_state"):
             d["prev_state_id"], d["prev_state_origin"] = (
@@ -95,7 +90,7 @@ class PduCodec(object):
         kwargs = copy.deepcopy(event.unrecognized_keys)
         kwargs.update({
             k: v for k, v in d.items()
-            if k not in ["event_id", "room_id", "type", "prev_events"]
+            if k not in ["event_id", "room_id", "type"]
         })
 
         if "ts" not in kwargs:
diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py
index 9363ac7300..788a49b8e8 100644
--- a/synapse/federation/replication.py
+++ b/synapse/federation/replication.py
@@ -443,7 +443,7 @@ class ReplicationLayer(object):
             min_depth = yield self.store.get_min_depth_for_context(pdu.context)
 
             if min_depth and pdu.depth > min_depth:
-                for pdu_id, origin in pdu.prev_pdus:
+                for pdu_id, origin, hashes in pdu.prev_pdus:
                     exists = yield self._get_persisted_pdu(pdu_id, origin)
 
                     if not exists:
diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index 3518efb215..6a43007837 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -141,8 +141,16 @@ class Pdu(JsonEncodedObject):
                 for kid, sig in pdu_tuple.signatures.items()
             }
 
+            prev_pdus = []
+            for prev_pdu in pdu_tuple.prev_pdu_list:
+                prev_hashes = pdu_tuple.edge_hashes.get(prev_pdu, {})
+                prev_hashes = {
+                    alg: encode_base64(hsh) for alg, hsh in prev_hashes.items()
+                }
+                prev_pdus.append((prev_pdu[0], prev_pdu[1], prev_hashes))
+
             return Pdu(
-                prev_pdus=pdu_tuple.prev_pdu_list,
+                prev_pdus=prev_pdus,
                 **args
             )
         else:
diff --git a/synapse/state.py b/synapse/state.py
index 9db84c9b5c..bc6b928ec7 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -72,10 +72,6 @@ class StateHandler(object):
 
         snapshot.fill_out_prev_events(event)
 
-        event.prev_events = [
-            e for e in event.prev_events if e != event.event_id
-        ]
-
         current_state = snapshot.prev_state_pdu
 
         if current_state:
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b2a3f0b56c..af05b47932 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -177,6 +177,14 @@ class DataStore(RoomMemberStore, RoomStore,
                 txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes,
             )
 
+        for prev_pdu_id, prev_origin, prev_hashes in pdu.prev_pdus:
+            for alg, hash_base64 in prev_hashes.items():
+                hash_bytes = decode_base64(hash_base64)
+                self._store_prev_pdu_hash_txn(
+                    txn, pdu.pdu_id, pdu.origin, prev_pdu_id, prev_origin, alg,
+                    hash_bytes
+                )
+
         if pdu.is_state:
             self._persist_state_txn(txn, pdu.prev_pdus, cols)
         else:
@@ -352,6 +360,7 @@ class DataStore(RoomMemberStore, RoomStore,
             prev_pdus = self._get_latest_pdus_in_context(
                 txn, room_id
             )
+
             if state_type is not None and state_key is not None:
                 prev_state_pdu = self._get_current_state_pdu(
                     txn, room_id, state_type, state_key
@@ -401,17 +410,16 @@ class Snapshot(object):
         self.prev_state_pdu = prev_state_pdu
 
     def fill_out_prev_events(self, event):
-        if hasattr(event, "prev_events"):
+        if hasattr(event, "prev_pdus"):
             return
 
-        es = [
-            "%s@%s" % (p_id, origin) for p_id, origin, _ in self.prev_pdus
+        event.prev_pdus = [
+            (p_id, origin, hashes)
+            for p_id, origin, hashes, _ in self.prev_pdus
         ]
 
-        event.prev_events = [e for e in es if e != event.event_id]
-
         if self.prev_pdus:
-            event.depth = max([int(v) for _, _, v in self.prev_pdus]) + 1
+            event.depth = max([int(v) for _, _, _, v in self.prev_pdus]) + 1
         else:
             event.depth = 0
 
diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py
index 9d624429b7..a423b42dbd 100644
--- a/synapse/storage/pdu.py
+++ b/synapse/storage/pdu.py
@@ -20,10 +20,13 @@ from ._base import SQLBaseStore, Table, JoinHelper
 from synapse.federation.units import Pdu
 from synapse.util.logutils import log_function
 
+from syutil.base64util import encode_base64
+
 from collections import namedtuple
 
 import logging
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -64,6 +67,8 @@ class PduStore(SQLBaseStore):
                 for r in PduEdgesTable.decode_results(txn.fetchall())
             ]
 
+            edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin)
+
             hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin)
             signatures = self._get_pdu_origin_signatures_txn(
                 txn, pdu_id, origin
@@ -86,7 +91,7 @@ class PduStore(SQLBaseStore):
             row = txn.fetchone()
             if row:
                 results.append(PduTuple(
-                    PduEntry(*row), edges, hashes, signatures
+                    PduEntry(*row), edges, hashes, signatures, edge_hashes
                 ))
 
         return results
@@ -310,9 +315,14 @@ class PduStore(SQLBaseStore):
             (context, )
         )
 
-        results = txn.fetchall()
+        results = []
+        for pdu_id, origin, depth in txn.fetchall():
+            hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin)
+            sha256_bytes = hashes["sha256"]
+            prev_hashes = {"sha256": encode_base64(sha256_bytes)}
+            results.append((pdu_id, origin, prev_hashes, depth))
 
-        return [(row[0], row[1], row[2]) for row in results]
+        return results
 
     @defer.inlineCallbacks
     def get_oldest_pdus_in_context(self, context):
@@ -431,7 +441,7 @@ class PduStore(SQLBaseStore):
                 "DELETE FROM %s WHERE pdu_id = ? AND origin = ?"
                 % PduForwardExtremitiesTable.table_name
             )
-            txn.executemany(query, prev_pdus)
+            txn.executemany(query, list(p[:2] for p in prev_pdus))
 
             # We only insert as a forward extremety the new pdu if there are no
             # other pdus that reference it as a prev pdu
@@ -454,7 +464,7 @@ class PduStore(SQLBaseStore):
             # deleted in a second if they're incorrect anyway.
             txn.executemany(
                 PduBackwardExtremitiesTable.insert_statement(),
-                [(i, o, context) for i, o in prev_pdus]
+                [(i, o, context) for i, o, _ in prev_pdus]
             )
 
             # Also delete from the backwards extremities table all ones that
@@ -915,7 +925,7 @@ This does not include a prev_pdus key.
 
 PduTuple = namedtuple(
     "PduTuple",
-    ("pdu_entry", "prev_pdu_list", "hashes", "signatures")
+    ("pdu_entry", "prev_pdu_list", "hashes", "signatures", "edge_hashes")
 )
 """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent
 the `prev_pdus` key of a PDU.
diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql
index 86ee0f2377..a72c4dc35f 100644
--- a/synapse/storage/schema/signatures.sql
+++ b/synapse/storage/schema/signatures.sql
@@ -34,3 +34,19 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures (
 CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures (
     pdu_id, origin
 );
+
+CREATE TABLE IF NOT EXISTS pdu_edge_hashes(
+    pdu_id TEXT,
+    origin TEXT,
+    prev_pdu_id TEXT,
+    prev_origin TEXT,
+    algorithm TEXT,
+    hash BLOB,
+    CONSTRAINT uniqueness UNIQUE (
+        pdu_id, origin, prev_pdu_id, prev_origin, algorithm
+    )
+);
+
+CREATE INDEX IF NOT EXISTS pdu_edge_hashes_id ON pdu_edge_hashes(
+    pdu_id, origin
+);
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 1f0a680500..1147102489 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -88,3 +88,34 @@ class SignatureStore(SQLBaseStore):
             "signature": buffer(signature_bytes),
         })
 
+    def _get_prev_pdu_hashes_txn(self, txn, pdu_id, origin):
+        """Get all the hashes for previous PDUs of a PDU
+        Args:
+            txn (cursor):
+            pdu_id (str): Id of the PDU.
+            origin (str): Origin of the PDU.
+        Returns:
+            dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes.
+        """
+        query = (
+            "SELECT prev_pdu_id, prev_origin, algorithm, hash"
+            " FROM pdu_edge_hashes"
+            " WHERE pdu_id = ? and origin = ?"
+        )
+        txn.execute(query, (pdu_id, origin))
+        results = {}
+        for prev_pdu_id, prev_origin, algorithm, hash_bytes in txn.fetchall():
+            hashes = results.setdefault((prev_pdu_id, prev_origin), {})
+            hashes[algorithm] = hash_bytes
+        return results
+
+    def _store_prev_pdu_hash_txn(self, txn, pdu_id, origin, prev_pdu_id,
+                             prev_origin, algorithm, hash_bytes):
+        self._simple_insert_txn(txn, "pdu_edge_hashes", {
+            "pdu_id": pdu_id,
+            "origin": origin,
+            "prev_pdu_id": prev_pdu_id,
+            "prev_origin": prev_origin,
+            "algorithm": algorithm,
+            "hash": buffer(hash_bytes),
+        })
diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py
index 03b2167cf7..eed50e6335 100644
--- a/tests/federation/test_federation.py
+++ b/tests/federation/test_federation.py
@@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs):
     }
     pdu_fields.update(kwargs)
 
-    return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {})
+    return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {})
 
 
 class FederationTestCase(unittest.TestCase):
diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py
index 80851a4258..0ad8cf6641 100644
--- a/tests/federation/test_pdu_codec.py
+++ b/tests/federation/test_pdu_codec.py
@@ -88,7 +88,7 @@ class PduCodecTestCase(unittest.TestCase):
         self.assertEquals(pdu.context, event.room_id)
         self.assertEquals(pdu.is_state, event.is_state)
         self.assertEquals(pdu.depth, event.depth)
-        self.assertEquals(["alice@bob.com"], event.prev_events)
+        self.assertEquals(pdu.prev_pdus, event.prev_pdus)
         self.assertEquals(pdu.content, event.content)
 
     def test_pdu_from_event(self):
@@ -144,7 +144,7 @@ class PduCodecTestCase(unittest.TestCase):
         self.assertEquals(pdu.context, event.room_id)
         self.assertEquals(pdu.is_state, event.is_state)
         self.assertEquals(pdu.depth, event.depth)
-        self.assertEquals(["alice@bob.com"], event.prev_events)
+        self.assertEquals(pdu.prev_pdus, event.prev_pdus)
         self.assertEquals(pdu.content, event.content)
         self.assertEquals(pdu.state_key, event.state_key)
 
-- 
cgit 1.4.1


From c8f996e29ffd7055bc6521ea610fc12ff50502e5 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Fri, 17 Oct 2014 11:40:35 +0100
Subject: Hash the same content covered by the signature when referencing
 previous PDUs rather than reusing the PDU content hashes

---
 synapse/crypto/event_signing.py       | 19 +++++++++++----
 synapse/federation/pdu_codec.py       |  6 +++--
 synapse/storage/__init__.py           |  9 ++++++-
 synapse/storage/pdu.py                |  4 ++--
 synapse/storage/schema/signatures.sql | 18 ++++++++++++--
 synapse/storage/signatures.py         | 44 +++++++++++++++++++++++++++++++----
 6 files changed, 84 insertions(+), 16 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py
index a115967c0a..32d60bd30a 100644
--- a/synapse/crypto/event_signing.py
+++ b/synapse/crypto/event_signing.py
@@ -24,15 +24,15 @@ from syutil.crypto.jsonsign import sign_json, verify_signed_json
 import hashlib
 
 
-def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256):
-    hashed = _compute_hash(pdu, hash_algortithm)
+def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256):
+    hashed = _compute_content_hash(pdu, hash_algorithm)
     pdu.hashes[hashed.name] = encode_base64(hashed.digest())
     return pdu
 
 
-def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256):
+def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256):
     """Check whether the hash for this PDU matches the contents"""
-    computed_hash = _compute_hash(pdu, hash_algortithm)
+    computed_hash = _compute_content_hash(pdu, hash_algortithm)
     if computed_hash.name not in pdu.hashes:
         raise Exception("Algorithm %s not in hashes %s" % (
             computed_hash.name, list(pdu.hashes)
@@ -45,7 +45,7 @@ def check_event_pdu_hash(pdu, hash_algorithm=hashlib.sha256):
     return message_hash_bytes == computed_hash.digest()
 
 
-def _compute_hash(pdu, hash_algorithm):
+def _compute_content_hash(pdu, hash_algorithm):
     pdu_json = pdu.get_dict()
     pdu_json.pop("meta", None)
     pdu_json.pop("signatures", None)
@@ -54,6 +54,15 @@ def _compute_hash(pdu, hash_algorithm):
     return hash_algorithm(pdu_json_bytes)
 
 
+def compute_pdu_event_reference_hash(pdu, hash_algorithm=hashlib.sha256):
+    tmp_pdu = Pdu(**pdu.get_dict())
+    tmp_pdu = prune_pdu(tmp_pdu)
+    pdu_json = tmp_pdu.get_dict()
+    pdu_json_bytes = encode_canonical_json(pdu_json)
+    hashed = hash_algorithm(pdu_json_bytes)
+    return (hashed.name, hashed.digest())
+
+
 def sign_event_pdu(pdu, signature_name, signing_key):
     tmp_pdu = Pdu(**pdu.get_dict())
     tmp_pdu = prune_pdu(tmp_pdu)
diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py
index 11fd7264b3..7e574f451d 100644
--- a/synapse/federation/pdu_codec.py
+++ b/synapse/federation/pdu_codec.py
@@ -14,7 +14,9 @@
 # limitations under the License.
 
 from .units import Pdu
-from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu
+from synapse.crypto.event_signing import (
+    add_event_pdu_content_hash, sign_event_pdu
+)
 
 import copy
 
@@ -97,5 +99,5 @@ class PduCodec(object):
             kwargs["ts"] = int(self.clock.time_msec())
 
         pdu = Pdu(**kwargs)
-        pdu = hash_event_pdu(pdu)
+        pdu = add_event_pdu_content_hash(pdu)
         return sign_event_pdu(pdu, self.server_name, self.signing_key)
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index af05b47932..1738260cc1 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -44,6 +44,8 @@ from .signatures import SignatureStore
 
 from syutil.base64util import decode_base64
 
+from synapse.crypto.event_signing import compute_pdu_event_reference_hash
+
 import json
 import logging
 import os
@@ -165,7 +167,7 @@ class DataStore(RoomMemberStore, RoomStore,
 
         for hash_alg, hash_base64 in pdu.hashes.items():
             hash_bytes = decode_base64(hash_base64)
-            self._store_pdu_hash_txn(
+            self._store_pdu_content_hash_txn(
                 txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes,
             )
 
@@ -185,6 +187,11 @@ class DataStore(RoomMemberStore, RoomStore,
                     hash_bytes
                 )
 
+        (ref_alg, ref_hash_bytes) = compute_pdu_event_reference_hash(pdu)
+        self._store_pdu_reference_hash_txn(
+            txn, pdu.pdu_id, pdu.origin, ref_alg, ref_hash_bytes
+        )
+
         if pdu.is_state:
             self._persist_state_txn(txn, pdu.prev_pdus, cols)
         else:
diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py
index a423b42dbd..3a90c382f0 100644
--- a/synapse/storage/pdu.py
+++ b/synapse/storage/pdu.py
@@ -69,7 +69,7 @@ class PduStore(SQLBaseStore):
 
             edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin)
 
-            hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin)
+            hashes = self._get_pdu_content_hashes_txn(txn, pdu_id, origin)
             signatures = self._get_pdu_origin_signatures_txn(
                 txn, pdu_id, origin
             )
@@ -317,7 +317,7 @@ class PduStore(SQLBaseStore):
 
         results = []
         for pdu_id, origin, depth in txn.fetchall():
-            hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin)
+            hashes = self._get_pdu_reference_hashes_txn(txn, pdu_id, origin)
             sha256_bytes = hashes["sha256"]
             prev_hashes = {"sha256": encode_base64(sha256_bytes)}
             results.append((pdu_id, origin, prev_hashes, depth))
diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql
index a72c4dc35f..1c45a51bec 100644
--- a/synapse/storage/schema/signatures.sql
+++ b/synapse/storage/schema/signatures.sql
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-CREATE TABLE IF NOT EXISTS pdu_hashes (
+CREATE TABLE IF NOT EXISTS pdu_content_hashes (
   pdu_id TEXT,
   origin TEXT,
   algorithm TEXT,
@@ -21,7 +21,21 @@ CREATE TABLE IF NOT EXISTS pdu_hashes (
   CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS pdu_hashes_id ON pdu_hashes (pdu_id, origin);
+CREATE INDEX IF NOT EXISTS pdu_content_hashes_id ON pdu_content_hashes (
+    pdu_id, origin
+);
+
+CREATE TABLE IF NOT EXISTS pdu_reference_hashes (
+  pdu_id TEXT,
+  origin TEXT,
+  algorithm TEXT,
+  hash BLOB,
+  CONSTRAINT uniqueness UNIQUE (pdu_id, origin, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS pdu_reference_hashes_id ON pdu_reference_hashes (
+    pdu_id, origin
+);
 
 CREATE TABLE IF NOT EXISTS pdu_origin_signatures (
   pdu_id TEXT,
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 1147102489..85eec7ffbe 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -21,7 +21,7 @@ from twisted.internet import defer
 class SignatureStore(SQLBaseStore):
     """Persistence for PDU signatures and hashes"""
 
-    def _get_pdu_hashes_txn(self, txn, pdu_id, origin):
+    def _get_pdu_content_hashes_txn(self, txn, pdu_id, origin):
         """Get all the hashes for a given PDU.
         Args:
             txn (cursor):
@@ -32,13 +32,14 @@ class SignatureStore(SQLBaseStore):
         """
         query = (
             "SELECT algorithm, hash"
-            " FROM pdu_hashes"
+            " FROM pdu_content_hashes"
             " WHERE pdu_id = ? and origin = ?"
         )
         txn.execute(query, (pdu_id, origin))
         return dict(txn.fetchall())
 
-    def _store_pdu_hash_txn(self, txn, pdu_id, origin, algorithm, hash_bytes):
+    def _store_pdu_content_hash_txn(self, txn, pdu_id, origin, algorithm,
+                                    hash_bytes):
         """Store a hash for a PDU
         Args:
             txn (cursor):
@@ -47,13 +48,48 @@ class SignatureStore(SQLBaseStore):
             algorithm (str): Hashing algorithm.
             hash_bytes (bytes): Hash function output bytes.
         """
-        self._simple_insert_txn(txn, "pdu_hashes", {
+        self._simple_insert_txn(txn, "pdu_content_hashes", {
             "pdu_id": pdu_id,
             "origin": origin,
             "algorithm": algorithm,
             "hash": buffer(hash_bytes),
         })
 
+    def _get_pdu_reference_hashes_txn(self, txn, pdu_id, origin):
+        """Get all the hashes for a given PDU.
+        Args:
+            txn (cursor):
+            pdu_id (str): Id for the PDU.
+            origin (str): origin of the PDU.
+        Returns:
+            A dict of algorithm -> hash.
+        """
+        query = (
+            "SELECT algorithm, hash"
+            " FROM pdu_reference_hashes"
+            " WHERE pdu_id = ? and origin = ?"
+        )
+        txn.execute(query, (pdu_id, origin))
+        return dict(txn.fetchall())
+
+    def _store_pdu_reference_hash_txn(self, txn, pdu_id, origin, algorithm,
+                                      hash_bytes):
+        """Store a hash for a PDU
+        Args:
+            txn (cursor):
+            pdu_id (str): Id for the PDU.
+            origin (str): origin of the PDU.
+            algorithm (str): Hashing algorithm.
+            hash_bytes (bytes): Hash function output bytes.
+        """
+        self._simple_insert_txn(txn, "pdu_reference_hashes", {
+            "pdu_id": pdu_id,
+            "origin": origin,
+            "algorithm": algorithm,
+            "hash": buffer(hash_bytes),
+        })
+
+
     def _get_pdu_origin_signatures_txn(self, txn, pdu_id, origin):
         """Get all the signatures for a given PDU.
         Args:
-- 
cgit 1.4.1


From d56e389a95dfcbcbbf279655a6f4935c07cd674f Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 27 Oct 2014 10:33:17 +0000
Subject: Fix pyflakes warnings

---
 synapse/config/server.py    | 2 +-
 synapse/crypto/keyclient.py | 1 -
 synapse/rest/room.py        | 2 +-
 synapse/storage/keys.py     | 1 -
 4 files changed, 2 insertions(+), 4 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/config/server.py b/synapse/config/server.py
index 9332e4acd7..086937044f 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -74,7 +74,7 @@ class ServerConfig(Config):
             return syutil.crypto.signing_key.read_signing_keys(
                 signing_keys.splitlines(True)
             )
-        except Exception as e:
+        except Exception:
             raise ConfigError(
                 "Error reading signing_key."
                 " Try running again with --generate-config"
diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index 5949ea0573..7cfec5148e 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -17,7 +17,6 @@
 from twisted.web.http import HTTPClient
 from twisted.internet.protocol import Factory
 from twisted.internet import defer, reactor
-from twisted.internet.endpoints import connectProtocol
 from synapse.http.endpoint import matrix_endpoint
 import json
 import logging
diff --git a/synapse/rest/room.py b/synapse/rest/room.py
index a01dab1b8e..c72bdc2c34 100644
--- a/synapse/rest/room.py
+++ b/synapse/rest/room.py
@@ -344,7 +344,7 @@ class RoomInitialSyncRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request, room_id):
-        user = yield self.auth.get_user_by_req(request)
+        yield self.auth.get_user_by_req(request)
         # TODO: Get all the initial sync data for this room and return in the
         # same format as initial sync, that is:
         # {
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 8189e071a3..4feb8335ba 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -104,7 +104,6 @@ class KeyStore(SQLBaseStore):
             ts_now_ms (int): The time now in milliseconds
             verification_key (VerifyKey): The NACL verify key.
         """
-        verify_key_bytes = verify_key.encode()
         return self._simple_insert(
             table="server_signature_keys",
             values={
-- 
cgit 1.4.1


From 5e2236f9ffe3a66bbe0ff37b1793e8fa59a1c475 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 27 Oct 2014 11:19:15 +0000
Subject: fix pyflakes warnings

---
 synapse/crypto/event_signing.py | 8 ++++----
 synapse/federation/units.py     | 2 ++
 synapse/storage/signatures.py   | 2 --
 3 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'synapse/storage')

diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py
index d3b501c6e7..61edd2c6f9 100644
--- a/synapse/crypto/event_signing.py
+++ b/synapse/crypto/event_signing.py
@@ -35,12 +35,12 @@ def add_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256):
 
 def check_event_pdu_content_hash(pdu, hash_algorithm=hashlib.sha256):
     """Check whether the hash for this PDU matches the contents"""
-    computed_hash = _compute_content_hash(pdu, hash_algortithm)
+    computed_hash = _compute_content_hash(pdu, hash_algorithm)
     if computed_hash.name not in pdu.hashes:
         raise Exception("Algorithm %s not in hashes %s" % (
             computed_hash.name, list(pdu.hashes)
         ))
-    message_hash_base64 = hashes[computed_hash.name]
+    message_hash_base64 = pdu.hashes[computed_hash.name]
     try:
         message_hash_bytes = decode_base64(message_hash_base64)
     except:
@@ -54,7 +54,7 @@ def _compute_content_hash(pdu, hash_algorithm):
     pdu_json.pop("age_ts", None)
     pdu_json.pop("unsigned", None)
     pdu_json.pop("signatures", None)
-    hashes = pdu_json.pop("hashes", {})
+    pdu_json.pop("hashes", None)
     pdu_json_bytes = encode_canonical_json(pdu_json)
     return hash_algorithm(pdu_json_bytes)
 
@@ -73,7 +73,7 @@ def sign_event_pdu(pdu, signature_name, signing_key):
     tmp_pdu = Pdu(**pdu.get_dict())
     tmp_pdu = prune_pdu(tmp_pdu)
     pdu_json = tmp_pdu.get_dict()
-    pdu_jdon = sign_json(pdu_json, signature_name, signing_key)
+    pdu_json = sign_json(pdu_json, signature_name, signing_key)
     pdu.signatures = pdu_json["signatures"]
     return pdu
 
diff --git a/synapse/federation/units.py b/synapse/federation/units.py
index b779d259bd..adc3385644 100644
--- a/synapse/federation/units.py
+++ b/synapse/federation/units.py
@@ -155,6 +155,8 @@ class Pdu(JsonEncodedObject):
 
             return Pdu(
                 prev_pdus=prev_pdus,
+                hashes=hashes,
+                signatures=signatures,
                 **args
             )
         else:
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 85eec7ffbe..82be946d3f 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -15,8 +15,6 @@
 
 from _base import SQLBaseStore
 
-from twisted.internet import defer
-
 
 class SignatureStore(SQLBaseStore):
     """Persistence for PDU signatures and hashes"""
-- 
cgit 1.4.1