From 1c445f88f64beabf0bd9bec3950a4a4c0d529e8a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 15 Oct 2014 17:09:04 +0100 Subject: persist hashes and origin signatures for PDUs --- tests/federation/test_federation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index d86ce83b28..03b2167cf7 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus) + return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}) class FederationTestCase(unittest.TestCase): @@ -183,6 +183,8 @@ class FederationTestCase(unittest.TestCase): "is_state": False, "content": {"testing": "content here"}, "depth": 1, + "hashes": {}, + "signatures": {}, }, ] }, -- cgit 1.5.1 From 66104da10c4191aa1e048f2379190574755109e6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 16 Oct 2014 00:09:48 +0100 Subject: Sign outgoing PDUs. --- synapse/crypto/event_signing.py | 4 ++-- synapse/federation/pdu_codec.py | 6 +++++- synapse/storage/__init__.py | 7 ++++--- synapse/storage/signatures.py | 6 +++--- tests/federation/test_pdu_codec.py | 13 ++++++++++--- tests/rest/test_events.py | 7 +++++-- tests/rest/test_profile.py | 8 ++++++-- tests/rest/test_rooms.py | 32 +++++++++++++++++++++++++------- tests/utils.py | 3 ++- 9 files changed, 62 insertions(+), 24 deletions(-) (limited to 'tests') diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 6557727e06..a115967c0a 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,6 +15,7 @@ # limitations under the License. +from synapse.federation.units import Pdu from synapse.api.events.utils import prune_pdu from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 @@ -25,8 +26,7 @@ import hashlib def hash_event_pdu(pdu, hash_algortithm=hashlib.sha256): hashed = _compute_hash(pdu, hash_algortithm) - hashes[hashed.name] = encode_base64(hashed.digest()) - pdu.hashes = hashes + pdu.hashes[hashed.name] = encode_base64(hashed.digest()) return pdu diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108dd..bcac5f9ae8 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -14,6 +14,7 @@ # limitations under the License. from .units import Pdu +from synapse.crypto.event_signing import hash_event_pdu, sign_event_pdu import copy @@ -33,6 +34,7 @@ def encode_event_id(pdu_id, origin): class PduCodec(object): def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.event_factory = hs.get_event_factory() self.clock = hs.get_clock() @@ -99,4 +101,6 @@ class PduCodec(object): if "ts" not in kwargs: kwargs["ts"] = int(self.clock.time_msec()) - return Pdu(**kwargs) + pdu = Pdu(**kwargs) + pdu = hash_event_pdu(pdu) + return sign_event_pdu(pdu, self.server_name, self.signing_key) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index bfeab7d1e8..b2a3f0b56c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -42,6 +42,7 @@ from .transactions import TransactionStore from .keys import KeyStore from .signatures import SignatureStore +from syutil.base64util import decode_base64 import json import logging @@ -168,11 +169,11 @@ class DataStore(RoomMemberStore, RoomStore, txn, pdu.pdu_id, pdu.origin, hash_alg, hash_bytes, ) - signatures = pdu.sigatures.get(pdu.orgin, {}) + signatures = pdu.signatures.get(pdu.origin, {}) - for key_id, signature_base64 in signatures: + for key_id, signature_base64 in signatures.items(): signature_bytes = decode_base64(signature_base64) - self.store_pdu_origin_signatures_txn( + self._store_pdu_origin_signature_txn( txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, ) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index bb860f09f0..1f0a680500 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -47,7 +47,7 @@ class SignatureStore(SQLBaseStore): algorithm (str): Hashing algorithm. hash_bytes (bytes): Hash function output bytes. """ - self._simple_insert_txn(self, txn, "pdu_hashes", { + self._simple_insert_txn(txn, "pdu_hashes", { "pdu_id": pdu_id, "origin": origin, "algorithm": algorithm, @@ -66,7 +66,7 @@ class SignatureStore(SQLBaseStore): query = ( "SELECT key_id, signature" " FROM pdu_origin_signatures" - " WHERE WHERE pdu_id = ? and origin = ?" + " WHERE pdu_id = ? and origin = ?" ) txn.execute(query, (pdu_id, origin)) return dict(txn.fetchall()) @@ -81,7 +81,7 @@ class SignatureStore(SQLBaseStore): key_id (str): Id for the signing key. signature (bytes): The signature. """ - self._simple_insert_txn(self, txn, "pdu_origin_signatures", { + self._simple_insert_txn(txn, "pdu_origin_signatures", { "pdu_id": pdu_id, "origin": origin, "key_id": key_id, diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 344e1baf60..80851a4258 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -23,14 +23,21 @@ from synapse.federation.units import Pdu from synapse.server import HomeServer -from mock import Mock +from mock import Mock, NonCallableMock + +from ..utils import MockKey class PduCodecTestCase(unittest.TestCase): def setUp(self): - self.hs = HomeServer("blargle.net") - self.event_factory = self.hs.get_event_factory() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + self.hs = HomeServer( + "blargle.net", + config=self.mock_config, + ) + self.event_factory = self.hs.get_event_factory() self.codec = PduCodec(self.hs) def test_decode_event_id(self): diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 79b371c04d..362c7bc01c 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -28,7 +28,7 @@ from synapse.server import HomeServer # python imports import json -from ..utils import MockHttpResource, MemoryDataStore +from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -122,6 +122,9 @@ class EventStreamPermissionsTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "test", db_pool=None, @@ -139,7 +142,7 @@ class EventStreamPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py index b0f48e7fd8..3a0d1e700a 100644 --- a/tests/rest/test_profile.py +++ b/tests/rest/test_profile.py @@ -18,9 +18,9 @@ from tests import unittest from twisted.internet import defer -from mock import Mock +from mock import Mock, NonCallableMock -from ..utils import MockHttpResource +from ..utils import MockHttpResource, MockKey from synapse.api.errors import SynapseError, AuthError from synapse.server import HomeServer @@ -41,6 +41,9 @@ class ProfileTestCase(unittest.TestCase): "set_avatar_url", ]) + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=None, http_client=None, @@ -48,6 +51,7 @@ class ProfileTestCase(unittest.TestCase): federation=Mock(), replication_layer=Mock(), datastore=None, + config=self.mock_config, ) def _get_user_by_req(request=None): diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1ce9b8a83d..7170193051 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -27,7 +27,7 @@ from synapse.server import HomeServer import json import urllib -from ..utils import MockHttpResource, MemoryDataStore +from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -50,6 +50,9 @@ class RoomPermissionsTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -61,7 +64,7 @@ class RoomPermissionsTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -408,6 +411,9 @@ class RoomsMemberListTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -419,7 +425,7 @@ class RoomsMemberListTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -497,6 +503,9 @@ class RoomsCreateTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -508,7 +517,7 @@ class RoomsCreateTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -598,6 +607,9 @@ class RoomTopicTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -609,7 +621,7 @@ class RoomTopicTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -712,6 +724,9 @@ class RoomMemberStateTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -723,7 +738,7 @@ class RoomMemberStateTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -853,6 +868,9 @@ class RoomMessagesTestCase(RestTestCase): persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "red", db_pool=None, @@ -864,7 +882,7 @@ class RoomMessagesTestCase(RestTestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) diff --git a/tests/utils.py b/tests/utils.py index 60fd6085ac..d8be73dba8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -118,13 +118,14 @@ class MockHttpResource(HttpServer): class MockKey(object): alg = "mock_alg" version = "mock_version" + signature = b"\x9a\x87$" @property def verify_key(self): return self def sign(self, message): - return b"\x9a\x87$" + return self def verify(self, message, sig): assert sig == b"\x9a\x87$" -- cgit 1.5.1 From bb04447c44036ebf3ae5dde7a4cc7a7909d50ef6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 16 Oct 2014 23:25:12 +0100 Subject: Include hashes of previous pdus when referencing them --- synapse/api/events/__init__.py | 2 +- synapse/federation/pdu_codec.py | 13 ++++--------- synapse/federation/replication.py | 2 +- synapse/federation/units.py | 10 +++++++++- synapse/state.py | 4 ---- synapse/storage/__init__.py | 20 ++++++++++++++------ synapse/storage/pdu.py | 22 ++++++++++++++++------ synapse/storage/schema/signatures.sql | 16 ++++++++++++++++ synapse/storage/signatures.py | 31 +++++++++++++++++++++++++++++++ tests/federation/test_federation.py | 2 +- tests/federation/test_pdu_codec.py | 4 ++-- 11 files changed, 95 insertions(+), 31 deletions(-) (limited to 'tests') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index f66fea2904..a5a55742e0 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -65,13 +65,13 @@ class SynapseEvent(JsonEncodedObject): internal_keys = [ "is_state", - "prev_events", "depth", "destinations", "origin", "outlier", "power_level", "redacted", + "prev_pdus", ] required_keys = [ diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index bcac5f9ae8..11fd7264b3 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -45,9 +45,7 @@ class PduCodec(object): kwargs["event_id"] = encode_event_id(pdu.pdu_id, pdu.origin) kwargs["room_id"] = pdu.context kwargs["etype"] = pdu.pdu_type - kwargs["prev_events"] = [ - encode_event_id(p[0], p[1]) for p in pdu.prev_pdus - ] + kwargs["prev_pdus"] = pdu.prev_pdus if hasattr(pdu, "prev_state_id") and hasattr(pdu, "prev_state_origin"): kwargs["prev_state"] = encode_event_id( @@ -78,11 +76,8 @@ class PduCodec(object): d["context"] = event.room_id d["pdu_type"] = event.type - if hasattr(event, "prev_events"): - d["prev_pdus"] = [ - decode_event_id(e, self.server_name) - for e in event.prev_events - ] + if hasattr(event, "prev_pdus"): + d["prev_pdus"] = event.prev_pdus if hasattr(event, "prev_state"): d["prev_state_id"], d["prev_state_origin"] = ( @@ -95,7 +90,7 @@ class PduCodec(object): kwargs = copy.deepcopy(event.unrecognized_keys) kwargs.update({ k: v for k, v in d.items() - if k not in ["event_id", "room_id", "type", "prev_events"] + if k not in ["event_id", "room_id", "type"] }) if "ts" not in kwargs: diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9363ac7300..788a49b8e8 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -443,7 +443,7 @@ class ReplicationLayer(object): min_depth = yield self.store.get_min_depth_for_context(pdu.context) if min_depth and pdu.depth > min_depth: - for pdu_id, origin in pdu.prev_pdus: + for pdu_id, origin, hashes in pdu.prev_pdus: exists = yield self._get_persisted_pdu(pdu_id, origin) if not exists: diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 3518efb215..6a43007837 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -141,8 +141,16 @@ class Pdu(JsonEncodedObject): for kid, sig in pdu_tuple.signatures.items() } + prev_pdus = [] + for prev_pdu in pdu_tuple.prev_pdu_list: + prev_hashes = pdu_tuple.edge_hashes.get(prev_pdu, {}) + prev_hashes = { + alg: encode_base64(hsh) for alg, hsh in prev_hashes.items() + } + prev_pdus.append((prev_pdu[0], prev_pdu[1], prev_hashes)) + return Pdu( - prev_pdus=pdu_tuple.prev_pdu_list, + prev_pdus=prev_pdus, **args ) else: diff --git a/synapse/state.py b/synapse/state.py index 9db84c9b5c..bc6b928ec7 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -72,10 +72,6 @@ class StateHandler(object): snapshot.fill_out_prev_events(event) - event.prev_events = [ - e for e in event.prev_events if e != event.event_id - ] - current_state = snapshot.prev_state_pdu if current_state: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b2a3f0b56c..af05b47932 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -177,6 +177,14 @@ class DataStore(RoomMemberStore, RoomStore, txn, pdu.pdu_id, pdu.origin, key_id, signature_bytes, ) + for prev_pdu_id, prev_origin, prev_hashes in pdu.prev_pdus: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_pdu_hash_txn( + txn, pdu.pdu_id, pdu.origin, prev_pdu_id, prev_origin, alg, + hash_bytes + ) + if pdu.is_state: self._persist_state_txn(txn, pdu.prev_pdus, cols) else: @@ -352,6 +360,7 @@ class DataStore(RoomMemberStore, RoomStore, prev_pdus = self._get_latest_pdus_in_context( txn, room_id ) + if state_type is not None and state_key is not None: prev_state_pdu = self._get_current_state_pdu( txn, room_id, state_type, state_key @@ -401,17 +410,16 @@ class Snapshot(object): self.prev_state_pdu = prev_state_pdu def fill_out_prev_events(self, event): - if hasattr(event, "prev_events"): + if hasattr(event, "prev_pdus"): return - es = [ - "%s@%s" % (p_id, origin) for p_id, origin, _ in self.prev_pdus + event.prev_pdus = [ + (p_id, origin, hashes) + for p_id, origin, hashes, _ in self.prev_pdus ] - event.prev_events = [e for e in es if e != event.event_id] - if self.prev_pdus: - event.depth = max([int(v) for _, _, v in self.prev_pdus]) + 1 + event.depth = max([int(v) for _, _, _, v in self.prev_pdus]) + 1 else: event.depth = 0 diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index 9d624429b7..a423b42dbd 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -20,10 +20,13 @@ from ._base import SQLBaseStore, Table, JoinHelper from synapse.federation.units import Pdu from synapse.util.logutils import log_function +from syutil.base64util import encode_base64 + from collections import namedtuple import logging + logger = logging.getLogger(__name__) @@ -64,6 +67,8 @@ class PduStore(SQLBaseStore): for r in PduEdgesTable.decode_results(txn.fetchall()) ] + edge_hashes = self._get_prev_pdu_hashes_txn(txn, pdu_id, origin) + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) signatures = self._get_pdu_origin_signatures_txn( txn, pdu_id, origin @@ -86,7 +91,7 @@ class PduStore(SQLBaseStore): row = txn.fetchone() if row: results.append(PduTuple( - PduEntry(*row), edges, hashes, signatures + PduEntry(*row), edges, hashes, signatures, edge_hashes )) return results @@ -310,9 +315,14 @@ class PduStore(SQLBaseStore): (context, ) ) - results = txn.fetchall() + results = [] + for pdu_id, origin, depth in txn.fetchall(): + hashes = self._get_pdu_hashes_txn(txn, pdu_id, origin) + sha256_bytes = hashes["sha256"] + prev_hashes = {"sha256": encode_base64(sha256_bytes)} + results.append((pdu_id, origin, prev_hashes, depth)) - return [(row[0], row[1], row[2]) for row in results] + return results @defer.inlineCallbacks def get_oldest_pdus_in_context(self, context): @@ -431,7 +441,7 @@ class PduStore(SQLBaseStore): "DELETE FROM %s WHERE pdu_id = ? AND origin = ?" % PduForwardExtremitiesTable.table_name ) - txn.executemany(query, prev_pdus) + txn.executemany(query, list(p[:2] for p in prev_pdus)) # We only insert as a forward extremety the new pdu if there are no # other pdus that reference it as a prev pdu @@ -454,7 +464,7 @@ class PduStore(SQLBaseStore): # deleted in a second if they're incorrect anyway. txn.executemany( PduBackwardExtremitiesTable.insert_statement(), - [(i, o, context) for i, o in prev_pdus] + [(i, o, context) for i, o, _ in prev_pdus] ) # Also delete from the backwards extremities table all ones that @@ -915,7 +925,7 @@ This does not include a prev_pdus key. PduTuple = namedtuple( "PduTuple", - ("pdu_entry", "prev_pdu_list", "hashes", "signatures") + ("pdu_entry", "prev_pdu_list", "hashes", "signatures", "edge_hashes") ) """ This is a tuple of a `PduEntry` and a list of `PduIdTuple` that represent the `prev_pdus` key of a PDU. diff --git a/synapse/storage/schema/signatures.sql b/synapse/storage/schema/signatures.sql index 86ee0f2377..a72c4dc35f 100644 --- a/synapse/storage/schema/signatures.sql +++ b/synapse/storage/schema/signatures.sql @@ -34,3 +34,19 @@ CREATE TABLE IF NOT EXISTS pdu_origin_signatures ( CREATE INDEX IF NOT EXISTS pdu_origin_signatures_id ON pdu_origin_signatures ( pdu_id, origin ); + +CREATE TABLE IF NOT EXISTS pdu_edge_hashes( + pdu_id TEXT, + origin TEXT, + prev_pdu_id TEXT, + prev_origin TEXT, + algorithm TEXT, + hash BLOB, + CONSTRAINT uniqueness UNIQUE ( + pdu_id, origin, prev_pdu_id, prev_origin, algorithm + ) +); + +CREATE INDEX IF NOT EXISTS pdu_edge_hashes_id ON pdu_edge_hashes( + pdu_id, origin +); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 1f0a680500..1147102489 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -88,3 +88,34 @@ class SignatureStore(SQLBaseStore): "signature": buffer(signature_bytes), }) + def _get_prev_pdu_hashes_txn(self, txn, pdu_id, origin): + """Get all the hashes for previous PDUs of a PDU + Args: + txn (cursor): + pdu_id (str): Id of the PDU. + origin (str): Origin of the PDU. + Returns: + dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. + """ + query = ( + "SELECT prev_pdu_id, prev_origin, algorithm, hash" + " FROM pdu_edge_hashes" + " WHERE pdu_id = ? and origin = ?" + ) + txn.execute(query, (pdu_id, origin)) + results = {} + for prev_pdu_id, prev_origin, algorithm, hash_bytes in txn.fetchall(): + hashes = results.setdefault((prev_pdu_id, prev_origin), {}) + hashes[algorithm] = hash_bytes + return results + + def _store_prev_pdu_hash_txn(self, txn, pdu_id, origin, prev_pdu_id, + prev_origin, algorithm, hash_bytes): + self._simple_insert_txn(txn, "pdu_edge_hashes", { + "pdu_id": pdu_id, + "origin": origin, + "prev_pdu_id": prev_pdu_id, + "prev_origin": prev_origin, + "algorithm": algorithm, + "hash": buffer(hash_bytes), + }) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 03b2167cf7..eed50e6335 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -41,7 +41,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}) + return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {}) class FederationTestCase(unittest.TestCase): diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 80851a4258..0ad8cf6641 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -88,7 +88,7 @@ class PduCodecTestCase(unittest.TestCase): self.assertEquals(pdu.context, event.room_id) self.assertEquals(pdu.is_state, event.is_state) self.assertEquals(pdu.depth, event.depth) - self.assertEquals(["alice@bob.com"], event.prev_events) + self.assertEquals(pdu.prev_pdus, event.prev_pdus) self.assertEquals(pdu.content, event.content) def test_pdu_from_event(self): @@ -144,7 +144,7 @@ class PduCodecTestCase(unittest.TestCase): self.assertEquals(pdu.context, event.room_id) self.assertEquals(pdu.is_state, event.is_state) self.assertEquals(pdu.depth, event.depth) - self.assertEquals(["alice@bob.com"], event.prev_events) + self.assertEquals(pdu.prev_pdus, event.prev_pdus) self.assertEquals(pdu.content, event.content) self.assertEquals(pdu.state_key, event.state_key) -- cgit 1.5.1 From b29517bd013b82302b1a73072da8bfc39564dc1a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Oct 2014 01:21:33 +0000 Subject: Add a request-id to each log line --- setup.py | 2 +- synapse/app/homeserver.py | 12 +++++- synapse/config/logger.py | 23 ++++++++---- synapse/crypto/keyclient.py | 10 +++-- synapse/http/client.py | 26 +++++++------ synapse/http/server.py | 13 ++++++- synapse/storage/_base.py | 16 ++++++-- synapse/util/async.py | 5 ++- synapse/util/logcontext.py | 85 ++++++++++++++++++++++++++++++++++++++++++ synapse/util/logutils.py | 1 + tests/util/test_log_context.py | 43 +++++++++++++++++++++ 11 files changed, 205 insertions(+), 31 deletions(-) create mode 100644 synapse/util/logcontext.py create mode 100644 tests/util/test_log_context.py (limited to 'tests') diff --git a/setup.py b/setup.py index 660efd5b89..74eee31a78 100755 --- a/setup.py +++ b/setup.py @@ -54,6 +54,6 @@ setup( long_description=read("README.rst"), entry_points=""" [console_scripts] - synapse-homeserver=synapse.app.homeserver:run + synapse-homeserver=synapse.app.homeserver:main """ ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6394bc27d1..4e74f4d14c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -33,6 +33,7 @@ from synapse.api.urls import ( ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory +from synapse.util.logcontext import LoggingContext from daemonize import Daemonize import twisted.manhole.telnet @@ -240,7 +241,7 @@ def setup(): daemon = Daemonize( app="synapse-homeserver", pid=config.pid_file, - action=reactor.run, + action=run, auto_close_fds=False, verbose=True, logger=logger, @@ -250,6 +251,13 @@ def setup(): else: reactor.run() +def run(): + with LoggingContext("run") as context: + reactor.run() + +def main(): + with LoggingContext("main") as context: + setup() if __name__ == '__main__': - setup() + main() diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 56cd095433..2a59bf9d15 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -14,7 +14,7 @@ # limitations under the License. from ._base import Config - +from synapse.util.logcontext import LoggingContextFilter from twisted.python.log import PythonLoggingObserver import logging import logging.config @@ -45,7 +45,8 @@ class LoggingConfig(Config): def setup_logging(self): log_format = ( - '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s' + "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" + " - %(message)s" ) if self.log_config is None: @@ -54,12 +55,20 @@ class LoggingConfig(Config): level = logging.DEBUG # FIXME: we need a logging.WARN for a -q quiet option + logger = logging.getLogger('') + logger.setLevel(level) + formatter = logging.Formatter(log_format) + if self.log_file: + handler = logging.FileHandler(self.log_file) + else: + handler = logging.StreamHandler() + print handler + handler.setFormatter(formatter) + + handler.addFilter(LoggingContextFilter(request="")) - logging.basicConfig( - level=level, - filename=self.log_file, - format=log_format - ) + logger.addHandler(handler) + logger.info("Test") else: logging.config.fileConfig(self.log_config) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 7cfec5148e..33fa9ca837 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -18,6 +18,7 @@ from twisted.web.http import HTTPClient from twisted.internet.protocol import Factory from twisted.internet import defer, reactor from synapse.http.endpoint import matrix_endpoint +from synapse.util.logcontext import PreserveLoggingContext import json import logging @@ -36,10 +37,11 @@ def fetch_server_key(server_name, ssl_context_factory): for i in range(5): try: - protocol = yield endpoint.connect(factory) - server_response, server_certificate = yield protocol.remote_key - defer.returnValue((server_response, server_certificate)) - return + with PreserveLoggingContext(): + protocol = yield endpoint.connect(factory) + server_response, server_certificate = yield protocol.remote_key + defer.returnValue((server_response, server_certificate)) + return except Exception as e: logger.exception(e) raise IOError("Cannot get key for %s" % server_name) diff --git a/synapse/http/client.py b/synapse/http/client.py index 46c90dbb76..8bda42364b 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -16,11 +16,14 @@ from twisted.internet import defer, reactor from twisted.internet.error import DNSLookupError -from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer, PartialDownloadError +from twisted.web.client import ( + _AgentBase, _URI, readBody, FileBodyProducer, PartialDownloadError +) from twisted.web.http_headers import Headers from synapse.http.endpoint import matrix_endpoint from synapse.util.async import sleep +from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json @@ -106,16 +109,17 @@ class BaseHttpClient(object): producer = body_callback(method, url_bytes, headers_dict) try: - response = yield self.agent.request( - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) + with PreserveLoggingContext(): + response = yield self.agent.request( + destination, + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer + ) logger.debug("Got response to %s", method) break diff --git a/synapse/http/server.py b/synapse/http/server.py index 8d419c02dd..ed1f1170cb 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -20,6 +20,7 @@ from syutil.jsonutil import ( from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) +from synapse.util.logcontext import LoggingContext from twisted.internet import defer, reactor from twisted.web import server, resource @@ -88,9 +89,19 @@ class JsonResource(HttpServer, resource.Resource): def render(self, request): """ This get's called by twisted every time someone sends us a request. """ - self._async_render(request) + self._async_render_with_logging_context(request) return server.NOT_DONE_YET + _request_id = 0 + + @defer.inlineCallbacks + def _async_render_with_logging_context(self, request): + request_id = "%s-%s" % (request.method, JsonResource._request_id) + JsonResource._request_id += 1 + with LoggingContext(request_id) as request_context: + request_context.request = request_id + yield self._async_render(request) + @defer.inlineCallbacks def _async_render(self, request): """ This get's called by twisted every time someone sends us a request. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 65a86e9056..2faa63904e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.api.events.utils import prune_event from synapse.util.logutils import log_function +from synapse.util.logcontext import PreserveLoggingContext, LoggingContext import collections import copy @@ -74,12 +75,19 @@ class SQLBaseStore(object): self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() + @defer.inlineCallbacks def runInteraction(self, func, *args, **kwargs): """Wraps the .runInteraction() method on the underlying db_pool.""" + current_context = LoggingContext.current_context() def inner_func(txn, *args, **kwargs): - return func(LoggingTransaction(txn), *args, **kwargs) - - return self._db_pool.runInteraction(inner_func, *args, **kwargs) + with LoggingContext("runInteraction") as context: + current_context.copy_to(context) + return func(LoggingTransaction(txn), *args, **kwargs) + with PreserveLoggingContext(): + result = yield self._db_pool.runInteraction( + inner_func, *args, **kwargs + ) + defer.returnValue(result) def cursor_to_dict(self, cursor): """Converts a SQL cursor into an list of dicts. @@ -146,7 +154,7 @@ class SQLBaseStore(object): ) logger.debug( - "[SQL] %s Args=%s Func=%s", + "[SQL] %s Args=%s", sql, values.values(), ) diff --git a/synapse/util/async.py b/synapse/util/async.py index 647ea6142c..3d3fbe182c 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -16,8 +16,11 @@ from twisted.internet import defer, reactor +from .logcontext import PreserveLoggingContext +@defer.inlineCallbacks def sleep(seconds): d = defer.Deferred() reactor.callLater(seconds, d.callback, seconds) - return d + with PreserveLoggingContext(): + yield d diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py new file mode 100644 index 0000000000..46a2855a15 --- /dev/null +++ b/synapse/util/logcontext.py @@ -0,0 +1,85 @@ +from functools import wraps + +import threading +import logging + +class LoggingContext(object): + __slots__ = ["parent_context", "name", "__dict__"] + + thread_local = threading.local() + + class Sentinel(object): + __slots__ = [] + def copy_to(self, record): + pass + + sentinel = Sentinel() + + def __init__(self, name=None): + self.parent_context = None + self.name = name + + def __str__(self): + return "%s@%x" % (self.name, id(self)) + + @classmethod + def current_context(cls): + return getattr(cls.thread_local, "current_context", cls.sentinel) + + def __enter__(self): + if self.parent_context is not None: + raise Exception("Attempt to enter logging context multiple times") + self.parent_context = self.current_context() + self.thread_local.current_context = self + return self + + def __exit__(self, type, value, traceback): + if self.thread_local.current_context is not self: + logging.error( + "Current logging context %s is not the expected context %s", + self.thread_local.current_context, + self + ) + self.thread_local.current_context = self.parent_context + self.parent_context = None + + def __getattr__(self, name): + return getattr(self.parent_context, name) + + def copy_to(self, record): + if self.parent_context is not None: + self.parent_context.copy_to(record) + for key, value in self.__dict__.items(): + setattr(record, key, value) + + @classmethod + def wrap_callback(cls, callback): + context = cls.current_context() + @wraps(callback) + def wrapped(*args, **kargs): + cls.thread_local.current_context = context + return callback(*args, **kargs) + return wrapped + + +class LoggingContextFilter(logging.Filter): + def __init__(self, **defaults): + self.defaults = defaults + + def filter(self, record): + context = LoggingContext.current_context() + for key, value in self.defaults.items(): + setattr(record, key, value) + context.copy_to(record) + return True + + +class PreserveLoggingContext(object): + __slots__ = ["current_context"] + def __enter__(self): + self.current_context = LoggingContext.current_context() + + def __exit__(self, type, value, traceback): + LoggingContext.thread_local.current_context = self.current_context + + diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index fadf0bd510..903a6cf1b3 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -75,6 +75,7 @@ def trace_function(f): linenum = f.func_code.co_firstlineno pathname = f.func_code.co_filename + @wraps(f) def wrapped(*args, **kwargs): name = f.__module__ logger = logging.getLogger(name) diff --git a/tests/util/test_log_context.py b/tests/util/test_log_context.py new file mode 100644 index 0000000000..efa0f28bad --- /dev/null +++ b/tests/util/test_log_context.py @@ -0,0 +1,43 @@ +from twisted.internet import defer +from twisted.internet import reactor +from .. import unittest + +from synapse.util.async import sleep +from synapse.util.logcontext import LoggingContext + +class LoggingContextTestCase(unittest.TestCase): + + def _check_test_key(self, value): + self.assertEquals( + LoggingContext.current_context().test_key, value + ) + + def test_with_context(self): + with LoggingContext() as context_one: + context_one.test_key = "test" + self._check_test_key("test") + + def test_chaining(self): + with LoggingContext() as context_one: + context_one.test_key = "one" + with LoggingContext() as context_two: + self._check_test_key("one") + context_two.test_key = "two" + self._check_test_key("two") + self._check_test_key("one") + + @defer.inlineCallbacks + def test_sleep(self): + @defer.inlineCallbacks + def competing_callback(): + with LoggingContext() as competing_context: + competing_context.test_key = "competing" + yield sleep(0) + self._check_test_key("competing") + + reactor.callLater(0, competing_callback) + + with LoggingContext() as context_one: + context_one.test_key = "one" + yield sleep(0) + self._check_test_key("one") -- cgit 1.5.1 From 24305ba5bf39538f692e108a4f9dcc13c0e4ab85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:15:30 +0000 Subject: Fix up federation tests --- tests/federation/test_federation.py | 238 +++++++++++++++++++----------------- 1 file changed, 124 insertions(+), 114 deletions(-) (limited to 'tests') diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 3a14c7c3b9..eb329eec50 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -24,7 +24,6 @@ from ..utils import MockHttpResource, MockClock, MockKey from synapse.server import HomeServer from synapse.federation import initialize_http_replication from synapse.federation.units import Pdu -from synapse.storage.pdu import PduTuple, PduEntry def make_pdu(prev_pdus=[], **kwargs): @@ -41,7 +40,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return PduTuple(PduEntry(**pdu_fields), prev_pdus, {}, {}, {}) + return Pdu(prev_pdus=prev_pdus, **pdu_fields) class FederationTestCase(unittest.TestCase): @@ -52,179 +51,185 @@ class FederationTestCase(unittest.TestCase): "put_json", ]) self.mock_persistence = Mock(spec=[ - "get_current_state_for_context", - "get_pdu", - "persist_event", - "update_min_depth_for_context", "prep_send_transaction", "delivered_txn", "get_received_txn_response", "set_received_txn_response", ]) self.mock_persistence.get_received_txn_response.return_value = ( - defer.succeed(None) + defer.succeed(None) ) self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] self.clock = MockClock() - hs = HomeServer("test", - resource_for_federation=self.mock_resource, - http_client=self.mock_http_client, - db_pool=None, - datastore=self.mock_persistence, - clock=self.clock, - config=self.mock_config, - keyring=Mock(), + hs = HomeServer( + "test", + resource_for_federation=self.mock_resource, + http_client=self.mock_http_client, + db_pool=None, + datastore=self.mock_persistence, + clock=self.clock, + config=self.mock_config, + keyring=Mock(), ) self.federation = initialize_http_replication(hs) self.distributor = hs.get_distributor() @defer.inlineCallbacks def test_get_state(self): - self.mock_persistence.get_current_state_for_context.return_value = ( - defer.succeed([]) - ) + mock_handler = Mock(spec=[ + "get_state_for_pdu", + ]) + + self.federation.set_handler(mock_handler) + + mock_handler.get_state_for_pdu.return_value = defer.succeed([]) # Empty context initially - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/state/my-context/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/state/my-context/", + None + ) self.assertEquals(200, code) self.assertFalse(response["pdus"]) # Now lets give the context some state - self.mock_persistence.get_current_state_for_context.return_value = ( + mock_handler.get_state_for_pdu.return_value = ( defer.succeed([ make_pdu( - pdu_id="the-pdu-id", + event_id="the-pdu-id", origin="red", - context="my-context", - pdu_type="m.topic", - ts=123456789000, + room_id="my-context", + type="m.topic", + origin_server_ts=123456789000, depth=1, - is_state=True, - content_json='{"topic":"The topic"}', + content={"topic": "The topic"}, state_key="", power_level=1000, - prev_state_id="last-pdu-id", - prev_state_origin="blue", + prev_state="last-pdu-id", ), ]) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/state/my-context/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/state/my-context/", + None + ) self.assertEquals(200, code) self.assertEquals(1, len(response["pdus"])) @defer.inlineCallbacks def test_get_pdu(self): - self.mock_persistence.get_pdu.return_value = ( + mock_handler = Mock(spec=[ + "get_persisted_pdu", + ]) + + self.federation.set_handler(mock_handler) + + mock_handler.get_persisted_pdu.return_value = ( defer.succeed(None) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/pdu/red/abc123def456/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/event/abc123def456/", + None + ) self.assertEquals(404, code) # Now insert such a PDU - self.mock_persistence.get_pdu.return_value = ( + mock_handler.get_persisted_pdu.return_value = ( defer.succeed( make_pdu( - pdu_id="abc123def456", + event_id="abc123def456", origin="red", - context="my-context", - pdu_type="m.text", - ts=123456789001, + room_id="my-context", + type="m.text", + origin_server_ts=123456789001, depth=1, - content_json='{"text":"Here is the message"}', + content={"text": "Here is the message"}, ) ) ) - (code, response) = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/pdu/red/abc123def456/", None) + (code, response) = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/event/abc123def456/", + None + ) self.assertEquals(200, code) self.assertEquals(1, len(response["pdus"])) - self.assertEquals("m.text", response["pdus"][0]["pdu_type"]) + self.assertEquals("m.text", response["pdus"][0]["type"]) @defer.inlineCallbacks def test_send_pdu(self): self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") + (200, "OK") ) pdu = Pdu( - pdu_id="abc123def456", - origin="red", - destinations=["remote"], - context="my-context", - origin_server_ts=123456789002, - pdu_type="m.test", - content={"testing": "content here"}, - depth=1, + event_id="abc123def456", + origin="red", + room_id="my-context", + type="m.text", + origin_server_ts=123456789001, + depth=1, + content={"text": "Here is the message"}, + destinations=["remote"], ) yield self.federation.send_pdu(pdu) self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin_server_ts": 1000000, - "origin": "test", - "pdus": [ - { - "origin": "red", - "pdu_id": "abc123def456", - "prev_pdus": [], - "origin_server_ts": 123456789002, - "context": "my-context", - "pdu_type": "m.test", - "is_state": False, - "content": {"testing": "content here"}, - "depth": 1, - "hashes": {}, - "signatures": {}, - }, - ] - }, - json_data_callback=ANY, + "remote", + path="/_matrix/federation/v1/send/1000000/", + data={ + "origin_server_ts": 1000000, + "origin": "test", + "pdus": [ + pdu.get_dict(), + ], + 'pdu_failures': [], + }, + json_data_callback=ANY, ) @defer.inlineCallbacks def test_send_edu(self): self.mock_http_client.put_json.return_value = defer.succeed( - (200, "OK") + (200, "OK") ) yield self.federation.send_edu( - destination="remote", - edu_type="m.test", - content={"testing": "content here"}, + destination="remote", + edu_type="m.test", + content={"testing": "content here"}, ) # MockClock ensures we can guess these timestamps self.mock_http_client.put_json.assert_called_with( - "remote", - path="/_matrix/federation/v1/send/1000000/", - data={ - "origin": "test", - "origin_server_ts": 1000000, - "pdus": [], - "edus": [ - { - # TODO: SYN-103: Remove "origin" and "destination" - "origin": "test", - "destination": "remote", - "edu_type": "m.test", - "content": {"testing": "content here"}, - } - ], - }, - json_data_callback=ANY, + "remote", + path="/_matrix/federation/v1/send/1000000/", + data={ + "origin": "test", + "origin_server_ts": 1000000, + "pdus": [], + "edus": [ + { + # TODO: SYN-103: Remove "origin" and "destination" + "origin": "test", + "destination": "remote", + "edu_type": "m.test", + "content": {"testing": "content here"}, + } + ], + 'pdu_failures': [], + }, + json_data_callback=ANY, ) - @defer.inlineCallbacks def test_recv_edu(self): recv_observer = Mock() @@ -232,24 +237,26 @@ class FederationTestCase(unittest.TestCase): self.federation.register_edu_handler("m.test", recv_observer) - yield self.mock_resource.trigger("PUT", - "/_matrix/federation/v1/send/1001000/", - """{ - "origin": "remote", - "origin_server_ts": 1001000, - "pdus": [], - "edus": [ - { - "origin": "remote", - "destination": "test", - "edu_type": "m.test", - "content": {"testing": "reply here"} - } - ] - }""") + yield self.mock_resource.trigger( + "PUT", + "/_matrix/federation/v1/send/1001000/", + """{ + "origin": "remote", + "origin_server_ts": 1001000, + "pdus": [], + "edus": [ + { + "origin": "remote", + "destination": "test", + "edu_type": "m.test", + "content": {"testing": "reply here"} + } + ] + }""" + ) recv_observer.assert_called_with( - "remote", {"testing": "reply here"} + "remote", {"testing": "reply here"} ) @defer.inlineCallbacks @@ -280,8 +287,11 @@ class FederationTestCase(unittest.TestCase): self.federation.register_query_handler("a-question", recv_handler) - code, response = yield self.mock_resource.trigger("GET", - "/_matrix/federation/v1/query/a-question?three=3&four=4", None) + code, response = yield self.mock_resource.trigger( + "GET", + "/_matrix/federation/v1/query/a-question?three=3&four=4", + None + ) self.assertEquals(200, code) self.assertEquals({"another": "response"}, response) -- cgit 1.5.1 From b15e8d5bbc58ccce49b852a07c5f6364e18c39cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:20:02 +0000 Subject: event <-> pdu mappings are now trivial and will soon be scrapped --- tests/federation/test_pdu_codec.py | 167 ------------------------------------- 1 file changed, 167 deletions(-) delete mode 100644 tests/federation/test_pdu_codec.py (limited to 'tests') diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py deleted file mode 100644 index fd0f72c734..0000000000 --- a/tests/federation/test_pdu_codec.py +++ /dev/null @@ -1,167 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tests import unittest - -from synapse.federation.pdu_codec import ( - PduCodec, encode_event_id, decode_event_id -) -from synapse.federation.units import Pdu -#from synapse.api.events.room import MessageEvent - -from synapse.server import HomeServer - -from mock import Mock, NonCallableMock - -from ..utils import MockKey - - -class PduCodecTestCase(unittest.TestCase): - def setUp(self): - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - self.hs = HomeServer( - "blargle.net", - config=self.mock_config, - ) - self.event_factory = self.hs.get_event_factory() - self.codec = PduCodec(self.hs) - - def test_decode_event_id(self): - self.assertEquals( - ("foo", "bar.com"), - decode_event_id("foo@bar.com", "A") - ) - - self.assertEquals( - ("foo", "bar.com"), - decode_event_id("foo", "bar.com") - ) - - def test_encode_event_id(self): - self.assertEquals("A@B", encode_event_id("A", "B")) - - def test_codec_event_id(self): - event_id = "aa@bb.com" - - self.assertEquals( - event_id, - encode_event_id(*decode_event_id(event_id, None)) - ) - - pdu_id = ("aa", "bb.com") - - self.assertEquals( - pdu_id, - decode_event_id(encode_event_id(*pdu_id), None) - ) - - def test_event_from_pdu(self): - pdu = Pdu( - pdu_id="foo", - context="rooooom", - pdu_type="m.room.message", - origin="bar.com", - origin_server_ts=12345, - depth=5, - prev_pdus=[("alice", "bob.com")], - is_state=False, - content={"msgtype": u"test"}, - ) - - event = self.codec.event_from_pdu(pdu) - - self.assertEquals("foo@bar.com", event.event_id) - self.assertEquals(pdu.context, event.room_id) - self.assertEquals(pdu.is_state, event.is_state) - self.assertEquals(pdu.depth, event.depth) - self.assertEquals(pdu.prev_pdus, event.prev_pdus) - self.assertEquals(pdu.content, event.content) - - def test_pdu_from_event(self): - event = self.event_factory.create_event( - etype="m.room.message", - event_id="gargh_id", - room_id="rooom", - user_id="sender", - content={"msgtype": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals(event.event_id, pdu.pdu_id) - self.assertEquals(self.hs.hostname, pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - - event = self.event_factory.create_event( - etype="m.room.message", - event_id="gargh_id@bob.com", - room_id="rooom", - user_id="sender", - content={"msgtype": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals("gargh_id", pdu.pdu_id) - self.assertEquals("bob.com", pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - - def test_event_from_state_pdu(self): - pdu = Pdu( - pdu_id="foo", - context="rooooom", - pdu_type="m.room.topic", - origin="bar.com", - origin_server_ts=12345, - depth=5, - prev_pdus=[("alice", "bob.com")], - is_state=True, - content={"topic": u"test"}, - state_key="", - ) - - event = self.codec.event_from_pdu(pdu) - - self.assertEquals("foo@bar.com", event.event_id) - self.assertEquals(pdu.context, event.room_id) - self.assertEquals(pdu.is_state, event.is_state) - self.assertEquals(pdu.depth, event.depth) - self.assertEquals(pdu.prev_pdus, event.prev_pdus) - self.assertEquals(pdu.content, event.content) - self.assertEquals(pdu.state_key, event.state_key) - - def test_pdu_from_state_event(self): - event = self.event_factory.create_event( - etype="m.room.topic", - event_id="gargh_id", - room_id="rooom", - user_id="sender", - content={"topic": u"test"}, - ) - - pdu = self.codec.pdu_from_event(event) - - self.assertEquals(event.event_id, pdu.pdu_id) - self.assertEquals(self.hs.hostname, pdu.origin) - self.assertEquals(event.room_id, pdu.context) - self.assertEquals(event.content, pdu.content) - self.assertEquals(event.type, pdu.pdu_type) - self.assertEquals(event.state_key, pdu.state_key) -- cgit 1.5.1 From dd3711bdbd2a439208a9e22b7b15f0c3676e611f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:33:52 +0000 Subject: Fix tests.handlers.test_directory --- tests/handlers/test_directory.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index e10a49a8ac..16c446498d 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -21,9 +21,8 @@ from mock import Mock from synapse.server import HomeServer from synapse.handlers.directory import DirectoryHandler -from synapse.storage.directory import RoomAliasMapping -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey class DirectoryHandlers(object): @@ -48,11 +47,15 @@ class DirectoryTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=db_pool, http_client=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, + config=self.mock_config, ) hs.handlers = DirectoryHandlers(hs) -- cgit 1.5.1 From 7f4c7fe4e823eeacaa5b8c83201d504de53beefe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:35:38 +0000 Subject: PEP8 --- tests/handlers/test_directory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 16c446498d..8e164e4be0 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -40,6 +40,7 @@ class DirectoryTestCase(unittest.TestCase): ]) self.query_handlers = {} + def register_query_handler(query_type, handler): self.query_handlers[query_type] = handler self.mock_federation.register_query_handler = register_query_handler @@ -50,7 +51,8 @@ class DirectoryTestCase(unittest.TestCase): self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] - hs = HomeServer("test", + hs = HomeServer( + "test", db_pool=db_pool, http_client=None, resource_for_federation=Mock(), -- cgit 1.5.1 From b0554682ed0c037f8726f3e37dba2113b5e8764f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 16:51:59 +0000 Subject: Fix federation handler tests. I've removed the invite/join dance ones as they are completely out of date. --- tests/handlers/test_federation.py | 84 +++++++++++++-------------------------- 1 file changed, 27 insertions(+), 57 deletions(-) (limited to 'tests') diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 219b2c4c5e..63a9990f7a 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -17,16 +17,15 @@ from twisted.internet import defer from tests import unittest from synapse.api.events.room import ( - InviteJoinEvent, MessageEvent, RoomMemberEvent + MessageEvent, ) -from synapse.api.constants import Membership from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer from synapse.federation.units import Pdu from mock import NonCallableMock, ANY -from ..utils import get_mock_call_args, MockKey +from ..utils import MockKey class FederationTestCase(unittest.TestCase): @@ -36,6 +35,14 @@ class FederationTestCase(unittest.TestCase): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + self.state_handler = NonCallableMock(spec_set=[ + "annotate_state_groups", + ]) + + self.auth = NonCallableMock(spec_set=[ + "check", + ]) + self.hostname = "test" hs = HomeServer( self.hostname, @@ -53,6 +60,8 @@ class FederationTestCase(unittest.TestCase): "federation_handler", ]), config=self.mock_config, + auth=self.auth, + state_handler=self.state_handler, ) self.datastore = hs.get_datastore() @@ -65,74 +74,35 @@ class FederationTestCase(unittest.TestCase): @defer.inlineCallbacks def test_msg(self): pdu = Pdu( - pdu_type=MessageEvent.TYPE, - context="foo", + type=MessageEvent.TYPE, + room_id="foo", content={"msgtype": u"fooo"}, origin_server_ts=0, - pdu_id="a", + event_id="$a:b", origin="b", ) - store_id = "ASD" - self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) + self.state_handler.annotate_state_groups.return_value = ( + defer.succeed(False) + ) + yield self.handlers.federation_handler.on_receive_pdu(pdu, False) self.datastore.persist_event.assert_called_once_with( ANY, False, is_new_state=False ) - self.notifier.on_new_room_event.assert_called_once_with(ANY, extra_users=[]) - - @defer.inlineCallbacks - def test_invite_join_target_this(self): - room_id = "foo" - user_id = "@bob:red" - pdu = Pdu( - pdu_type=InviteJoinEvent.TYPE, - user_id=user_id, - target_host=self.hostname, - context=room_id, - content={}, - origin_server_ts=0, - pdu_id="a", - origin="b", + self.state_handler.annotate_state_groups.assert_called_once_with( + ANY, + old_state=None, ) - yield self.handlers.federation_handler.on_receive_pdu(pdu, False) + self.auth.check.assert_called_once_with(ANY, None, raises=True) - mem_handler = self.handlers.room_member_handler - self.assertEquals(1, mem_handler.change_membership.call_count) - call_args = get_mock_call_args( - lambda event, do_auth: None, - mem_handler.change_membership + self.notifier.on_new_room_event.assert_called_once_with( + ANY, + extra_users=[] ) - self.assertEquals(False, call_args["do_auth"]) - - new_event = call_args["event"] - self.assertEquals(RoomMemberEvent.TYPE, new_event.type) - self.assertEquals(room_id, new_event.room_id) - self.assertEquals(user_id, new_event.state_key) - self.assertEquals(Membership.JOIN, new_event.membership) - - @defer.inlineCallbacks - def test_invite_join_target_other(self): - room_id = "foo" - user_id = "@bob:red" - - pdu = Pdu( - pdu_type=InviteJoinEvent.TYPE, - user_id=user_id, - state_key="@red:not%s" % self.hostname, - context=room_id, - content={}, - origin_server_ts=0, - pdu_id="a", - origin="b", - ) - - yield self.handlers.federation_handler.on_receive_pdu(pdu, False) - - mem_handler = self.handlers.room_member_handler - self.assertEquals(0, mem_handler.change_membership.call_count) -- cgit 1.5.1 From dfb3d21a6d2560668a637bba149bf34c7d413125 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 17:12:39 +0000 Subject: Fix room handler tests --- synapse/handlers/room.py | 1 - tests/handlers/test_room.py | 146 ++++++-------------------------------------- 2 files changed, 19 insertions(+), 128 deletions(-) (limited to 'tests') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ffc0892f1a..f176ad39bf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -408,7 +408,6 @@ class RoomMemberHandler(BaseHandler): defer.returnValue({}) return - yield self.state_handler.handle_new_event(event, snapshot) yield self._do_local_membership_update( event, membership=event.content["membership"], diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index c88d1c8840..57a8a1b13d 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from tests import unittest from synapse.api.events.room import ( - InviteJoinEvent, RoomMemberEvent, RoomConfigEvent + RoomMemberEvent, ) from synapse.api.constants import Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler @@ -34,6 +34,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def setUp(self): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + self.hostname = "red" hs = HomeServer( self.hostname, @@ -58,7 +59,10 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "federation_handler", ]), auth=NonCallableMock(spec_set=["check"]), - state_handler=NonCallableMock(spec_set=["handle_new_event"]), + state_handler=NonCallableMock(spec_set=[ + "handle_new_event", + "annotate_state_groups", + ]), config=self.mock_config, ) @@ -114,6 +118,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) + self.datastore.get_room_member.return_value = defer.succeed(None) + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -202,133 +208,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): join_signal_observer.assert_called_with( user=user, room_id=room_id) - @defer.inlineCallbacks - def STALE_test_invite_join(self): - room_id = "foo" - user_id = "@bob:red" - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - target_user_id=target_user_id, - room_id=room_id, - membership=Membership.JOIN, - content=content, - ) - - joined = ["red", "blue", "green"] - - self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) - - store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(None) - - prev_state = NonCallableMock(name="prev_state") - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:blue" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.datastore.get_room_member.assert_called_once_with( - target_user_id, room_id - ) - - self.assertTrue(self.federation.handle_new_event.called) - args = self.federation.handle_new_event.call_args[0] - invite_join_event = args[0] - - self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE) - self.assertTrue("blue", invite_join_event.target_host) - self.assertTrue(room_id, invite_join_event.room_id) - self.assertTrue(user_id, invite_join_event.user_id) - self.assertFalse(hasattr(invite_join_event, "state_key")) - - self.assertEquals( - set(["blue"]), - set(invite_join_event.destinations) - ) - - self.federation.get_state_for_room.assert_called_once_with( - "blue", room_id - ) - - self.assertFalse(self.datastore.store_room_member.called) - - self.assertFalse(self.notifier.on_new_room_event.called) - self.assertFalse(self.state_handler.handle_new_event.called) - - @defer.inlineCallbacks - def STALE_test_invite_join_public(self): - room_id = "#foo:blue" - user_id = "@bob:red" - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - target_user_id=target_user_id, - room_id=room_id, - membership=Membership.JOIN, - content=content, - ) - - joined = ["red", "blue", "green"] - - self.state_handler.handle_new_event.return_value = defer.succeed(True) - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) - - store_id = "store_id_fooo" - self.datastore.store_room_member.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(None) - - prev_state = NonCallableMock(name="prev_state") - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:blue" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.assertTrue(self.federation.handle_new_event.called) - args = self.federation.handle_new_event.call_args[0] - invite_join_event = args[0] - - self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE) - self.assertTrue("blue", invite_join_event.target_host) - self.assertTrue("foo", invite_join_event.room_id) - self.assertTrue(user_id, invite_join_event.user_id) - self.assertFalse(hasattr(invite_join_event, "state_key")) - - self.assertEquals( - set(["blue"]), - set(invite_join_event.destinations) - ) - - self.federation.get_state_for_room.assert_called_once_with( - "blue", "foo" - ) - - self.assertFalse(self.datastore.store_room_member.called) - - self.assertFalse(self.notifier.on_new_room_event.called) - self.assertFalse(self.state_handler.handle_new_event.called) - class RoomCreationTest(unittest.TestCase): def setUp(self): self.hostname = "red" + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + hs = HomeServer( self.hostname, db_pool=None, @@ -346,11 +235,14 @@ class RoomCreationTest(unittest.TestCase): "federation_handler", ]), auth=NonCallableMock(spec_set=["check"]), - state_handler=NonCallableMock(spec_set=["handle_new_event"]), + state_handler=NonCallableMock(spec_set=[ + "handle_new_event", + "annotate_state_groups", + ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), + config=self.mock_config, ) self.federation = NonCallableMock(spec_set=[ @@ -400,6 +292,6 @@ class RoomCreationTest(unittest.TestCase): self.assertEquals(user_id, join_event.user_id) self.assertEquals(user_id, join_event.state_key) - self.assertTrue(self.state_handler.handle_new_event.called) + self.assertTrue(self.state_handler.annotate_state_groups.called) self.assertTrue(self.federation.handle_new_event.called) -- cgit 1.5.1 From 4a5e95511e31aa1836259e94d501a9cfe359d466 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Nov 2014 17:13:21 +0000 Subject: PEP8 --- tests/handlers/test_room.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 57a8a1b13d..f0b77a2496 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -171,7 +171,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_joined_hosts_for_room.side_effect = get_joined - store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) self.datastore.get_room.return_value = defer.succeed(1) # Not None. @@ -203,10 +202,12 @@ class RoomMemberHandlerTestCase(unittest.TestCase): event ) self.notifier.on_new_room_event.assert_called_once_with( - event, extra_users=[user]) + event, extra_users=[user] + ) join_signal_observer.assert_called_with( - user=user, room_id=room_id) + user=user, room_id=room_id + ) class RoomCreationTest(unittest.TestCase): @@ -217,7 +218,6 @@ class RoomCreationTest(unittest.TestCase): self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] - hs = HomeServer( self.hostname, db_pool=None, -- cgit 1.5.1 From 9e429239ab0c304cce4d2f076be5b30691fd93a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:10:36 +0000 Subject: Fix Federation test --- tests/handlers/test_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 63a9990f7a..a9d6b2bb17 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -100,7 +100,7 @@ class FederationTestCase(unittest.TestCase): old_state=None, ) - self.auth.check.assert_called_once_with(ANY, None, raises=True) + self.auth.check.assert_called_once_with(ANY, raises=True) self.notifier.on_new_room_event.assert_called_once_with( ANY, -- cgit 1.5.1 From 34c4614682b08fa89a9f94ded4ecaf882fdc4160 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:10:54 +0000 Subject: Fix presence test --- tests/handlers/test_presence.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index fdc2e8de4a..a6af648def 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -51,6 +51,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "content": content, } ], + "pdu_failures": [], } def _make_edu_json(origin, edu_type, content): -- cgit 1.5.1 From 416ab4ebf0cd12a14f9d6d98226ae52d7d86f3a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:12:47 +0000 Subject: Don't execute empty tests. Formatting. --- tests/rest/test_events.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) (limited to 'tests') diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 362c7bc01c..82f5d277e6 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -25,9 +25,6 @@ import synapse.rest.room from synapse.server import HomeServer -# python imports -import json - from ..utils import MockHttpResource, MemoryDataStore, MockKey from .utils import RestTestCase @@ -49,7 +46,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): def tearDown(self): pass - def test_long_poll(self): + def TODO_test_long_poll(self): # stream from 'end' key, send (self+other) message, expect message. # stream from 'END', send (self+other) message, expect message. @@ -64,7 +61,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): pass - def test_stream_forward(self): + def TODO_test_stream_forward(self): # stream from START, expect injected items # stream from 'start' key, expect same content @@ -80,14 +77,14 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): # returned as end key pass - def test_limits(self): + def TODO_test_limits(self): # stream from a key, expect limit_num items # stream from START, expect limit_num items pass - def test_range(self): + def TODO_test_range(self): # stream from key to key, expect X items # stream from key to END, expect X items @@ -97,7 +94,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase): # stream from START to END, expect all items pass - def test_direction(self): + def TODO_test_direction(self): # stream from END to START and fwds, expect newest first # stream from END to START and bwds, expect oldest first @@ -116,7 +113,10 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=["handle_new_event"]) + state_handler = Mock(spec=[ + "handle_new_event", + "annotate_state_groups" + ]) state_handler.handle_new_event.return_value = True persistence_service = Mock(spec=["get_latest_pdus_in_context"]) @@ -151,6 +151,7 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() hs.get_clock().time_msec.return_value = 1000000 + hs.get_clock().time.return_value = 1000 synapse.rest.register.register_servlets(hs, self.mock_resource) synapse.rest.events.register_servlets(hs, self.mock_resource) @@ -175,12 +176,14 @@ class EventStreamPermissionsTestCase(RestTestCase): def test_stream_basic_permissions(self): # invalid token, expect 403 (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s" % ("invalid" + self.token)) + "/events?access_token=%s" % ("invalid" + self.token, ) + ) self.assertEquals(403, code, msg=str(response)) # valid token, expect content (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token)) + "/events?access_token=%s&timeout=0" % (self.token,) + ) self.assertEquals(200, code, msg=str(response)) self.assertTrue("chunk" in response) self.assertTrue("start" in response) @@ -188,15 +191,23 @@ class EventStreamPermissionsTestCase(RestTestCase): @defer.inlineCallbacks def test_stream_room_permissions(self): - room_id = yield self.create_room_as(self.other_user, - tok=self.other_token) + room_id = yield self.create_room_as( + self.other_user, + tok=self.other_token + ) yield self.send(room_id, tok=self.other_token) # invited to room (expect no content for room) - yield self.invite(room_id, src=self.other_user, targ=self.user_id, - tok=self.other_token) + yield self.invite( + room_id, + src=self.other_user, + targ=self.user_id, + tok=self.other_token + ) + (code, response) = yield self.mock_resource.trigger_get( - "/events?access_token=%s&timeout=0" % (self.token)) + "/events?access_token=%s&timeout=0" % (self.token,) + ) self.assertEquals(200, code, msg=str(response)) self.assertEquals(0, len(response["chunk"])) @@ -206,7 +217,7 @@ class EventStreamPermissionsTestCase(RestTestCase): # left to room (expect no content for room) - def test_stream_items(self): + def TODO_test_stream_items(self): # new user, no content # join room, expect 1 item (join) -- cgit 1.5.1 From 24e2da455783009df1664cf94edf7653b80391a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:13:01 +0000 Subject: Fix presencelike test --- tests/handlers/test_presencelike.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 047752ad68..532ecf0f2c 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -21,7 +21,7 @@ from twisted.internet import defer from mock import Mock, call, ANY -from ..utils import MockClock +from ..utils import MockClock, MockKey from synapse.server import HomeServer from synapse.api.constants import PresenceState @@ -57,6 +57,9 @@ class PresenceAndProfileHandlers(object): class PresenceProfilelikeDataTestCase(unittest.TestCase): def setUp(self): + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=MockClock(), db_pool=None, @@ -72,6 +75,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): resource_for_federation=Mock(), http_client=None, replication_layer=MockReplication(), + config=self.mock_config, ) hs.handlers = PresenceAndProfileHandlers(hs) -- cgit 1.5.1 From f48fce8bd37add27a43306eae255ca40b774a041 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:13:58 +0000 Subject: Fix profile tests --- tests/handlers/test_profile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5dc9b456e1..1660e7e928 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -24,7 +24,7 @@ from synapse.server import HomeServer from synapse.handlers.profile import ProfileHandler from synapse.api.constants import Membership -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey class ProfileHandlers(object): @@ -49,12 +49,16 @@ class ProfileTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", db_pool=db_pool, http_client=None, handlers=None, resource_for_federation=Mock(), replication_layer=self.mock_federation, + config=self.mock_config, ) hs.handlers = ProfileHandlers(hs) -- cgit 1.5.1 From c670ce416bcbdeb929ce06dabcd4bce841ec815d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:15:15 +0000 Subject: Fix room tests --- tests/handlers/test_room.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'tests') diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index f0b77a2496..146ed21b65 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -123,9 +123,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield self.room_member_handler.change_membership(event) - self.state_handler.handle_new_event.assert_called_once_with( - event, self.snapshot, - ) self.federation.handle_new_event.assert_called_once_with( event, self.snapshot, ) @@ -186,9 +183,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield self.room_member_handler.change_membership(event) - self.state_handler.handle_new_event.assert_called_once_with( - event, self.snapshot - ) self.federation.handle_new_event.assert_called_once_with( event, self.snapshot ) -- cgit 1.5.1 From 1616df2f610ea2cc7c1fa32b98f5a63004d978de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 11:15:56 +0000 Subject: Fix typing tests --- tests/handlers/test_typing.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests') diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index f1d3b27f74..07acda5eee 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -40,6 +40,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "content": content, } ], + "pdu_failures": [], } -- cgit 1.5.1 From da80ebcc6b0bd15095d092eacd0ea55c81bb03b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:27:20 +0000 Subject: Fix redaction storage test --- tests/storage/test_redaction.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'tests') diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index dae1641ea1..fb175a5c4c 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -61,6 +61,7 @@ class RedactionTestCase(unittest.TestCase): membership=membership, content={"membership": membership}, depth=self.depth, + prev_events=[], ) event.content.update(extra_content) @@ -68,6 +69,9 @@ class RedactionTestCase(unittest.TestCase): if prev_state: event.prev_state = prev_state + event.state_events = None + event.hashes = {} + # Have to create a join event using the eventfactory yield self.store.persist_event( event @@ -85,8 +89,12 @@ class RedactionTestCase(unittest.TestCase): room_id=room.to_string(), content={"body": body, "msgtype": u"message"}, depth=self.depth, + prev_events=[], ) + event.state_events = None + event.hashes = {} + yield self.store.persist_event( event ) @@ -102,8 +110,12 @@ class RedactionTestCase(unittest.TestCase): content={"reason": reason}, depth=self.depth, redacts=event_id, + prev_events=[], ) + event.state_events = None + event.hashes = {} + yield self.store.persist_event( event ) -- cgit 1.5.1 From 2fcce3b3c508ba1b10117e16511fb577da82f0b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:43:36 +0000 Subject: Remove stale tests --- tests/storage/test_room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index 369a73d917..4ff02c306b 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -127,7 +127,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_room_name(self): + def STALE_test_room_name(self): name = u"A-Room-Name" yield self.inject_room_event( @@ -150,7 +150,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_room_name(self): + def STALE_test_room_topic(self): topic = u"A place for things" yield self.inject_room_event( -- cgit 1.5.1 From 3791b75000f9787806dbdce3e6e216da1ac07388 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Nov 2014 13:48:36 +0000 Subject: Fix a couple more storage tests --- tests/storage/test_roommember.py | 24 +++++++++++++++--------- tests/storage/test_stream.py | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index eae278ee8d..9e2b2e31c8 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -51,16 +51,22 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership): # Have to create a join event using the eventfactory + event = self.event_factory.create_event( + etype=RoomMemberEvent.TYPE, + user_id=user.to_string(), + state_key=user.to_string(), + room_id=room.to_string(), + membership=membership, + content={"membership": membership}, + depth=1, + prev_events=[], + ) + + event.state_events = None + event.hashes = {} + yield self.store.persist_event( - self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - user_id=user.to_string(), - state_key=user.to_string(), - room_id=room.to_string(), - membership=membership, - content={"membership": membership}, - depth=1, - ) + event ) @defer.inlineCallbacks diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index ab30e6ea25..b034fd8f16 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -59,8 +59,12 @@ class StreamStoreTestCase(unittest.TestCase): membership=membership, content={"membership": membership}, depth=self.depth, + prev_events=[], ) + event.state_events = None + event.hashes = {} + if prev_state: event.prev_state = prev_state @@ -75,15 +79,21 @@ class StreamStoreTestCase(unittest.TestCase): def inject_message(self, room, user, body): self.depth += 1 + event = self.event_factory.create_event( + etype=MessageEvent.TYPE, + user_id=user.to_string(), + room_id=room.to_string(), + content={"body": body, "msgtype": u"message"}, + depth=self.depth, + prev_events=[], + ) + + event.state_events = None + event.hashes = {} + # Have to create a join event using the eventfactory yield self.store.persist_event( - self.event_factory.create_event( - etype=MessageEvent.TYPE, - user_id=user.to_string(), - room_id=room.to_string(), - content={"body": body, "msgtype": u"message"}, - depth=self.depth, - ) + event ) @defer.inlineCallbacks -- cgit 1.5.1 From b01159f2348de61a085d7ea20cc0fc90db2fc080 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 14:58:33 +0000 Subject: Fix room handler test --- tests/handlers/test_room.py | 55 ++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 15 deletions(-) (limited to 'tests') diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 146ed21b65..55c9f6e142 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -58,9 +58,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "profile_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check"]), + auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "handle_new_event", "annotate_state_groups", ]), config=self.mock_config, @@ -68,6 +67,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.federation = NonCallableMock(spec_set=[ "handle_new_event", + "send_invite", "get_state_for_room", ]) @@ -110,7 +110,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): joined = ["red", "green"] - self.state_handler.handle_new_event.return_value = defer.succeed(True) self.datastore.get_joined_hosts_for_room.return_value = ( defer.succeed(joined) ) @@ -120,6 +119,18 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.datastore.get_room_member.return_value = defer.succeed(None) + event.state_events = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + user_id="@bob:red", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, target_user_id): event, + } + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -128,7 +139,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ) self.assertEquals( - set(["blue", "red", "green"]), + set(["red", "green"]), set(event.destinations) ) @@ -147,22 +158,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): room_id = "!foo:red" user_id = "@bob:red" user = self.hs.parse_userid(user_id) - target_user_id = "@bob:red" - content = {"membership": Membership.JOIN} - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, + event = self._create_member( user_id=user_id, - state_key=target_user_id, room_id=room_id, - membership=Membership.JOIN, - content=content, ) joined = ["red", "green"] - self.state_handler.handle_new_event.return_value = defer.succeed(True) - def get_joined(*args): return defer.succeed(joined) @@ -180,6 +183,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) + event.state_events = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, user_id): event, + } + # Actual invocation yield self.room_member_handler.change_membership(event) @@ -203,6 +214,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user=user, room_id=room_id ) + def _create_member(self, user_id, room_id): + return self.hs.get_event_factory().create_event( + etype=RoomMemberEvent.TYPE, + user_id=user_id, + state_key=user_id, + room_id=room_id, + membership=Membership.JOIN, + content={"membership": Membership.JOIN}, + ) + class RoomCreationTest(unittest.TestCase): @@ -228,9 +249,8 @@ class RoomCreationTest(unittest.TestCase): "room_member_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check"]), + auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "handle_new_event", "annotate_state_groups", ]), ratelimiter=NonCallableMock(spec_set=[ @@ -259,6 +279,11 @@ class RoomCreationTest(unittest.TestCase): ]) self.room_member_handler = self.handlers.room_member_handler + def annotate(event): + event.state_events = {} + return defer.succeed(None) + self.state_handler.annotate_state_groups.side_effect = annotate + def hosts(room): return defer.succeed([]) self.datastore.get_joined_hosts_for_room.side_effect = hosts -- cgit 1.5.1 From c174d19d1e107c31fe5d2d2b919bbd49a5b86e56 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:21:41 +0000 Subject: Fix redaction storage test --- tests/storage/test_redaction.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tests') diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index fb175a5c4c..adfe64a980 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -71,6 +71,8 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = [] + event.auth_events = [] # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -94,6 +96,7 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] yield self.store.persist_event( event @@ -115,6 +118,7 @@ class RedactionTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] yield self.store.persist_event( event -- cgit 1.5.1 From e2cebe26e8ba1d7c2a39594ea1f4be0dee73ace4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:24:15 +0000 Subject: Fix room_member storage test --- tests/storage/test_roommember.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tests') diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 9e2b2e31c8..8614e5ca9d 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -64,6 +64,8 @@ class RoomMemberStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = {} + event.auth_events = {} yield self.store.persist_event( event -- cgit 1.5.1 From 3cd9c02f71f967918f494e61121a8ecb18e67611 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:29:19 +0000 Subject: Fix stream test. --- tests/storage/test_stream.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index b034fd8f16..5038546aee 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -48,7 +48,7 @@ class StreamStoreTestCase(unittest.TestCase): self.depth = 1 @defer.inlineCallbacks - def inject_room_member(self, room, user, membership, prev_state=None): + def inject_room_member(self, room, user, membership, replaces_state=None): self.depth += 1 event = self.event_factory.create_event( @@ -64,9 +64,12 @@ class StreamStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.prev_state = [] + event.auth_events = [] - if prev_state: - event.prev_state = prev_state + if replaces_state: + event.prev_state = [(replaces_state, "hash")] + event.replaces_state = replaces_state # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -90,6 +93,7 @@ class StreamStoreTestCase(unittest.TestCase): event.state_events = None event.hashes = {} + event.auth_events = [] # Have to create a join event using the eventfactory yield self.store.persist_event( @@ -216,7 +220,7 @@ class StreamStoreTestCase(unittest.TestCase): event2 = yield self.inject_room_member( self.room1, self.u_alice, Membership.JOIN, - prev_state=event1.event_id, + replaces_state=event1.event_id, ) end = yield self.store.get_room_events_max_id() @@ -233,4 +237,7 @@ class StreamStoreTestCase(unittest.TestCase): event = results[0] - self.assertTrue(hasattr(event, "prev_content"), msg="No prev_content key") + self.assertTrue( + hasattr(event, "prev_content"), + msg="No prev_content key" + ) -- cgit 1.5.1 From 4ebdb1968225243bab86ef5d8d9f8feb274dd9ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:32:35 +0000 Subject: Fix SQLBaseStoreTestCase --- tests/storage/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 3ad9a4b0c0..fabd364be9 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -74,7 +74,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_select_one_1col(self): self.mock_txn.rowcount = 1 - self.mock_txn.fetchone.return_value = ("Value",) + self.mock_txn.fetchall.return_value = [("Value",)] value = yield self.datastore._simple_select_one_onecol( table="tablename", -- cgit 1.5.1 From ec824927c1e1af76abad1ea0c5eea1098a5c9cfa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Nov 2014 15:37:53 +0000 Subject: Fix rest.test_events. Convert to use SQLiteMemoryDbPool --- tests/rest/test_events.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'tests') diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 82f5d277e6..4a3234c332 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -25,7 +25,7 @@ import synapse.rest.room from synapse.server import HomeServer -from ..utils import MockHttpResource, MemoryDataStore, MockKey +from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -113,25 +113,20 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=[ - "handle_new_event", - "annotate_state_groups" - ]) - state_handler.handle_new_event.return_value = True - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) persistence_service.get_latest_pdus_in_context.return_value = [] self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "test", - db_pool=None, + db_pool=db_pool, http_client=None, replication_layer=Mock(), - state_handler=state_handler, - datastore=MemoryDataStore(), persistence_service=persistence_service, clock=Mock(spec=[ "call_later", -- cgit 1.5.1 From 50c8e3fcda0a892e9205be046bf8e0a931514e15 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 18:07:55 +0000 Subject: Initial (empty) test that room initialSync at least returns 200 OK --- tests/rest/test_rooms.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) (limited to 'tests') diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1ce9b8a83d..69dade4548 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -940,6 +940,59 @@ class RoomMessagesTestCase(RestTestCase): (code, response) = yield self.mock_resource.trigger("PUT", path, content) self.assertEquals(200, code, msg=str(response)) + +class RoomInitialSyncTestCase(RestTestCase): + """ Tests /rooms/$room_id/initialSync. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + state_handler = Mock(spec=["handle_new_event"]) + state_handler.handle_new_event.return_value = True + + persistence_service = Mock(spec=["get_latest_pdus_in_context"]) + persistence_service.get_latest_pdus_in_context.return_value = [] + + hs = HomeServer( + "red", + db_pool=None, + http_client=None, + datastore=MemoryDataStore(), + replication_layer=Mock(), + state_handler=state_handler, + persistence_service=persistence_service, + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=NonCallableMock(), + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + synapse.rest.room.register_servlets(hs, self.mock_resource) + + # create the room + self.room_id = yield self.create_room_as(self.user_id) + + @defer.inlineCallbacks + def test_initial_sync(self): + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/initialSync" % self.room_id) + self.assertEquals(200, code) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) -- cgit 1.5.1 From 269f80bf8ecc4a89c2bb71f0872893bb53a252ca Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 19:02:19 +0000 Subject: Have room initialSync return the room's room_id --- synapse/handlers/message.py | 11 +++++++++++ synapse/rest/room.py | 27 ++++++--------------------- tests/rest/test_rooms.py | 2 ++ 3 files changed, 19 insertions(+), 21 deletions(-) (limited to 'tests') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 72894869ea..bb1290fd05 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -331,4 +331,15 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def snapshot_room(self, user_id, room_id, pagin_config=None, + feedback=False): + yield self.auth.check_joined_room(room_id, user_id) + defer.returnValue({ + #"membership": membership, + "room_id": room_id, + #"messages": messages, + #"state": state, + #"presence": presence, + }) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 7724967061..b762de1cb3 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -353,27 +353,12 @@ class RoomInitialSyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - yield self.auth.get_user_by_req(request) - # TODO: Get all the initial sync data for this room and return in the - # same format as initial sync, that is: - # { - # membership: join, - # messages: [ - # chunk: [ msg events ], - # start: s_tok, - # end: e_tok - # ], - # room_id: foo, - # state: [ - # { state event } , { state event } - # ] - # } - # Probably worth keeping the keys room_id and membership for parity - # with /initialSync even though they must be joined to sync this and - # know the room ID, so clients can reuse the same code (room_id and - # membership are MANDATORY for /initialSync, so the code will expect - # it to be there) - defer.returnValue((200, {})) + user = yield self.auth.get_user_by_req(request) + events = yield self.handlers.message_handler.snapshot_room( + room_id=room_id, + user_id=user.to_string(), + ) + defer.returnValue((200, events)) class RoomTriggerBackfill(RestServlet): diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 69dade4548..129f4d49f4 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -993,6 +993,8 @@ class RoomInitialSyncTestCase(RestTestCase): "/rooms/%s/initialSync" % self.room_id) self.assertEquals(200, code) + self.assertEquals(self.room_id, response["room_id"]) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) -- cgit 1.5.1 From 1fd81391385703475bd5c30e7617228cf33c7087 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 19:29:58 +0000 Subject: Put room state in room initialSync output - I guess this is right; I really can't find any other tests similar... --- synapse/handlers/message.py | 7 +++++-- tests/rest/test_rooms.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bb1290fd05..d8764a8933 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -336,10 +336,13 @@ class MessageHandler(BaseHandler): feedback=False): yield self.auth.check_joined_room(room_id, user_id) + state_tuples = yield self.store.get_current_state(room_id) + state = [self.hs.serialize_event(x) for x in state_tuples] + defer.returnValue({ #"membership": membership, "room_id": room_id, #"messages": messages, - #"state": state, - #"presence": presence, + "state": state, + #"presence": presence }) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 129f4d49f4..1e8b16c36a 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -984,6 +984,29 @@ class RoomInitialSyncTestCase(RestTestCase): synapse.rest.room.register_servlets(hs, self.mock_resource) + # MemoryDataStore's get_current_state() isn't even approximately + # right. Longterm we'll be killing it in favour of a real + # SQLite :memory:-based test. For now lets patch it + efact = hs.get_event_factory() + def get_current_state(room_id): + """ + TODO: these probably need content + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.send_event_level', ''), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.power_levels', ''), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.ops_levels', ''), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.member', '@sid1:red'), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.add_state_level', '')] + """ + return [ + efact.create_event(etype="m.room.create", room_id=room_id, + content={}, + ), + efact.create_event(etype="m.room.join_rules", room_id=room_id, + content={}, + ), + ] + hs.datastore.get_current_state = get_current_state + # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -995,6 +1018,18 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertEquals(self.room_id, response["room_id"]) + # Room state is easier to assert on if we unpack it into a dict + state = {} + for event in response["state"]: + if "state_key" not in event: + continue + t = event["type"] + if t not in state: + state[t] = [] + state[t].append(event) + + self.assertTrue("m.room.create" in state) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) -- cgit 1.5.1 From c23afed39a3a2796a53caaec19de2d53873956e5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 19:34:47 +0000 Subject: Include room membership in room initialSync --- synapse/handlers/message.py | 8 +++++++- tests/rest/test_rooms.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index d8764a8933..811e280c2d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -336,11 +336,17 @@ class MessageHandler(BaseHandler): feedback=False): yield self.auth.check_joined_room(room_id, user_id) + # TODO: These concurrently state_tuples = yield self.store.get_current_state(room_id) state = [self.hs.serialize_event(x) for x in state_tuples] + member_event = (yield self.store.get_room_member( + user_id=user_id, + room_id=room_id + )) + defer.returnValue({ - #"membership": membership, + "membership": member_event.membership, "room_id": room_id, #"messages": messages, "state": state, diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1e8b16c36a..29f5776f49 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1017,6 +1017,7 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertEquals(200, code) self.assertEquals(self.room_id, response["room_id"]) + self.assertEquals("join", response["membership"]) # Room state is easier to assert on if we unpack it into a dict state = {} -- cgit 1.5.1 From 339c11dd86022a2fb4235c01a791cd60cffad162 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 11 Nov 2014 08:09:42 +0000 Subject: Fix rest.test_rooms --- tests/rest/test_rooms.py | 203 ++++++++++++++++++++++++++--------------------- 1 file changed, 113 insertions(+), 90 deletions(-) (limited to 'tests') diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 7170193051..61b01d369d 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -23,11 +23,14 @@ from synapse.api.constants import Membership from synapse.server import HomeServer +from tests import unittest + # python imports import json import urllib +import types -from ..utils import MockHttpResource, MemoryDataStore, MockKey +from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -44,23 +47,17 @@ class RoomPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=["handle_new_event"]) - state_handler.handle_new_event.return_value = True - - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "red", - db_pool=None, + db_pool=db_pool, http_client=None, - datastore=MemoryDataStore(), replication_layer=Mock(), - state_handler=state_handler, - persistence_service=persistence_service, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -79,6 +76,10 @@ class RoomPermissionsTestCase(RestTestCase): } hs.get_auth().get_user_by_token = _get_user_by_token + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + self.auth_user_id = self.rmcreator_id synapse.rest.room.register_servlets(hs, self.mock_resource) @@ -150,38 +151,55 @@ class RoomPermissionsTestCase(RestTestCase): @defer.inlineCallbacks def test_send_message(self): msg_content = '{"msgtype":"m.text","body":"hello"}' - send_msg_path = ("/rooms/%s/send/m.room.message/mid1" % - (self.created_rmid)) + send_msg_path = ( + "/rooms/%s/send/m.room.message/mid1" % (self.created_rmid,) + ) # send message in uncreated room, expect 403 (code, response) = yield self.mock_resource.trigger( - "PUT", - "/rooms/%s/send/m.room.message/mid2" % - (self.uncreated_rmid), msg_content) + "PUT", + "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,), + msg_content + ) self.assertEquals(403, code, msg=str(response)) # send message in created room not joined (no state), expect 403 (code, response) = yield self.mock_resource.trigger( - "PUT", send_msg_path, msg_content) + "PUT", + send_msg_path, + msg_content + ) self.assertEquals(403, code, msg=str(response)) # send message in created room and invited, expect 403 - yield self.invite(room=self.created_rmid, src=self.rmcreator_id, - targ=self.user_id) + yield self.invite( + room=self.created_rmid, + src=self.rmcreator_id, + targ=self.user_id + ) (code, response) = yield self.mock_resource.trigger( - "PUT", send_msg_path, msg_content) + "PUT", + send_msg_path, + msg_content + ) self.assertEquals(403, code, msg=str(response)) # send message in created room and joined, expect 200 yield self.join(room=self.created_rmid, user=self.user_id) (code, response) = yield self.mock_resource.trigger( - "PUT", send_msg_path, msg_content) + "PUT", + send_msg_path, + msg_content + ) self.assertEquals(200, code, msg=str(response)) # send message in created room and left, expect 403 yield self.leave(room=self.created_rmid, user=self.user_id) (code, response) = yield self.mock_resource.trigger( - "PUT", send_msg_path, msg_content) + "PUT", + send_msg_path, + msg_content + ) self.assertEquals(403, code, msg=str(response)) @defer.inlineCallbacks @@ -218,9 +236,14 @@ class RoomPermissionsTestCase(RestTestCase): # set/get topic in created PRIVATE room and joined, expect 200 yield self.join(room=self.created_rmid, user=self.user_id) + + # Only room ops can set topic by default + self.auth_user_id = self.rmcreator_id (code, response) = yield self.mock_resource.trigger( "PUT", topic_path, topic_content) self.assertEquals(200, code, msg=str(response)) + self.auth_user_id = self.user_id + (code, response) = yield self.mock_resource.trigger_get(topic_path) self.assertEquals(200, code, msg=str(response)) self.assert_dict(json.loads(topic_content), response) @@ -384,44 +407,51 @@ class RoomPermissionsTestCase(RestTestCase): # set [invite/join/left] of self, set [invite/join/left] of other, # expect all 403s for usr in [self.user_id, self.rmcreator_id]: - yield self.change_membership(room=room, src=self.user_id, - targ=usr, - membership=Membership.INVITE, - expect_code=403) - yield self.change_membership(room=room, src=self.user_id, - targ=usr, - membership=Membership.JOIN, - expect_code=403) - yield self.change_membership(room=room, src=self.user_id, - targ=usr, - membership=Membership.LEAVE, - expect_code=403) + yield self.change_membership( + room=room, + src=self.user_id, + targ=usr, + membership=Membership.INVITE, + expect_code=403 + ) + + yield self.change_membership( + room=room, + src=self.user_id, + targ=usr, + membership=Membership.JOIN, + expect_code=403 + ) + + # It is always valid to LEAVE if you've already left (currently.) + yield self.change_membership( + room=room, + src=self.user_id, + targ=self.rmcreator_id, + membership=Membership.LEAVE, + expect_code=403 + ) class RoomsMemberListTestCase(RestTestCase): """ Tests /rooms/$room_id/members/list REST events.""" user_id = "@sid1:red" + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - state_handler = Mock(spec=["handle_new_event"]) - state_handler.handle_new_event.return_value = True - - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "red", - db_pool=None, + db_pool=db_pool, http_client=None, - datastore=MemoryDataStore(), replication_layer=Mock(), - state_handler=state_handler, - persistence_service=persistence_service, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -442,6 +472,10 @@ class RoomsMemberListTestCase(RestTestCase): } hs.get_auth().get_user_by_token = _get_user_by_token + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + synapse.rest.room.register_servlets(hs, self.mock_resource) def tearDown(self): @@ -493,27 +527,22 @@ class RoomsCreateTestCase(RestTestCase): """ Tests /rooms and /rooms/$room_id REST events. """ user_id = "@sid1:red" + @defer.inlineCallbacks def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - state_handler = Mock(spec=["handle_new_event"]) - state_handler.handle_new_event.return_value = True - - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "red", - db_pool=None, + db_pool=db_pool, http_client=None, - datastore=MemoryDataStore(), replication_layer=Mock(), - state_handler=state_handler, - persistence_service=persistence_service, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -532,6 +561,10 @@ class RoomsCreateTestCase(RestTestCase): } hs.get_auth().get_user_by_token = _get_user_by_token + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + synapse.rest.room.register_servlets(hs, self.mock_resource) def tearDown(self): @@ -601,23 +634,17 @@ class RoomTopicTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - state_handler = Mock(spec=["handle_new_event"]) - state_handler.handle_new_event.return_value = True - - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "red", - db_pool=None, + db_pool=db_pool, http_client=None, - datastore=MemoryDataStore(), replication_layer=Mock(), - state_handler=state_handler, - persistence_service=persistence_service, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -634,13 +661,18 @@ class RoomTopicTestCase(RestTestCase): "admin": False, "device_id": None, } + hs.get_auth().get_user_by_token = _get_user_by_token + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + synapse.rest.room.register_servlets(hs, self.mock_resource) # create the room self.room_id = yield self.create_room_as(self.user_id) - self.path = "/rooms/%s/state/m.room.topic" % self.room_id + self.path = "/rooms/%s/state/m.room.topic" % (self.room_id,) def tearDown(self): pass @@ -718,23 +750,17 @@ class RoomMemberStateTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - state_handler = Mock(spec=["handle_new_event"]) - state_handler.handle_new_event.return_value = True - - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "red", - db_pool=None, + db_pool=db_pool, http_client=None, - datastore=MemoryDataStore(), replication_layer=Mock(), - state_handler=state_handler, - persistence_service=persistence_service, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -751,13 +777,12 @@ class RoomMemberStateTestCase(RestTestCase): "admin": False, "device_id": None, } - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } hs.get_auth().get_user_by_token = _get_user_by_token + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + synapse.rest.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -862,23 +887,17 @@ class RoomMessagesTestCase(RestTestCase): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id - state_handler = Mock(spec=["handle_new_event"]) - state_handler.handle_new_event.return_value = True - - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + hs = HomeServer( "red", - db_pool=None, + db_pool=db_pool, http_client=None, - datastore=MemoryDataStore(), replication_layer=Mock(), - state_handler=state_handler, - persistence_service=persistence_service, ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -897,6 +916,10 @@ class RoomMessagesTestCase(RestTestCase): } hs.get_auth().get_user_by_token = _get_user_by_token + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + synapse.rest.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) -- cgit 1.5.1 From ed8b7d400c32408732ed68fe5230314d1bf10b9c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 11 Nov 2014 10:31:59 +0000 Subject: Fix validation tests --- tests/events/test_events.py | 55 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/tests/events/test_events.py b/tests/events/test_events.py index a4b6cb3afd..91d1d44fee 100644 --- a/tests/events/test_events.py +++ b/tests/events/test_events.py @@ -14,6 +14,8 @@ # limitations under the License. from synapse.api.events import SynapseEvent +from synapse.api.events.validator import EventValidator +from synapse.api.errors import SynapseError from tests import unittest @@ -21,7 +23,7 @@ from tests import unittest class SynapseTemplateCheckTestCase(unittest.TestCase): def setUp(self): - pass + self.validator = EventValidator(None) def tearDown(self): pass @@ -38,22 +40,28 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): } event = MockSynapseEvent(template) - self.assertTrue(event.check_json(content, raises=False)) + event.content = content + self.assertTrue(self.validator.validate(event)) content = { "person": {"name": "bob"}, "friends": ["jill"], "enemies": ["mike"] } - event = MockSynapseEvent(template) - self.assertTrue(event.check_json(content, raises=False)) + event.content = content + self.assertTrue(self.validator.validate(event)) content = { "person": {"name": "bob"}, # missing friends "enemies": ["mike", "jill"] } - self.assertFalse(event.check_json(content, raises=False)) + event.content = content + self.assertRaises( + SynapseError, + self.validator.validate, + event + ) def test_lists(self): template = { @@ -67,13 +75,19 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): } event = MockSynapseEvent(template) - self.assertFalse(event.check_json(content, raises=False)) + event.content = content + self.assertRaises( + SynapseError, + self.validator.validate, + event + ) content = { "person": {"name": "bob"}, "friends": [{"name": "jill"}, {"name": "mike"}] } - self.assertTrue(event.check_json(content, raises=False)) + event.content = content + self.assertTrue(self.validator.validate(event)) def test_nested_lists(self): template = { @@ -103,7 +117,12 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): } event = MockSynapseEvent(template) - self.assertFalse(event.check_json(content, raises=False)) + event.content = content + self.assertRaises( + SynapseError, + self.validator.validate, + event + ) content = { "results": { @@ -117,7 +136,8 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): ] } } - self.assertTrue(event.check_json(content, raises=False)) + event.content = content + self.assertTrue(self.validator.validate(event)) def test_nested_keys(self): template = { @@ -145,7 +165,8 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): } } - self.assertTrue(event.check_json(content, raises=False)) + event.content = content + self.assertTrue(self.validator.validate(event)) content = { "person": { @@ -159,7 +180,12 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): } } - self.assertFalse(event.check_json(content, raises=False)) + event.content = content + self.assertRaises( + SynapseError, + self.validator.validate, + event + ) content = { "person": { @@ -173,7 +199,12 @@ class SynapseTemplateCheckTestCase(unittest.TestCase): } } - self.assertFalse(event.check_json(content, raises=False)) + event.content = content + self.assertRaises( + SynapseError, + self.validator.validate, + event + ) class MockSynapseEvent(SynapseEvent): -- cgit 1.5.1 From 02ebb9f0c3ab56bc8e9f19721e6189e9486252a3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 11 Nov 2014 14:16:48 +0000 Subject: Fix state tests --- tests/test_state.py | 693 +++++++++++++--------------------------------------- 1 file changed, 176 insertions(+), 517 deletions(-) (limited to 'tests') diff --git a/tests/test_state.py b/tests/test_state.py index 4b1feaf410..3cc358be32 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -15,599 +15,258 @@ from tests import unittest from twisted.internet import defer -from twisted.python.log import PythonLoggingObserver from synapse.state import StateHandler -from synapse.storage.pdu import PduEntry -from synapse.federation.pdu_codec import encode_event_id -from synapse.federation.units import Pdu - -from collections import namedtuple from mock import Mock -import mock - - -ReturnType = namedtuple( - "StateReturnType", ["new_branch", "current_branch"] -) - - -def _gen_get_power_level(power_level_list): - def get_power_level(room_id, user_id): - return defer.succeed(power_level_list.get(user_id, None)) - return get_power_level class StateTestCase(unittest.TestCase): def setUp(self): - self.persistence = Mock(spec=[ - "get_unresolved_state_tree", - "update_current_state", - "get_latest_pdus_in_context", - "get_current_state_pdu", - "get_pdu", - "get_power_level", - ]) - self.replication = Mock(spec=["get_pdu"]) - - hs = Mock(spec=["get_datastore", "get_replication_layer"]) - hs.get_datastore.return_value = self.persistence - hs.get_replication_layer.return_value = self.replication - hs.hostname = "bob.com" - - self.state = StateHandler(hs) - - @defer.inlineCallbacks - def test_new_state_key(self): - # We've never seen anything for this state before - new_pdu = new_fake_pdu("A", "test", "mem", "x", None, "u") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({}) - - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu], []), None) - ) - - is_new = yield self.state.handle_new_state(new_pdu) - - self.assertTrue(is_new) - - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu - ) - - self.assertEqual(1, self.persistence.update_current_state.call_count) - - self.assertFalse(self.replication.get_pdu.called) - - @defer.inlineCallbacks - def test_direct_overwrite(self): - # We do a direct overwriting of the old state, i.e., the new state - # points to the old state. - - old_pdu = new_fake_pdu("A", "test", "mem", "x", None, "u1") - new_pdu = new_fake_pdu("B", "test", "mem", "x", "A", "u2") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 5, - }) - - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu, old_pdu], [old_pdu]), None) - ) - - is_new = yield self.state.handle_new_state(new_pdu) - - self.assertTrue(is_new) - - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu + self.store = Mock( + spec_set=[ + "get_state_groups", + ] ) + hs = Mock(spec=["get_datastore"]) + hs.get_datastore.return_value = self.store - self.assertEqual(1, self.persistence.update_current_state.call_count) - - self.assertFalse(self.replication.get_pdu.called) + self.state = StateHandler(hs) + self.event_id = 0 @defer.inlineCallbacks - def test_overwrite(self): - old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") - old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", "A", "u2") - new_pdu = new_fake_pdu("C", "test", "mem", "x", "B", "u3") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 5, - "u3": 0, - }) - - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu, old_pdu_2, old_pdu_1], [old_pdu_1]), None) - ) + def test_annotate_with_old_message(self): + event = self.create_event(type="test_message", name="event") - is_new = yield self.state.handle_new_state(new_pdu) + old_state = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), + ] - self.assertTrue(is_new) + yield self.state.annotate_state_groups(event, old_state=old_state) - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu - ) + for k, v in event.old_state_events.items(): + type, state_key = k + self.assertEqual(type, v.type) + self.assertEqual(state_key, v.state_key) - self.assertEqual(1, self.persistence.update_current_state.call_count) + self.assertEqual(set(old_state), set(event.old_state_events.values())) + self.assertDictEqual(event.old_state_events, event.state_events) - self.assertFalse(self.replication.get_pdu.called) + self.assertIsNone(event.state_group) @defer.inlineCallbacks - def test_power_level_fail(self): - # We try to update the state based on an outdated state, and have a - # too low power level. - - old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") - old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") - new_pdu = new_fake_pdu("C", "test", "mem", "x", "A", "u3") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 5, - }) - - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) - ) - - is_new = yield self.state.handle_new_state(new_pdu) - - self.assertFalse(is_new) - - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu - ) - - self.assertEqual(0, self.persistence.update_current_state.call_count) + def test_annotate_with_old_state(self): + event = self.create_event(type="state", state_key="", name="event") - self.assertFalse(self.replication.get_pdu.called) - - @defer.inlineCallbacks - def test_power_level_succeed(self): - # We try to update the state based on an outdated state, but have - # sufficient power level to force the update. - - old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") - old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") - new_pdu = new_fake_pdu("C", "test", "mem", "x", "A", "u3") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 15, - }) - - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) - ) + old_state = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), + ] - is_new = yield self.state.handle_new_state(new_pdu) + yield self.state.annotate_state_groups(event, old_state=old_state) - self.assertTrue(is_new) + for k, v in event.old_state_events.items(): + type, state_key = k + self.assertEqual(type, v.type) + self.assertEqual(state_key, v.state_key) - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu + self.assertEqual( + set(old_state + [event]), + set(event.old_state_events.values()) ) - self.assertEqual(1, self.persistence.update_current_state.call_count) + self.assertDictEqual(event.old_state_events, event.state_events) - self.assertFalse(self.replication.get_pdu.called) + self.assertIsNone(event.state_group) @defer.inlineCallbacks - def test_power_level_equal_same_len(self): - # We try to update the state based on an outdated state, the power - # levels are the same and so are the branch lengths - - old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") - old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") - new_pdu = new_fake_pdu("C", "test", "mem", "x", "A", "u3") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 10, - }) - - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1]), None) - ) - - is_new = yield self.state.handle_new_state(new_pdu) + def test_trivial_annotate_message(self): + event = self.create_event(type="test_message", name="event") + event.prev_events = [] + + old_state = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), + ] - self.assertTrue(is_new) + group_name = "group_name_1" - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu - ) + self.store.get_state_groups.return_value = { + group_name: old_state, + } - self.assertEqual(1, self.persistence.update_current_state.call_count) + yield self.state.annotate_state_groups(event) - self.assertFalse(self.replication.get_pdu.called) + for k, v in event.old_state_events.items(): + type, state_key = k + self.assertEqual(type, v.type) + self.assertEqual(state_key, v.state_key) - @defer.inlineCallbacks - def test_power_level_equal_diff_len(self): - # We try to update the state based on an outdated state, the power - # levels are the same but the branch length of the new one is longer. - - old_pdu_1 = new_fake_pdu("A", "test", "mem", "x", None, "u1") - old_pdu_2 = new_fake_pdu("B", "test", "mem", "x", None, "u2") - old_pdu_3 = new_fake_pdu("C", "test", "mem", "x", "A", "u3") - new_pdu = new_fake_pdu("D", "test", "mem", "x", "C", "u4") - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 10, - "u4": 10, - }) - - self.persistence.get_unresolved_state_tree.return_value = ( - ( - ReturnType( - [new_pdu, old_pdu_3, old_pdu_1], - [old_pdu_2, old_pdu_1] - ), - None - ) + self.assertEqual( + set([e.event_id for e in old_state]), + set([e.event_id for e in event.old_state_events.values()]) ) - is_new = yield self.state.handle_new_state(new_pdu) - - self.assertTrue(is_new) - - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu + self.assertDictEqual( + { + k: v.event_id + for k, v in event.old_state_events.items() + }, + { + k: v.event_id + for k, v in event.state_events.items() + } ) - self.assertEqual(1, self.persistence.update_current_state.call_count) - - self.assertFalse(self.replication.get_pdu.called) + self.assertEqual(group_name, event.state_group) @defer.inlineCallbacks - def test_missing_pdu(self): - # We try to update state against a PDU we haven't yet seen, - # triggering a get_pdu request - - # The pdu we haven't seen - old_pdu_1 = new_fake_pdu( - "A", "test", "mem", "x", None, "u1", depth=0 - ) - - old_pdu_2 = new_fake_pdu( - "B", "test", "mem", "x", "A", "u2", depth=1 - ) - new_pdu = new_fake_pdu( - "C", "test", "mem", "x", "A", "u3", depth=2 - ) - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 20, - }) - - # The return_value of `get_unresolved_state_tree`, which changes after - # the call to get_pdu - tree_to_return = [(ReturnType([new_pdu], [old_pdu_2]), 0)] - - def return_tree(p): - return tree_to_return[0] - - def set_return_tree(destination, pdu_origin, pdu_id, outlier=False): - tree_to_return[0] = ( - ReturnType( - [new_pdu, old_pdu_1], [old_pdu_2, old_pdu_1] - ), - None - ) - return defer.succeed(None) - - self.persistence.get_unresolved_state_tree.side_effect = return_tree + def test_trivial_annotate_state(self): + event = self.create_event(type="state", state_key="", name="event") + event.prev_events = [] + + old_state = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), + ] - self.replication.get_pdu.side_effect = set_return_tree + group_name = "group_name_1" - self.persistence.get_pdu.return_value = None + self.store.get_state_groups.return_value = { + group_name: old_state, + } - is_new = yield self.state.handle_new_state(new_pdu) + yield self.state.annotate_state_groups(event) - self.assertTrue(is_new) + for k, v in event.old_state_events.items(): + type, state_key = k + self.assertEqual(type, v.type) + self.assertEqual(state_key, v.state_key) - self.replication.get_pdu.assert_called_with( - destination=new_pdu.origin, - pdu_origin=old_pdu_1.origin, - pdu_id=old_pdu_1.pdu_id, - outlier=True + self.assertEqual( + set([e.event_id for e in old_state]), + set([e.event_id for e in event.old_state_events.values()]) ) - self.persistence.get_unresolved_state_tree.assert_called_with( - new_pdu + self.assertEqual( + set([e.event_id for e in old_state] + [event.event_id]), + set([e.event_id for e in event.state_events.values()]) ) - self.assertEquals( - 2, self.persistence.get_unresolved_state_tree.call_count + new_state = { + k: v.event_id + for k, v in event.state_events.items() + } + old_state = { + k: v.event_id + for k, v in event.old_state_events.items() + } + old_state[(event.type, event.state_key)] = event.event_id + self.assertDictEqual( + old_state, + new_state ) - self.assertEqual(1, self.persistence.update_current_state.call_count) + self.assertIsNone(event.state_group) @defer.inlineCallbacks - def test_missing_pdu_depth_1(self): - # We try to update state against a PDU we haven't yet seen, - # triggering a get_pdu request - - # The pdu we haven't seen - old_pdu_1 = new_fake_pdu( - "A", "test", "mem", "x", None, "u1", depth=0 - ) - - old_pdu_2 = new_fake_pdu( - "B", "test", "mem", "x", "A", "u2", depth=2 - ) - old_pdu_3 = new_fake_pdu( - "C", "test", "mem", "x", "B", "u3", depth=3 - ) - new_pdu = new_fake_pdu( - "D", "test", "mem", "x", "A", "u4", depth=4 - ) - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 10, - "u4": 20, - }) - - # The return_value of `get_unresolved_state_tree`, which changes after - # the call to get_pdu - tree_to_return = [ - ( - ReturnType([new_pdu], [old_pdu_3]), - 0 - ), - ( - ReturnType( - [new_pdu, old_pdu_1], [old_pdu_3] - ), - 1 - ), - ( - ReturnType( - [new_pdu, old_pdu_1], [old_pdu_3, old_pdu_2, old_pdu_1] - ), - None - ), + def test_resolve_message_conflict(self): + event = self.create_event(type="test_message", name="event") + event.prev_events = [] + + old_state_1 = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), ] - to_return = [0] - - def return_tree(p): - return tree_to_return[to_return[0]] - - def set_return_tree(destination, pdu_origin, pdu_id, outlier=False): - to_return[0] += 1 - return defer.succeed(None) - - self.persistence.get_unresolved_state_tree.side_effect = return_tree - - self.replication.get_pdu.side_effect = set_return_tree - - self.persistence.get_pdu.return_value = None - - is_new = yield self.state.handle_new_state(new_pdu) + old_state_2 = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test3", state_key="2"), + self.create_event(type="test4", state_key=""), + ] - self.assertTrue(is_new) + group_name_1 = "group_name_1" + group_name_2 = "group_name_2" - self.assertEqual(2, self.replication.get_pdu.call_count) + self.store.get_state_groups.return_value = { + group_name_1: old_state_1, + group_name_2: old_state_2, + } - self.replication.get_pdu.assert_has_calls( - [ - mock.call( - destination=new_pdu.origin, - pdu_origin=old_pdu_1.origin, - pdu_id=old_pdu_1.pdu_id, - outlier=True - ), - mock.call( - destination=old_pdu_3.origin, - pdu_origin=old_pdu_2.origin, - pdu_id=old_pdu_2.pdu_id, - outlier=True - ), - ] - ) + yield self.state.annotate_state_groups(event) - self.persistence.get_unresolved_state_tree.assert_called_with( - new_pdu - ) + self.assertEqual(len(event.old_state_events), 5) - self.assertEquals( - 3, self.persistence.get_unresolved_state_tree.call_count + self.assertEqual( + set([e.event_id for e in event.state_events.values()]), + set([e.event_id for e in event.old_state_events.values()]) ) - self.assertEqual(1, self.persistence.update_current_state.call_count) + self.assertIsNone(event.state_group) @defer.inlineCallbacks - def test_missing_pdu_depth_2(self): - # We try to update state against a PDU we haven't yet seen, - # triggering a get_pdu request - - # The pdu we haven't seen - old_pdu_1 = new_fake_pdu( - "A", "test", "mem", "x", None, "u1", depth=0 - ) - - old_pdu_2 = new_fake_pdu( - "B", "test", "mem", "x", "A", "u2", depth=2 - ) - old_pdu_3 = new_fake_pdu( - "C", "test", "mem", "x", "B", "u3", depth=3 - ) - new_pdu = new_fake_pdu( - "D", "test", "mem", "x", "A", "u4", depth=1 - ) - - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 10, - "u2": 10, - "u3": 10, - "u4": 20, - }) - - # The return_value of `get_unresolved_state_tree`, which changes after - # the call to get_pdu - tree_to_return = [ - ( - ReturnType([new_pdu], [old_pdu_3]), - 1, - ), - ( - ReturnType( - [new_pdu], [old_pdu_3, old_pdu_2] - ), - 0, - ), - ( - ReturnType( - [new_pdu, old_pdu_1], [old_pdu_3, old_pdu_2, old_pdu_1] - ), - None - ), + def test_resolve_state_conflict(self): + event = self.create_event(type="test4", state_key="", name="event") + event.prev_events = [] + + old_state_1 = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), ] - to_return = [0] - - def return_tree(p): - return tree_to_return[to_return[0]] - - def set_return_tree(destination, pdu_origin, pdu_id, outlier=False): - to_return[0] += 1 - return defer.succeed(None) - - self.persistence.get_unresolved_state_tree.side_effect = return_tree - - self.replication.get_pdu.side_effect = set_return_tree - - self.persistence.get_pdu.return_value = None - - is_new = yield self.state.handle_new_state(new_pdu) - - self.assertTrue(is_new) - - self.assertEqual(2, self.replication.get_pdu.call_count) - - self.replication.get_pdu.assert_has_calls( - [ - mock.call( - destination=old_pdu_3.origin, - pdu_origin=old_pdu_2.origin, - pdu_id=old_pdu_2.pdu_id, - outlier=True - ), - mock.call( - destination=new_pdu.origin, - pdu_origin=old_pdu_1.origin, - pdu_id=old_pdu_1.pdu_id, - outlier=True - ), - ] - ) - - self.persistence.get_unresolved_state_tree.assert_called_with( - new_pdu - ) - - self.assertEquals( - 3, self.persistence.get_unresolved_state_tree.call_count - ) - - self.assertEqual(1, self.persistence.update_current_state.call_count) - - @defer.inlineCallbacks - def test_no_common_ancestor(self): - # We do a direct overwriting of the old state, i.e., the new state - # points to the old state. + old_state_2 = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test3", state_key="2"), + self.create_event(type="test4", state_key=""), + ] - old_pdu = new_fake_pdu("A", "test", "mem", "x", None, "u1") - new_pdu = new_fake_pdu("B", "test", "mem", "x", None, "u2") + group_name_1 = "group_name_1" + group_name_2 = "group_name_2" - self.persistence.get_power_level.side_effect = _gen_get_power_level({ - "u1": 5, - "u2": 10, - }) + self.store.get_state_groups.return_value = { + group_name_1: old_state_1, + group_name_2: old_state_2, + } - self.persistence.get_unresolved_state_tree.return_value = ( - (ReturnType([new_pdu], [old_pdu]), None) - ) + yield self.state.annotate_state_groups(event) - is_new = yield self.state.handle_new_state(new_pdu) + self.assertEqual(len(event.old_state_events), 5) - self.assertTrue(is_new) + expected_new = event.old_state_events + expected_new[(event.type, event.state_key)] = event - self.persistence.get_unresolved_state_tree.assert_called_once_with( - new_pdu + self.assertEqual( + set([e.event_id for e in expected_new.values()]), + set([e.event_id for e in event.state_events.values()]), ) - self.assertEqual(1, self.persistence.update_current_state.call_count) - - self.assertFalse(self.replication.get_pdu.called) - - @defer.inlineCallbacks - def test_new_event(self): - event = Mock() - event.event_id = "12123123@test" + self.assertIsNone(event.state_group) - state_pdu = new_fake_pdu("C", "test", "mem", "x", "A", 20) + def create_event(self, name=None, type=None, state_key=None): + self.event_id += 1 + event_id = str(self.event_id) - snapshot = Mock() - snapshot.prev_state_pdu = state_pdu - event_id = "pdu_id@origin.com" + if not name: + if state_key is not None: + name = "<%s-%s>" % (type, state_key) + else: + name = "<%s>" % (type, ) - def fill_out_prev_events(event): - event.prev_events = [event_id] - event.depth = 6 - snapshot.fill_out_prev_events = fill_out_prev_events + event = Mock(name=name, spec=[]) + event.type = type - yield self.state.handle_new_event(event, snapshot) - - self.assertLess(5, event.depth) - - self.assertEquals(1, len(event.prev_events)) - - prev_id = event.prev_events[0] - - self.assertEqual(event_id, prev_id) - - self.assertEqual( - encode_event_id(state_pdu.pdu_id, state_pdu.origin), - event.prev_state - ) + if state_key is not None: + event.state_key = state_key + event.event_id = event_id + event.user_id = "@user_id:example.com" + event.room_id = "!room_id:example.com" -def new_fake_pdu(pdu_id, context, pdu_type, state_key, prev_state_id, - user_id, depth=0): - new_pdu = Pdu( - pdu_id=pdu_id, - pdu_type=pdu_type, - state_key=state_key, - user_id=user_id, - prev_state_id=prev_state_id, - origin="example.com", - context="context", - origin_server_ts=1405353060021, - depth=depth, - content_json="{}", - unrecognized_keys="{}", - outlier=True, - is_state=True, - prev_state_origin="example.com", - have_processed=True, - content={}, - ) - - return new_pdu + return event -- cgit 1.5.1 From 3db2c0d43e6859e522859be271e5d361053f22b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 11 Nov 2014 16:58:53 +0000 Subject: Rename annotate_state_groups to annotate_event_with_state --- synapse/handlers/_base.py | 2 +- synapse/handlers/federation.py | 14 +++++++------- synapse/state.py | 2 +- tests/handlers/test_federation.py | 6 +++--- tests/handlers/test_room.py | 8 ++++---- tests/test_state.py | 12 ++++++------ 6 files changed, 22 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 07a8464107..30c6733063 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -64,7 +64,7 @@ class BaseHandler(object): snapshot.fill_out_prev_events(event) - yield self.state_handler.annotate_state_groups(event) + yield self.state_handler.annotate_event_with_state(event) yield self.auth.add_auth_events(event) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c2cd91bb39..d8d5730b65 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -111,7 +111,7 @@ class FederationHandler(BaseHandler): if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] - is_new_state = yield self.state_handler.annotate_state_groups( + is_new_state = yield self.state_handler.annotate_event_with_state( event, old_state=state ) @@ -202,7 +202,7 @@ class FederationHandler(BaseHandler): event = self.pdu_codec.event_from_pdu(pdu) # FIXME (erikj): Not sure this actually works :/ - yield self.state_handler.annotate_state_groups(event) + yield self.state_handler.annotate_event_with_state(event) events.append(event) @@ -268,7 +268,7 @@ class FederationHandler(BaseHandler): logger.debug("do_invite_join state: %s", state) - is_new_state = yield self.state_handler.annotate_state_groups( + is_new_state = yield self.state_handler.annotate_event_with_state( event, old_state=state ) @@ -289,7 +289,7 @@ class FederationHandler(BaseHandler): # FIXME: Auth these. e.outlier = True - yield self.state_handler.annotate_state_groups( + yield self.state_handler.annotate_event_with_state( e, ) @@ -330,7 +330,7 @@ class FederationHandler(BaseHandler): snapshot = yield self.store.snapshot_room(event) snapshot.fill_out_prev_events(event) - yield self.state_handler.annotate_state_groups(event) + yield self.state_handler.annotate_event_with_state(event) yield self.auth.add_auth_events(event) self.auth.check(event, raises=True) @@ -345,7 +345,7 @@ class FederationHandler(BaseHandler): event.outlier = False - is_new_state = yield self.state_handler.annotate_state_groups(event) + is_new_state = yield self.state_handler.annotate_event_with_state(event) self.auth.check(event, raises=True) # FIXME (erikj): All this is duplicated above :( @@ -421,7 +421,7 @@ class FederationHandler(BaseHandler): ) ) - yield self.state_handler.annotate_state_groups(event) + yield self.state_handler.annotate_event_with_state(event) yield self.store.persist_event( event, diff --git a/synapse/state.py b/synapse/state.py index 11c54fd38c..9c22cf7701 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -45,7 +45,7 @@ class StateHandler(object): @defer.inlineCallbacks @log_function - def annotate_state_groups(self, event, old_state=None): + def annotate_event_with_state(self, event, old_state=None): yield run_on_reactor() if old_state: diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index a9d6b2bb17..e386cddb38 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -36,7 +36,7 @@ class FederationTestCase(unittest.TestCase): self.mock_config.signing_key = [MockKey()] self.state_handler = NonCallableMock(spec_set=[ - "annotate_state_groups", + "annotate_event_with_state", ]) self.auth = NonCallableMock(spec_set=[ @@ -85,7 +85,7 @@ class FederationTestCase(unittest.TestCase): self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) - self.state_handler.annotate_state_groups.return_value = ( + self.state_handler.annotate_event_with_state.return_value = ( defer.succeed(False) ) @@ -95,7 +95,7 @@ class FederationTestCase(unittest.TestCase): ANY, False, is_new_state=False ) - self.state_handler.annotate_state_groups.assert_called_once_with( + self.state_handler.annotate_event_with_state.assert_called_once_with( ANY, old_state=None, ) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 55c9f6e142..ee264e5ee2 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -60,7 +60,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ]), auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "annotate_state_groups", + "annotate_event_with_state", ]), config=self.mock_config, ) @@ -251,7 +251,7 @@ class RoomCreationTest(unittest.TestCase): ]), auth=NonCallableMock(spec_set=["check", "add_auth_events"]), state_handler=NonCallableMock(spec_set=[ - "annotate_state_groups", + "annotate_event_with_state", ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", @@ -282,7 +282,7 @@ class RoomCreationTest(unittest.TestCase): def annotate(event): event.state_events = {} return defer.succeed(None) - self.state_handler.annotate_state_groups.side_effect = annotate + self.state_handler.annotate_event_with_state.side_effect = annotate def hosts(room): return defer.succeed([]) @@ -311,6 +311,6 @@ class RoomCreationTest(unittest.TestCase): self.assertEquals(user_id, join_event.user_id) self.assertEquals(user_id, join_event.state_key) - self.assertTrue(self.state_handler.annotate_state_groups.called) + self.assertTrue(self.state_handler.annotate_event_with_state.called) self.assertTrue(self.federation.handle_new_event.called) diff --git a/tests/test_state.py b/tests/test_state.py index 3cc358be32..7979b54a35 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -44,7 +44,7 @@ class StateTestCase(unittest.TestCase): self.create_event(type="test2", state_key=""), ] - yield self.state.annotate_state_groups(event, old_state=old_state) + yield self.state.annotate_event_with_state(event, old_state=old_state) for k, v in event.old_state_events.items(): type, state_key = k @@ -66,7 +66,7 @@ class StateTestCase(unittest.TestCase): self.create_event(type="test2", state_key=""), ] - yield self.state.annotate_state_groups(event, old_state=old_state) + yield self.state.annotate_event_with_state(event, old_state=old_state) for k, v in event.old_state_events.items(): type, state_key = k @@ -99,7 +99,7 @@ class StateTestCase(unittest.TestCase): group_name: old_state, } - yield self.state.annotate_state_groups(event) + yield self.state.annotate_event_with_state(event) for k, v in event.old_state_events.items(): type, state_key = k @@ -141,7 +141,7 @@ class StateTestCase(unittest.TestCase): group_name: old_state, } - yield self.state.annotate_state_groups(event) + yield self.state.annotate_event_with_state(event) for k, v in event.old_state_events.items(): type, state_key = k @@ -199,7 +199,7 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - yield self.state.annotate_state_groups(event) + yield self.state.annotate_event_with_state(event) self.assertEqual(len(event.old_state_events), 5) @@ -235,7 +235,7 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - yield self.state.annotate_state_groups(event) + yield self.state.annotate_event_with_state(event) self.assertEqual(len(event.old_state_events), 5) -- cgit 1.5.1 From 37900a92dbfd269c17c88e1b6cd7eb7881ed7b13 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 11 Nov 2014 17:55:32 +0000 Subject: Only allow people in a room to look up room state. --- synapse/handlers/message.py | 38 ++++---------------------------------- tests/rest/test_rooms.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 42 deletions(-) (limited to 'tests') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 8394013df3..4da5c046bf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -147,49 +147,19 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_room_data(self, user_id=None, room_id=None, - event_type=None, state_key="", - public_room_rules=[], - private_room_rules=["join"]): + event_type=None, state_key=""): """ Get data from a room. Args: event : The room path event - public_room_rules : A list of membership states the user can be in, - in order to read this data IN A PUBLIC ROOM. An empty list means - 'any state'. - private_room_rules : A list of membership states the user can be - in, in order to read this data IN A PRIVATE ROOM. An empty list - means 'any state'. Returns: The path data content. Raises: SynapseError if something went wrong. """ - if event_type == RoomTopicEvent.TYPE: - # anyone invited/joined can read the topic - private_room_rules = ["invite", "join"] - - # does this room exist - room = yield self.store.get_room(room_id) - if not room: - raise RoomError(403, "Room does not exist.") - - # does this user exist in this room - member = yield self.store.get_room_member( - room_id=room_id, - user_id="" if not user_id else user_id) - - member_state = member.membership if member else None - - if room.is_public and public_room_rules: - # make sure the user meets public room rules - if member_state not in public_room_rules: - raise RoomError(403, "Member does not meet public room rules.") - elif not room.is_public and private_room_rules: - # make sure the user meets private room rules - if member_state not in private_room_rules: - raise RoomError( - 403, "Member does not meet private room rules.") + have_joined = yield self.auth.check_joined_room(room_id, user_id) + if not have_joined: + raise RoomError(403, "User not in room.") data = yield self.state_handler.get_current_state( room_id, event_type, state_key diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 61b01d369d..e27990dace 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -230,9 +230,9 @@ class RoomPermissionsTestCase(RestTestCase): "PUT", topic_path, topic_content) self.assertEquals(403, code, msg=str(response)) - # get topic in created PRIVATE room and invited, expect 200 (or 404) + # get topic in created PRIVATE room and invited, expect 403 (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(404, code, msg=str(response)) + self.assertEquals(403, code, msg=str(response)) # set/get topic in created PRIVATE room and joined, expect 200 yield self.join(room=self.created_rmid, user=self.user_id) @@ -256,10 +256,10 @@ class RoomPermissionsTestCase(RestTestCase): (code, response) = yield self.mock_resource.trigger_get(topic_path) self.assertEquals(403, code, msg=str(response)) - # get topic in PUBLIC room, not joined, expect 200 (or 404) + # get topic in PUBLIC room, not joined, expect 403 (code, response) = yield self.mock_resource.trigger_get( "/rooms/%s/state/m.room.topic" % self.created_public_rmid) - self.assertEquals(200, code, msg=str(response)) + self.assertEquals(403, code, msg=str(response)) # set topic in PUBLIC room, not joined, expect 403 (code, response) = yield self.mock_resource.trigger( @@ -326,12 +326,12 @@ class RoomPermissionsTestCase(RestTestCase): def test_membership_public_room_perms(self): room = self.created_public_rmid # get membership of self, get membership of other, public room + invite - # expect all 200s - public rooms, you can see who is in them. + # expect 403 yield self.invite(room=room, src=self.rmcreator_id, targ=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=200) + room=room, expect_code=403) # get membership of self, get membership of other, public room + joined # expect all 200s @@ -341,11 +341,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, public room + left - # expect all 200s - public rooms, you can always see who is in them. + # expect 403. yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=200) + room=room, expect_code=403) @defer.inlineCallbacks def test_invited_permissions(self): -- cgit 1.5.1 From 6fea478d2e7737c2462b074b935d4427ced5f3d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 12 Nov 2014 11:22:51 +0000 Subject: Fix bugs with invites/joins across federatiom. Both in terms of auth and not trying to fetch missing PDUs for invites, joins etc. --- synapse/api/auth.py | 19 ++++++++++++++++--- synapse/federation/replication.py | 7 +------ synapse/handlers/federation.py | 12 +++--------- synapse/handlers/room.py | 10 ++++++++-- synapse/storage/__init__.py | 14 +++++++++++--- synapse/storage/state.py | 9 ++++++--- tests/handlers/test_room.py | 22 +++++++++------------- 7 files changed, 54 insertions(+), 39 deletions(-) (limited to 'tests') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 6c2d3db26e..87f19a96d6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -36,6 +36,7 @@ class Auth(object): def __init__(self, hs): self.hs = hs self.store = hs.get_datastore() + self.state = hs.get_state_handler() def check(self, event, raises=False): """ Checks if this event is correctly authed. @@ -90,7 +91,7 @@ class Auth(object): ) logger.info("Denying! %s", event) if raises: - raise e + raise return False @@ -109,9 +110,21 @@ class Auth(object): @defer.inlineCallbacks def check_host_in_room(self, room_id, host): - joined_hosts = yield self.store.get_joined_hosts_for_room(room_id) + curr_state = yield self.state.get_current_state(room_id) + + for event in curr_state: + if event.type == RoomMemberEvent.TYPE: + try: + if self.hs.parse_userid(event.state_key).domain != host: + continue + except: + logger.warn("state_key not user_id: %s", event.state_key) + continue + + if event.content["membership"] == Membership.JOIN: + defer.returnValue(True) - defer.returnValue(host in joined_hosts) + defer.returnValue(False) def check_event_sender_in_room(self, event): key = (RoomMemberEvent.TYPE, event.user_id, ) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 5c625ddabf..beec17e386 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -267,8 +267,6 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) pdus = [Pdu(outlier=True, **p) for p in transaction.pdus] - for pdu in pdus: - yield self._handle_new_pdu(destination, pdu) defer.returnValue(pdus) @@ -452,15 +450,12 @@ class ReplicationLayer(object): ) logger.debug("Got content: %s", content) + state = [Pdu(outlier=True, **p) for p in content.get("state", [])] - for pdu in state: - yield self._handle_new_pdu(destination, pdu) auth_chain = [ Pdu(outlier=True, **p) for p in content.get("auth_chain", []) ] - for pdu in auth_chain: - yield self._handle_new_pdu(destination, pdu) defer.returnValue(state) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d8d5730b65..99655c8bb0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -229,12 +229,6 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks def do_invite_join(self, target_host, room_id, joinee, content, snapshot): - hosts = yield self.store.get_joined_hosts_for_room(room_id) - if self.hs.hostname in hosts: - # We are already in the room. - logger.debug("We're already in the room apparently") - defer.returnValue(False) - pdu = yield self.replication_layer.make_join( target_host, room_id, @@ -268,7 +262,7 @@ class FederationHandler(BaseHandler): logger.debug("do_invite_join state: %s", state) - is_new_state = yield self.state_handler.annotate_event_with_state( + yield self.state_handler.annotate_event_with_state( event, old_state=state ) @@ -296,13 +290,13 @@ class FederationHandler(BaseHandler): yield self.store.persist_event( e, backfilled=False, - is_new_state=False + is_new_state=True ) yield self.store.persist_event( event, backfilled=False, - is_new_state=is_new_state + is_new_state=True ) finally: room_queue = self.room_queues[room_id] diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3642fcfc6d..825957f721 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomTopicEvent, RoomNameEvent, RoomJoinRulesEvent, ) from synapse.util import stringutils +from synapse.util.async import run_on_reactor from ._base import BaseHandler import logging @@ -432,9 +433,12 @@ class RoomMemberHandler(BaseHandler): # that we are allowed to join when we decide whether or not we # need to do the invite/join dance. - hosts = yield self.store.get_joined_hosts_for_room(room_id) + is_host_in_room = yield self.auth.check_host_in_room( + event.room_id, + self.hs.hostname + ) - if self.hs.hostname in hosts: + if is_host_in_room: should_do_dance = False elif room_host: should_do_dance = True @@ -517,6 +521,8 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _do_local_membership_update(self, event, membership, snapshot, do_auth): + yield run_on_reactor() + # If we're inviting someone, then we should also send it to that # HS. target_user_id = event.state_key diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4034437f6b..72290eb5a0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -186,6 +186,7 @@ class DataStore(RoomMemberStore, RoomStore, "events", vals, or_replace=(not outlier), + or_ignore=bool(outlier), ) except: logger.warn( @@ -217,7 +218,12 @@ class DataStore(RoomMemberStore, RoomStore, if hasattr(event, "replaces_state"): vals["prev_state"] = event.replaces_state - self._simple_insert_txn(txn, "state_events", vals) + self._simple_insert_txn( + txn, + "state_events", + vals, + or_replace=True, + ) self._simple_insert_txn( txn, @@ -227,7 +233,8 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "type": event.type, "state_key": event.state_key, - } + }, + or_replace=True, ) for e_id, h in event.prev_state: @@ -252,7 +259,8 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "type": event.type, "state_key": event.state_key, - } + }, + or_replace=True, ) for prev_state_id, _ in event.prev_state: diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 68975969f5..2f3a70b4e5 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -70,7 +70,8 @@ class StateStore(SQLBaseStore): values={ "room_id": event.room_id, "event_id": event.event_id, - } + }, + or_ignore=True, ) for state in event.state_events.values(): @@ -83,7 +84,8 @@ class StateStore(SQLBaseStore): "type": state.type, "state_key": state.state_key, "event_id": state.event_id, - } + }, + or_ignore=True, ) self._simple_insert_txn( @@ -92,5 +94,6 @@ class StateStore(SQLBaseStore): values={ "state_group": state_group, "event_id": event.event_id, - } + }, + or_replace=True, ) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index ee264e5ee2..cbe591ab90 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -44,7 +44,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ]), datastore=NonCallableMock(spec_set=[ "persist_event", - "get_joined_hosts_for_room", "get_room_member", "get_room", "store_room", @@ -58,9 +57,14 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "profile_handler", "federation_handler", ]), - auth=NonCallableMock(spec_set=["check", "add_auth_events"]), + auth=NonCallableMock(spec_set=[ + "check", + "add_auth_events", + "check_host_in_room", + ]), state_handler=NonCallableMock(spec_set=[ "annotate_event_with_state", + "get_current_state", ]), config=self.mock_config, ) @@ -76,6 +80,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.notifier = hs.get_notifier() self.state_handler = hs.get_state_handler() self.distributor = hs.get_distributor() + self.auth = hs.get_auth() self.hs = hs self.handlers.federation_handler = self.federation @@ -108,11 +113,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): content=content, ) - joined = ["red", "green"] - - self.datastore.get_joined_hosts_for_room.return_value = ( - defer.succeed(joined) - ) + self.auth.check_host_in_room.return_value = defer.succeed(True) store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) @@ -164,12 +165,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): room_id=room_id, ) - joined = ["red", "green"] - - def get_joined(*args): - return defer.succeed(joined) - - self.datastore.get_joined_hosts_for_room.side_effect = get_joined + self.auth.check_host_in_room.return_value = defer.succeed(True) store_id = "store_id_fooo" self.datastore.persist_event.return_value = defer.succeed(store_id) -- cgit 1.5.1 From 8d8a133c89925086452bdec9739db589f0715363 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 13 Nov 2014 15:48:51 +0000 Subject: SYN-103: Remove "origin" and "destination" keys from edus --- synapse/federation/units.py | 9 ++++----- tests/federation/test_federation.py | 3 --- tests/handlers/test_presence.py | 3 --- tests/handlers/test_typing.py | 3 --- 4 files changed, 4 insertions(+), 14 deletions(-) (limited to 'tests') diff --git a/synapse/federation/units.py b/synapse/federation/units.py index f4e7b62bd9..70412439cd 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -122,11 +122,10 @@ class Edu(JsonEncodedObject): "edu_type", ] -# TODO: SYN-103: Remove "origin" and "destination" keys. -# internal_keys = [ -# "origin", -# "destination", -# ] + internal_keys = [ + "origin", + "destination", + ] class Transaction(JsonEncodedObject): diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index eb329eec50..ad09fab392 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -218,9 +218,6 @@ class FederationTestCase(unittest.TestCase): "pdus": [], "edus": [ { - # TODO: SYN-103: Remove "origin" and "destination" - "origin": "test", - "destination": "remote", "edu_type": "m.test", "content": {"testing": "content here"}, } diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index a6af648def..cdaf93429b 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -44,9 +44,6 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { - # TODO: SYN-103: Remove "origin" and "destination" keys. - "origin": origin, - "destination": destination, "edu_type": edu_type, "content": content, } diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 07acda5eee..adb5148351 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -33,9 +33,6 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { - # TODO: SYN-103: Remove "origin" and "destination" keys. - "origin": origin, - "destination": destination, "edu_type": edu_type, "content": content, } -- cgit 1.5.1 From de1ec90133031cf6a043c47adae515ae1690c6d8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 14 Nov 2014 16:45:39 +0000 Subject: Validate signatures on incoming events --- synapse/crypto/event_signing.py | 18 ++++++++++++++---- synapse/handlers/federation.py | 37 +++++++++++++++++++++++++++++++++++-- tests/handlers/test_federation.py | 4 +++- 3 files changed, 52 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index baa93b0ee4..c7e6bec8f5 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -19,6 +19,7 @@ from synapse.api.events.utils import prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json +from synapse.api.errors import SynapseError, Codes import hashlib import logging @@ -29,15 +30,24 @@ logger = logging.getLogger(__name__) def check_event_content_hash(event, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" computed_hash = _compute_content_hash(event, hash_algorithm) + logging.debug("Expecting hash: %s", encode_base64(computed_hash.digest())) if computed_hash.name not in event.hashes: - raise Exception("Algorithm %s not in hashes %s" % ( - computed_hash.name, list(event.hashes) - )) + raise SynapseError( + 400, + "Algorithm %s not in hashes %s" % ( + computed_hash.name, list(event.hashes), + ), + Codes.UNAUTHORIZED, + ) message_hash_base64 = event.hashes[computed_hash.name] try: message_hash_bytes = decode_base64(message_hash_base64) except: - raise Exception("Invalid base64: %s" % (message_hash_base64,)) + raise SynapseError( + 400, + "Invalid base64: %s" % (message_hash_base64,), + Codes.UNAUTHORIZED, + ) return message_hash_bytes == computed_hash.digest() diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5e096f4652..fce935b444 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,13 +17,17 @@ from ._base import BaseHandler -from synapse.api.errors import AuthError, FederationError +from synapse.api.events.utils import prune_event +from synapse.api.errors import AuthError, FederationError, SynapseError from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function from synapse.federation.pdu_codec import PduCodec from synapse.util.async import run_on_reactor -from synapse.crypto.event_signing import compute_event_signature +from synapse.crypto.event_signing import ( + compute_event_signature, check_event_content_hash +) +from syutil.jsonutil import encode_canonical_json from twisted.internet import defer @@ -59,6 +63,7 @@ class FederationHandler(BaseHandler): self.state_handler = hs.get_state_handler() # self.auth_handler = gs.get_auth_handler() self.server_name = hs.hostname + self.keyring = hs.get_keyring() self.lock_manager = hs.get_room_lock_manager() @@ -112,6 +117,34 @@ class FederationHandler(BaseHandler): logger.debug("Processing event: %s", event.event_id) + redacted_event = prune_event(event) + redacted_event.origin = pdu.origin + redacted_event.origin_server_ts = pdu.origin_server_ts + redacted_pdu = self.pdu_codec.pdu_from_event(redacted_event) + + redacted_pdu_json = redacted_pdu.get_dict() + try: + yield self.keyring.verify_json_for_server( + event.origin, redacted_pdu_json + ) + except SynapseError as e: + logger.warn("Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_dict()), + encode_canonical_json(redacted_pdu_json), + ) + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s", event.event_id + ) + event = redacted_event + if state: state = [self.pdu_codec.event_from_pdu(p) for p in state] diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index e386cddb38..3f17ca8fb0 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -23,7 +23,7 @@ from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer from synapse.federation.units import Pdu -from mock import NonCallableMock, ANY +from mock import NonCallableMock, ANY, Mock from ..utils import MockKey @@ -62,6 +62,7 @@ class FederationTestCase(unittest.TestCase): config=self.mock_config, auth=self.auth, state_handler=self.state_handler, + keyring=Mock(), ) self.datastore = hs.get_datastore() @@ -80,6 +81,7 @@ class FederationTestCase(unittest.TestCase): origin_server_ts=0, event_id="$a:b", origin="b", + hashes={"sha256":"PvbCLWrTBxnBsSO7/cJ76072ySTCgI/XGadESRAe02M"}, ) self.datastore.persist_event.return_value = defer.succeed(None) -- cgit 1.5.1 From cb4b6c844a0c9e2d4a96165958ff5680ed82e160 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 14 Nov 2014 21:25:02 +0000 Subject: Merge PDUs and Events into one object --- synapse/api/events/__init__.py | 6 +++ synapse/api/events/utils.py | 2 + synapse/crypto/event_signing.py | 15 ++----- synapse/federation/pdu_codec.py | 54 ------------------------- synapse/federation/replication.py | 61 +++++++++++++++++++--------- synapse/federation/units.py | 79 +------------------------------------ synapse/handlers/federation.py | 65 +++++++++++------------------- synapse/storage/_base.py | 8 ++++ tests/federation/test_federation.py | 8 ++-- tests/handlers/test_federation.py | 5 ++- 10 files changed, 91 insertions(+), 212 deletions(-) delete mode 100644 synapse/federation/pdu_codec.py (limited to 'tests') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 1d8bed2906..63c0bd7ae7 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -117,6 +117,12 @@ class SynapseEvent(JsonEncodedObject): """ raise NotImplementedError("get_content_template not implemented.") + def get_pdu_json(self): + pdu_json = self.get_full_dict() + pdu_json.pop("destination", None) + pdu_json.pop("outlier", None) + return pdu_json + class SynapseStateEvent(SynapseEvent): diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py index 802648f8f7..d6019d56eb 100644 --- a/synapse/api/events/utils.py +++ b/synapse/api/events/utils.py @@ -42,6 +42,8 @@ def prune_event(event): "prev_events", "prev_state", "auth_events", + "origin", + "origin_server_ts", ] new_content = {} diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 79274fd552..4dff2c0ec2 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -16,7 +16,6 @@ from synapse.api.events.utils import prune_event -from synapse.federation.units import Pdu from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json @@ -53,8 +52,7 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256): def _compute_content_hash(event, hash_algorithm): - event_json = event.get_full_dict() - # TODO: We need to sign the JSON that is going out via fedaration. + event_json = event.get_pdu_json() event_json.pop("age_ts", None) event_json.pop("unsigned", None) event_json.pop("signatures", None) @@ -67,7 +65,7 @@ def _compute_content_hash(event, hash_algorithm): def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): tmp_event = prune_event(event) - event_json = tmp_event.get_dict() + event_json = tmp_event.get_pdu_json() event_json.pop("signatures", None) event_json.pop("age_ts", None) event_json.pop("unsigned", None) @@ -78,14 +76,7 @@ def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): def compute_event_signature(event, signature_name, signing_key): tmp_event = prune_event(event) - tmp_event.origin = event.origin - tmp_event.origin_server_ts = event.origin_server_ts - d = tmp_event.get_full_dict() - kwargs = dict(event.unrecognized_keys) - kwargs.update({k: v for k, v in d.items()}) - tmp_pdu = Pdu(**kwargs) - redact_json = tmp_pdu.get_dict() - redact_json.pop("signatures", None) + redact_json = tmp_event.get_pdu_json() redact_json.pop("age_ts", None) redact_json.pop("unsigned", None) logger.debug("Signing event: %s", redact_json) diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py deleted file mode 100644 index 52c84efb5b..0000000000 --- a/synapse/federation/pdu_codec.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .units import Pdu - -import copy - - -class PduCodec(object): - - def __init__(self, hs): - self.signing_key = hs.config.signing_key[0] - self.server_name = hs.hostname - self.event_factory = hs.get_event_factory() - self.clock = hs.get_clock() - self.hs = hs - - def event_from_pdu(self, pdu): - kwargs = {} - - kwargs["etype"] = pdu.type - - kwargs.update({ - k: v - for k, v in pdu.get_full_dict().items() - if k not in [ - "type", - ] - }) - - return self.event_factory.create_event(**kwargs) - - def pdu_from_event(self, event): - d = event.get_full_dict() - - kwargs = copy.deepcopy(event.unrecognized_keys) - kwargs.update({ - k: v for k, v in d.items() - }) - - pdu = Pdu(**kwargs) - return pdu diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index a07e307849..8ee74de005 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -19,7 +19,7 @@ a given transport. from twisted.internet import defer -from .units import Transaction, Pdu, Edu +from .units import Transaction, Edu from .persistence import TransactionActions @@ -72,6 +72,8 @@ class ReplicationLayer(object): self._clock = hs.get_clock() + self.event_factory = hs.get_event_factory() + def set_handler(self, handler): """Sets the handler that the replication layer will use to communicate receipt of new PDUs from other home servers. The required methods are @@ -203,7 +205,10 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) - pdus = [Pdu(outlier=False, **p) for p in transaction.pdus] + pdus = [ + self.event_from_pdu_json(p, outlier=False) + for p in transaction.pdus + ] for pdu in pdus: yield self._handle_new_pdu(dest, pdu, backfilled=True) @@ -235,7 +240,10 @@ class ReplicationLayer(object): transaction = Transaction(**transaction_data) - pdu_list = [Pdu(outlier=outlier, **p) for p in transaction.pdus] + pdu_list = [ + self.event_from_pdu_json(p, outlier=outlier) + for p in transaction.pdus + ] pdu = None if pdu_list: @@ -265,8 +273,10 @@ class ReplicationLayer(object): ) transaction = Transaction(**transaction_data) - - pdus = [Pdu(outlier=True, **p) for p in transaction.pdus] + pdus = [ + self.event_from_pdu_json(p, outlier=True) + for p in transaction.pdus + ] defer.returnValue(pdus) @@ -293,7 +303,9 @@ class ReplicationLayer(object): p["age_ts"] = int(self._clock.time_msec()) - int(p["age"]) del p["age"] - pdu_list = [Pdu(**p) for p in transaction.pdus] + pdu_list = [ + self.event_from_pdu_json(p) for p in transaction.pdus + ] logger.debug("[%s] Got transaction", transaction.transaction_id) @@ -388,30 +400,30 @@ class ReplicationLayer(object): def on_make_join_request(self, context, user_id): pdu = yield self.handler.on_make_join_request(context, user_id) defer.returnValue({ - "event": pdu.get_dict(), + "event": pdu.get_pdu_json(), }) @defer.inlineCallbacks def on_invite_request(self, origin, content): - pdu = Pdu(**content) + pdu = self.event_from_pdu_json(content) ret_pdu = yield self.handler.on_invite_request(origin, pdu) defer.returnValue( ( 200, { - "event": ret_pdu.get_dict(), + "event": ret_pdu.get_pdu_json(), } ) ) @defer.inlineCallbacks def on_send_join_request(self, origin, content): - pdu = Pdu(**content) + pdu = self.event_from_pdu_json(content) res_pdus = yield self.handler.on_send_join_request(origin, pdu) defer.returnValue((200, { - "state": [p.get_dict() for p in res_pdus["state"]], - "auth_chain": [p.get_dict() for p in res_pdus["auth_chain"]], + "state": [p.get_pdu_json() for p in res_pdus["state"]], + "auth_chain": [p.get_pdu_json() for p in res_pdus["auth_chain"]], })) @defer.inlineCallbacks @@ -421,7 +433,7 @@ class ReplicationLayer(object): ( 200, { - "auth_chain": [a.get_dict() for a in auth_pdus], + "auth_chain": [a.get_pdu_json() for a in auth_pdus], } ) ) @@ -438,7 +450,7 @@ class ReplicationLayer(object): logger.debug("Got response to make_join: %s", pdu_dict) - defer.returnValue(Pdu(**pdu_dict)) + defer.returnValue(self.event_from_pdu_json(pdu_dict)) @defer.inlineCallbacks def send_join(self, destination, pdu): @@ -446,12 +458,15 @@ class ReplicationLayer(object): destination, pdu.room_id, pdu.event_id, - pdu.get_dict(), + pdu.get_pdu_json(), ) logger.debug("Got content: %s", content) - state = [Pdu(outlier=True, **p) for p in content.get("state", [])] + state = [ + self.event_from_pdu_json(p, outlier=True) + for p in content.get("state", []) + ] # FIXME: We probably want to do something with the auth_chain given # to us @@ -468,14 +483,14 @@ class ReplicationLayer(object): destination=destination, context=context, event_id=event_id, - content=pdu.get_dict(), + content=pdu.get_pdu_json(), ) pdu_dict = content["event"] logger.debug("Got response to send_invite: %s", pdu_dict) - defer.returnValue(Pdu(**pdu_dict)) + defer.returnValue(self.event_from_pdu_json(pdu_dict)) @log_function def _get_persisted_pdu(self, origin, event_id): @@ -490,7 +505,7 @@ class ReplicationLayer(object): """Returns a new Transaction containing the given PDUs suitable for transmission. """ - pdus = [p.get_dict() for p in pdu_list] + pdus = [p.get_pdu_json() for p in pdu_list] time_now = self._clock.time_msec() for p in pdus: if "age_ts" in p: @@ -563,6 +578,14 @@ class ReplicationLayer(object): def __str__(self): return "" % self.server_name + def event_from_pdu_json(self, pdu_json, outlier=False): + #TODO: Check we have all the PDU keys here + pdu_json.setdefault("hashes", {}) + pdu_json.setdefault("signatures", {}) + return self.event_factory.create_event( + pdu_json["type"], outlier=outlier, **pdu_json + ) + class _TransactionQueue(object): """This class makes sure we only have one transaction in flight at diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 70412439cd..6e708edb8c 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -25,83 +25,6 @@ import logging logger = logging.getLogger(__name__) -class Pdu(JsonEncodedObject): - """ A Pdu represents a piece of data sent from a server and is associated - with a context. - - A Pdu can be classified as "state". For a given context, we can efficiently - retrieve all state pdu's that haven't been clobbered. Clobbering is done - via a unique constraint on the tuple (context, type, state_key). A pdu - is a state pdu if `is_state` is True. - - Example pdu:: - - { - "event_id": "$78c:example.com", - "origin_server_ts": 1404835423000, - "origin": "bar", - "prev_ids": [ - ["23b", "foo"], - ["56a", "bar"], - ], - "content": { ... }, - } - - """ - - valid_keys = [ - "event_id", - "room_id", - "origin", - "origin_server_ts", - "type", - "destinations", - "prev_events", - "depth", - "content", - "hashes", - "user_id", - "auth_events", - "signatures", # Below this are keys valid only for State Pdus. - "state_key", - "prev_state", - ] - - internal_keys = [ - "destinations", - "transaction_id", - "outlier", - ] - - required_keys = [ - "event_id", - "room_id", - "origin", - "origin_server_ts", - "type", - "content", - ] - - # TODO: We need to make this properly load content rather than - # just leaving it as a dict. (OR DO WE?!) - - def __init__(self, destinations=[], prev_events=[], - outlier=False, hashes={}, signatures={}, **kwargs): - super(Pdu, self).__init__( - destinations=destinations, - prev_events=prev_events, - outlier=outlier, - hashes=hashes, - signatures=signatures, - **kwargs - ) - - def __str__(self): - return "(%s, %s)" % (self.__class__.__name__, repr(self.__dict__)) - - def __repr__(self): - return "<%s, %s>" % (self.__class__.__name__, repr(self.__dict__)) - class Edu(JsonEncodedObject): """ An Edu represents a piece of data sent from one homeserver to another. @@ -202,6 +125,6 @@ class Transaction(JsonEncodedObject): for p in pdus: p.transaction_id = kwargs["transaction_id"] - kwargs["pdus"] = [p.get_dict() for p in pdus] + kwargs["pdus"] = [p.get_pdu_json() for p in pdus] return Transaction(**kwargs) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fc00128c56..da38f34e6a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -22,7 +22,6 @@ from synapse.api.errors import AuthError, FederationError, SynapseError from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership from synapse.util.logutils import log_function -from synapse.federation.pdu_codec import PduCodec from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import ( compute_event_signature, check_event_content_hash @@ -69,8 +68,6 @@ class FederationHandler(BaseHandler): self.replication_layer.set_handler(self) - self.pdu_codec = PduCodec(hs) - # When joining a room we need to queue any events for that room up self.room_queues = {} @@ -92,7 +89,7 @@ class FederationHandler(BaseHandler): yield run_on_reactor() - pdu = self.pdu_codec.pdu_from_event(event) + pdu = event if not hasattr(pdu, "destinations") or not pdu.destinations: pdu.destinations = [] @@ -105,7 +102,7 @@ class FederationHandler(BaseHandler): """ Called by the ReplicationLayer when we have a new pdu. We need to do auth checks and put it through the StateHandler. """ - event = self.pdu_codec.event_from_pdu(pdu) + event = pdu logger.debug("Got event: %s", event.event_id) @@ -118,18 +115,15 @@ class FederationHandler(BaseHandler): logger.debug("Processing event: %s", event.event_id) redacted_event = prune_event(event) - redacted_event.origin = pdu.origin - redacted_event.origin_server_ts = pdu.origin_server_ts - redacted_pdu = self.pdu_codec.pdu_from_event(redacted_event) - redacted_pdu_json = redacted_pdu.get_dict() + redacted_pdu_json = redacted_event.get_pdu_json() try: yield self.keyring.verify_json_for_server( event.origin, redacted_pdu_json ) except SynapseError as e: logger.warn("Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_dict()), + encode_canonical_json(pdu.get_pdu_json()), encode_canonical_json(redacted_pdu_json), ) raise FederationError( @@ -147,7 +141,7 @@ class FederationHandler(BaseHandler): event = redacted_event if state: - state = [self.pdu_codec.event_from_pdu(p) for p in state] + state = [p for p in state] is_new_state = yield self.state_handler.annotate_event_with_state( event, @@ -239,7 +233,7 @@ class FederationHandler(BaseHandler): events = [] for pdu in pdus: - event = self.pdu_codec.event_from_pdu(pdu) + event = pdu # FIXME (erikj): Not sure this actually works :/ yield self.state_handler.annotate_event_with_state(event) @@ -260,15 +254,15 @@ class FederationHandler(BaseHandler): destination=target_host, context=event.room_id, event_id=event.event_id, - pdu=self.pdu_codec.pdu_from_event(event) + pdu=event ) - defer.returnValue(self.pdu_codec.event_from_pdu(pdu)) + defer.returnValue(pdu) @defer.inlineCallbacks def on_event_auth(self, event_id): auth = yield self.store.get_auth_chain(event_id) - defer.returnValue([self.pdu_codec.pdu_from_event(e) for e in auth]) + defer.returnValue([e for e in auth]) @log_function @defer.inlineCallbacks @@ -292,7 +286,7 @@ class FederationHandler(BaseHandler): logger.debug("Got response to make_join: %s", pdu) - event = self.pdu_codec.event_from_pdu(pdu) + event = pdu # We should assert some things. assert(event.type == RoomMemberEvent.TYPE) @@ -310,10 +304,10 @@ class FederationHandler(BaseHandler): state = yield self.replication_layer.send_join( target_host, - self.pdu_codec.pdu_from_event(event) + event ) - state = [self.pdu_codec.event_from_pdu(p) for p in state] + state = [p for p in state] logger.debug("do_invite_join state: %s", state) @@ -387,7 +381,7 @@ class FederationHandler(BaseHandler): yield self.auth.add_auth_events(event) self.auth.check(event, raises=True) - pdu = self.pdu_codec.pdu_from_event(event) + pdu = event defer.returnValue(pdu) @@ -397,7 +391,7 @@ class FederationHandler(BaseHandler): """ We have received a join event for a room. Fully process it and respond with the current state and auth chains. """ - event = self.pdu_codec.event_from_pdu(pdu) + event = pdu event.outlier = False @@ -429,7 +423,7 @@ class FederationHandler(BaseHandler): "user_joined_room", user=user, room_id=event.room_id ) - new_pdu = self.pdu_codec.pdu_from_event(event) + new_pdu = event destinations = set() @@ -450,17 +444,10 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu) auth_chain = yield self.store.get_auth_chain(event.event_id) - pdu_auth_chain = [ - self.pdu_codec.pdu_from_event(e) - for e in auth_chain - ] defer.returnValue({ - "state": [ - self.pdu_codec.pdu_from_event(e) - for e in event.state_events.values() - ], - "auth_chain": pdu_auth_chain, + "state": event.state_events.values(), + "auth_chain": auth_chain, }) @defer.inlineCallbacks @@ -469,7 +456,7 @@ class FederationHandler(BaseHandler): Respond with the now signed event. """ - event = self.pdu_codec.event_from_pdu(pdu) + event = pdu event.outlier = True @@ -493,7 +480,7 @@ class FederationHandler(BaseHandler): event, extra_users=[target_user], ) - defer.returnValue(self.pdu_codec.pdu_from_event(event)) + defer.returnValue(event) @defer.inlineCallbacks def get_state_for_pdu(self, origin, room_id, event_id): @@ -524,12 +511,7 @@ class FederationHandler(BaseHandler): else: del results[(event.type, event.state_key)] - defer.returnValue( - [ - self.pdu_codec.pdu_from_event(s) - for s in results.values() - ] - ) + defer.returnValue(results.values()) else: defer.returnValue([]) @@ -546,10 +528,7 @@ class FederationHandler(BaseHandler): limit ) - defer.returnValue([ - self.pdu_codec.pdu_from_event(e) - for e in events - ]) + defer.returnValue(events) @defer.inlineCallbacks @log_function @@ -572,7 +551,7 @@ class FederationHandler(BaseHandler): if not in_room: raise AuthError(403, "Host not in room.") - defer.returnValue(self.pdu_codec.pdu_from_event(event)) + defer.returnValue(event) else: defer.returnValue(None) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 30e6eac8db..5d4be09a82 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -492,6 +492,14 @@ class SQLBaseStore(object): for n, s in signatures.items() } + hashes = self._get_event_content_hashes_txn( + txn, ev.event_id, + ) + + ev.hashes = { + k: encode_base64(v) for k, v in hashes.items() + } + prevs = self._get_prev_events_and_state(txn, ev.event_id) ev.prev_events = [ diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index ad09fab392..efac4075dc 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -23,7 +23,7 @@ from ..utils import MockHttpResource, MockClock, MockKey from synapse.server import HomeServer from synapse.federation import initialize_http_replication -from synapse.federation.units import Pdu +from synapse.api.events import SynapseEvent def make_pdu(prev_pdus=[], **kwargs): @@ -40,7 +40,7 @@ def make_pdu(prev_pdus=[], **kwargs): } pdu_fields.update(kwargs) - return Pdu(prev_pdus=prev_pdus, **pdu_fields) + return SynapseEvent(prev_pdus=prev_pdus, **pdu_fields) class FederationTestCase(unittest.TestCase): @@ -169,7 +169,7 @@ class FederationTestCase(unittest.TestCase): (200, "OK") ) - pdu = Pdu( + pdu = SynapseEvent( event_id="abc123def456", origin="red", room_id="my-context", @@ -189,7 +189,7 @@ class FederationTestCase(unittest.TestCase): "origin_server_ts": 1000000, "origin": "test", "pdus": [ - pdu.get_dict(), + pdu.get_pdu_json(), ], 'pdu_failures': [], }, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 3f17ca8fb0..e19b073817 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -19,9 +19,10 @@ from tests import unittest from synapse.api.events.room import ( MessageEvent, ) + +from synapse.api.events import SynapseEvent from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer -from synapse.federation.units import Pdu from mock import NonCallableMock, ANY, Mock @@ -74,7 +75,7 @@ class FederationTestCase(unittest.TestCase): @defer.inlineCallbacks def test_msg(self): - pdu = Pdu( + pdu = SynapseEvent( type=MessageEvent.TYPE, room_id="foo", content={"msgtype": u"fooo"}, -- cgit 1.5.1 From 17f977a9de458e375a8e09f8f68ec4393e0dd1bb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 14:07:51 +0000 Subject: Include 'messages' snapshot in room initialSync --- synapse/handlers/message.py | 21 ++++++++++++++++++++- tests/rest/test_rooms.py | 4 ++++ 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b77d9d1644..778cdb2317 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -308,10 +308,29 @@ class MessageHandler(BaseHandler): room_id=room_id )) + now_token = yield self.hs.get_event_sources().get_current_token() + + limit = pagin_config.limit if pagin_config else None + if limit is None: + limit = 10 + + messages, token = yield self.store.get_recent_events_for_room( + room_id, + limit=limit, + end_token=now_token.room_key, + ) + + start_token = now_token.copy_and_replace("room_key", token[0]) + end_token = now_token.copy_and_replace("room_key", token[1]) + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, - #"messages": messages, + "messages": { + "chunk": [self.hs.serialize_event(m) for m in messages], + "start": start_token.to_string(), + "end": end_token.to_string(), + }, "state": state, #"presence": presence }) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 98c6af97b0..b7d9a2bd06 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1050,6 +1050,10 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertTrue("m.room.create" in state) + self.assertTrue("messages" in response) + self.assertTrue("chunk" in response["messages"]) + self.assertTrue("end" in response["messages"]) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) -- cgit 1.5.1 From 33d328d967341d8819632bc3c5406b8c54f54504 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 15:28:58 +0000 Subject: Include room members' presence in room initialSync --- synapse/handlers/message.py | 23 ++++++++++++++++++++++- tests/rest/test_rooms.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 778cdb2317..21f9df30ec 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -299,6 +299,10 @@ class MessageHandler(BaseHandler): feedback=False): yield self.auth.check_joined_room(room_id, user_id) + # TODO(paul): I wish I was called with user objects not user_id + # strings... + auth_user = self.hs.parse_userid(user_id) + # TODO: These concurrently state_tuples = yield self.store.get_current_state(room_id) state = [self.hs.serialize_event(x) for x in state_tuples] @@ -323,6 +327,23 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) + room_members = yield self.store.get_room_members(room_id) + + presence_handler = self.hs.get_handlers().presence_handler + presence = [] + for m in room_members: + try: + member_presence = yield presence_handler.get_state( + target_user=self.hs.parse_userid(m.user_id), + auth_user=auth_user, + as_event=True, + ) + presence.append(member_presence) + except Exception as e: + logger.exception("Failed to get member presence of %r", + m.user_id + ) + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, @@ -332,5 +353,5 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), }, "state": state, - #"presence": presence + "presence": presence }) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index b7d9a2bd06..ad3631d510 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1026,6 +1026,12 @@ class RoomInitialSyncTestCase(RestTestCase): synapse.rest.room.register_servlets(hs, self.mock_resource) + # Since I'm getting my own presence I need to exist as far as presence + # is concerned. + hs.get_handlers().presence_handler.registered_user( + hs.parse_userid(self.user_id) + ) + # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -1054,6 +1060,14 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertTrue("chunk" in response["messages"]) self.assertTrue("end" in response["messages"]) + self.assertTrue("presence" in response) + + presence_by_user = {e["content"]["user_id"]: e + for e in response["presence"] + } + self.assertTrue(self.user_id in presence_by_user) + self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) -- cgit 1.5.1 From c5eabe3143bc3398961ed80da041f28e545386bf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 19 Nov 2014 16:38:40 +0000 Subject: replace user_id with sender --- synapse/api/events/__init__.py | 4 +++- synapse/federation/replication.py | 3 +++ tests/federation/test_federation.py | 3 +++ tests/handlers/test_federation.py | 3 ++- 4 files changed, 11 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py index 0019789dd1..a01c4a1351 100644 --- a/synapse/api/events/__init__.py +++ b/synapse/api/events/__init__.py @@ -119,7 +119,7 @@ class SynapseEvent(JsonEncodedObject): def get_pdu_json(self, time_now=None): pdu_json = self.get_full_dict() - pdu_json.pop("destination", None) + pdu_json.pop("destinations", None) pdu_json.pop("outlier", None) pdu_json.pop("replaces_state", None) pdu_json.pop("redacted", None) @@ -132,6 +132,8 @@ class SynapseEvent(JsonEncodedObject): age = time_now - pdu_json["age_ts"] pdu_json.setdefault("unsigned", {})["age"] = int(age) del pdu_json["age_ts"] + user_id = pdu_json.pop("user_id") + pdu_json["sender"] = user_id return pdu_json diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2ed303e12d..65a53ae17c 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -582,6 +582,9 @@ class ReplicationLayer(object): #TODO: Check we have all the PDU keys here pdu_json.setdefault("hashes", {}) pdu_json.setdefault("signatures", {}) + sender = pdu_json.pop("sender", None) + if sender is not None: + pdu_json["user_id"] = sender state_hash = pdu_json.get("unsigned", {}).pop("state_hash", None) if state_hash is not None: pdu_json["state_hash"] = state_hash diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index efac4075dc..73dd289276 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -100,6 +100,7 @@ class FederationTestCase(unittest.TestCase): make_pdu( event_id="the-pdu-id", origin="red", + user_id="@a:red", room_id="my-context", type="m.topic", origin_server_ts=123456789000, @@ -145,6 +146,7 @@ class FederationTestCase(unittest.TestCase): make_pdu( event_id="abc123def456", origin="red", + user_id="@a:red", room_id="my-context", type="m.text", origin_server_ts=123456789001, @@ -172,6 +174,7 @@ class FederationTestCase(unittest.TestCase): pdu = SynapseEvent( event_id="abc123def456", origin="red", + user_id="@a:red", room_id="my-context", type="m.text", origin_server_ts=123456789001, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index e19b073817..3487a090e9 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -81,8 +81,9 @@ class FederationTestCase(unittest.TestCase): content={"msgtype": u"fooo"}, origin_server_ts=0, event_id="$a:b", + user_id="@a:b", origin="b", - hashes={"sha256":"PvbCLWrTBxnBsSO7/cJ76072ySTCgI/XGadESRAe02M"}, + hashes={"sha256":"AcLrgtUIqqwaGoHhrEvYG1YLDIsVPYJdSRGhkp3jJp8"}, ) self.datastore.persist_event.return_value = defer.succeed(None) -- cgit 1.5.1 From 03dc63f6c86f82743b53adadf7d442d6b478a1bf Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 19 Nov 2014 17:30:35 +0000 Subject: Initialise UserPresenceCache instances to always contain a 'presence' key --- synapse/handlers/presence.py | 6 ++---- tests/handlers/test_presence.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 325ae45257..fcc92a8e32 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -139,9 +139,7 @@ class PresenceHandler(BaseHandler): if user in self._user_cachemap: return self._user_cachemap[user] else: - statuscache = UserPresenceCache() - statuscache.update({"presence": PresenceState.OFFLINE}, user) - return statuscache + return UserPresenceCache() def registered_user(self, user): self.store.create_presence(user.localpart) @@ -873,7 +871,7 @@ class UserPresenceCache(object): Includes the update timestamp. """ def __init__(self): - self.state = {} + self.state = {"presence": PresenceState.OFFLINE} self.serial = None def update(self, state, serial): diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index cdaf93429b..fe69ce47eb 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -1200,7 +1200,10 @@ class PresencePollingTestCase(unittest.TestCase): path=ANY, data=_expect_edu("remote", "m.presence", content={ - "push": [ {"user_id": "@clementine:test" }], + "push": [ { + "user_id": "@clementine:test", + "presence": OFFLINE, + }], }, ), json_data_callback=ANY, @@ -1229,7 +1232,10 @@ class PresencePollingTestCase(unittest.TestCase): path=ANY, data=_expect_edu("remote", "m.presence", content={ - "push": [ {"user_id": "@fig:test" }], + "push": [ { + "user_id": "@fig:test", + "presence": OFFLINE, + }], }, ), json_data_callback=ANY, -- cgit 1.5.1 From 493055731e40f1004679d7e76f8663d0439df4bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Nov 2014 18:00:07 +0000 Subject: Fix tests from prev commit --- tests/rest/test_rooms.py | 1 - 1 file changed, 1 deletion(-) (limited to 'tests') diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index ad3631d510..ff7c9f0530 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -840,7 +840,6 @@ class RoomMemberStateTestCase(RestTestCase): expected_response = { "membership": Membership.JOIN, - "prev": Membership.JOIN, } self.assertEquals(expected_response, response) -- cgit 1.5.1