Merge branch 'develop' into request_logging

Conflicts:
	setup.py
	synapse/storage/_base.py
	synapse/util/async.py

7 files changed, 108 insertions, 222 deletions

diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index e10a49a8ac..8e164e4be0 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -21,9 +21,8 @@ from mock import Mock
 
 from synapse.server import HomeServer
 from synapse.handlers.directory import DirectoryHandler
-from synapse.storage.directory import RoomAliasMapping
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import SQLiteMemoryDbPool, MockKey
 
 
 class DirectoryHandlers(object):
@@ -41,6 +40,7 @@ class DirectoryTestCase(unittest.TestCase):
         ])
 
         self.query_handlers = {}
+
         def register_query_handler(query_type, handler):
             self.query_handlers[query_type] = handler
         self.mock_federation.register_query_handler = register_query_handler
@@ -48,11 +48,16 @@ class DirectoryTestCase(unittest.TestCase):
         db_pool = SQLiteMemoryDbPool()
         yield db_pool.prepare()
 
-        hs = HomeServer("test",
+        self.mock_config = Mock()
+        self.mock_config.signing_key = [MockKey()]
+
+        hs = HomeServer(
+            "test",
             db_pool=db_pool,
             http_client=None,
             resource_for_federation=Mock(),
             replication_layer=self.mock_federation,
+            config=self.mock_config,
         )
         hs.handlers = DirectoryHandlers(hs)
 
diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py
index 219b2c4c5e..e386cddb38 100644
--- a/tests/handlers/test_federation.py
+++ b/tests/handlers/test_federation.py
@@ -17,16 +17,15 @@ from twisted.internet import defer
 from tests import unittest
 
 from synapse.api.events.room import (
-    InviteJoinEvent, MessageEvent, RoomMemberEvent
+    MessageEvent,
 )
-from synapse.api.constants import Membership
 from synapse.handlers.federation import FederationHandler
 from synapse.server import HomeServer
 from synapse.federation.units import Pdu
 
 from mock import NonCallableMock, ANY
 
-from ..utils import get_mock_call_args, MockKey
+from ..utils import MockKey
 
 
 class FederationTestCase(unittest.TestCase):
@@ -36,6 +35,14 @@ class FederationTestCase(unittest.TestCase):
         self.mock_config = NonCallableMock()
         self.mock_config.signing_key = [MockKey()]
 
+        self.state_handler = NonCallableMock(spec_set=[
+            "annotate_event_with_state",
+        ])
+
+        self.auth = NonCallableMock(spec_set=[
+            "check",
+        ])
+
         self.hostname = "test"
         hs = HomeServer(
             self.hostname,
@@ -53,6 +60,8 @@ class FederationTestCase(unittest.TestCase):
                 "federation_handler",
             ]),
             config=self.mock_config,
+            auth=self.auth,
+            state_handler=self.state_handler,
         )
 
         self.datastore = hs.get_datastore()
@@ -65,74 +74,35 @@ class FederationTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def test_msg(self):
         pdu = Pdu(
-            pdu_type=MessageEvent.TYPE,
-            context="foo",
+            type=MessageEvent.TYPE,
+            room_id="foo",
             content={"msgtype": u"fooo"},
             origin_server_ts=0,
-            pdu_id="a",
+            event_id="$a:b",
             origin="b",
         )
 
-        store_id = "ASD"
-        self.datastore.persist_event.return_value = defer.succeed(store_id)
+        self.datastore.persist_event.return_value = defer.succeed(None)
         self.datastore.get_room.return_value = defer.succeed(True)
 
+        self.state_handler.annotate_event_with_state.return_value = (
+            defer.succeed(False)
+        )
+
         yield self.handlers.federation_handler.on_receive_pdu(pdu, False)
 
         self.datastore.persist_event.assert_called_once_with(
             ANY, False, is_new_state=False
         )
-        self.notifier.on_new_room_event.assert_called_once_with(ANY, extra_users=[])
-
-    @defer.inlineCallbacks
-    def test_invite_join_target_this(self):
-        room_id = "foo"
-        user_id = "@bob:red"
 
-        pdu = Pdu(
-            pdu_type=InviteJoinEvent.TYPE,
-            user_id=user_id,
-            target_host=self.hostname,
-            context=room_id,
-            content={},
-            origin_server_ts=0,
-            pdu_id="a",
-            origin="b",
+        self.state_handler.annotate_event_with_state.assert_called_once_with(
+            ANY,
+            old_state=None,
         )
 
-        yield self.handlers.federation_handler.on_receive_pdu(pdu, False)
+        self.auth.check.assert_called_once_with(ANY, raises=True)
 
-        mem_handler = self.handlers.room_member_handler
-        self.assertEquals(1, mem_handler.change_membership.call_count)
-        call_args = get_mock_call_args(
-            lambda event, do_auth: None,
-            mem_handler.change_membership
+        self.notifier.on_new_room_event.assert_called_once_with(
+            ANY,
+            extra_users=[]
         )
-        self.assertEquals(False, call_args["do_auth"])
-
-        new_event = call_args["event"]
-        self.assertEquals(RoomMemberEvent.TYPE, new_event.type)
-        self.assertEquals(room_id, new_event.room_id)
-        self.assertEquals(user_id, new_event.state_key)
-        self.assertEquals(Membership.JOIN, new_event.membership)
-
-    @defer.inlineCallbacks
-    def test_invite_join_target_other(self):
-        room_id = "foo"
-        user_id = "@bob:red"
-
-        pdu = Pdu(
-            pdu_type=InviteJoinEvent.TYPE,
-            user_id=user_id,
-            state_key="@red:not%s" % self.hostname,
-            context=room_id,
-            content={},
-            origin_server_ts=0,
-            pdu_id="a",
-            origin="b",
-        )
-
-        yield self.handlers.federation_handler.on_receive_pdu(pdu, False)
-
-        mem_handler = self.handlers.room_member_handler
-        self.assertEquals(0, mem_handler.change_membership.call_count)
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index fdc2e8de4a..cdaf93429b 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -44,13 +44,11 @@ def _expect_edu(destination, edu_type, content, origin="test"):
         "pdus": [],
         "edus": [
             {
-                # TODO: SYN-103: Remove "origin" and "destination" keys.
-                "origin": origin,
-                "destination": destination,
                 "edu_type": edu_type,
                 "content": content,
             }
         ],
+        "pdu_failures": [],
     }
 
 def _make_edu_json(origin, edu_type, content):
diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py
index 047752ad68..532ecf0f2c 100644
--- a/tests/handlers/test_presencelike.py
+++ b/tests/handlers/test_presencelike.py
@@ -21,7 +21,7 @@ from twisted.internet import defer
 
 from mock import Mock, call, ANY
 
-from ..utils import MockClock
+from ..utils import MockClock, MockKey
 
 from synapse.server import HomeServer
 from synapse.api.constants import PresenceState
@@ -57,6 +57,9 @@ class PresenceAndProfileHandlers(object):
 class PresenceProfilelikeDataTestCase(unittest.TestCase):
 
     def setUp(self):
+        self.mock_config = Mock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer("test",
                 clock=MockClock(),
                 db_pool=None,
@@ -72,6 +75,7 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase):
                 resource_for_federation=Mock(),
                 http_client=None,
                 replication_layer=MockReplication(),
+                config=self.mock_config,
             )
         hs.handlers = PresenceAndProfileHandlers(hs)
 
diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py
index 5dc9b456e1..1660e7e928 100644
--- a/tests/handlers/test_profile.py
+++ b/tests/handlers/test_profile.py
@@ -24,7 +24,7 @@ from synapse.server import HomeServer
 from synapse.handlers.profile import ProfileHandler
 from synapse.api.constants import Membership
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import SQLiteMemoryDbPool, MockKey
 
 
 class ProfileHandlers(object):
@@ -49,12 +49,16 @@ class ProfileTestCase(unittest.TestCase):
         db_pool = SQLiteMemoryDbPool()
         yield db_pool.prepare()
 
+        self.mock_config = Mock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer("test",
                 db_pool=db_pool,
                 http_client=None,
                 handlers=None,
                 resource_for_federation=Mock(),
                 replication_layer=self.mock_federation,
+                config=self.mock_config,
             )
         hs.handlers = ProfileHandlers(hs)
 
diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py
index c88d1c8840..cbe591ab90 100644
--- a/tests/handlers/test_room.py
+++ b/tests/handlers/test_room.py
@@ -18,7 +18,7 @@ from twisted.internet import defer
 from tests import unittest
 
 from synapse.api.events.room import (
-    InviteJoinEvent, RoomMemberEvent, RoomConfigEvent
+    RoomMemberEvent,
 )
 from synapse.api.constants import Membership
 from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler
@@ -34,6 +34,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
     def setUp(self):
         self.mock_config = NonCallableMock()
         self.mock_config.signing_key = [MockKey()]
+
         self.hostname = "red"
         hs = HomeServer(
             self.hostname,
@@ -43,7 +44,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             ]),
             datastore=NonCallableMock(spec_set=[
                 "persist_event",
-                "get_joined_hosts_for_room",
                 "get_room_member",
                 "get_room",
                 "store_room",
@@ -57,13 +57,21 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
                 "profile_handler",
                 "federation_handler",
             ]),
-            auth=NonCallableMock(spec_set=["check"]),
-            state_handler=NonCallableMock(spec_set=["handle_new_event"]),
+            auth=NonCallableMock(spec_set=[
+                "check",
+                "add_auth_events",
+                "check_host_in_room",
+            ]),
+            state_handler=NonCallableMock(spec_set=[
+                "annotate_event_with_state",
+                "get_current_state",
+            ]),
             config=self.mock_config,
         )
 
         self.federation = NonCallableMock(spec_set=[
             "handle_new_event",
+            "send_invite",
             "get_state_for_room",
         ])
 
@@ -72,6 +80,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
         self.notifier = hs.get_notifier()
         self.state_handler = hs.get_state_handler()
         self.distributor = hs.get_distributor()
+        self.auth = hs.get_auth()
         self.hs = hs
 
         self.handlers.federation_handler = self.federation
@@ -104,28 +113,34 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             content=content,
         )
 
-        joined = ["red", "green"]
-
-        self.state_handler.handle_new_event.return_value = defer.succeed(True)
-        self.datastore.get_joined_hosts_for_room.return_value = (
-            defer.succeed(joined)
-        )
+        self.auth.check_host_in_room.return_value = defer.succeed(True)
 
         store_id = "store_id_fooo"
         self.datastore.persist_event.return_value = defer.succeed(store_id)
 
+        self.datastore.get_room_member.return_value = defer.succeed(None)
+
+        event.state_events = {
+            (RoomMemberEvent.TYPE, "@alice:green"): self._create_member(
+                user_id="@alice:green",
+                room_id=room_id,
+            ),
+            (RoomMemberEvent.TYPE, "@bob:red"): self._create_member(
+                user_id="@bob:red",
+                room_id=room_id,
+            ),
+            (RoomMemberEvent.TYPE, target_user_id): event,
+        }
+
         # Actual invocation
         yield self.room_member_handler.change_membership(event)
 
-        self.state_handler.handle_new_event.assert_called_once_with(
-            event, self.snapshot,
-        )
         self.federation.handle_new_event.assert_called_once_with(
             event, self.snapshot,
         )
 
         self.assertEquals(
-            set(["blue", "red", "green"]),
+            set(["red", "green"]),
             set(event.destinations)
         )
 
@@ -144,27 +159,13 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
         room_id = "!foo:red"
         user_id = "@bob:red"
         user = self.hs.parse_userid(user_id)
-        target_user_id = "@bob:red"
-        content = {"membership": Membership.JOIN}
 
-        event = self.hs.get_event_factory().create_event(
-            etype=RoomMemberEvent.TYPE,
+        event = self._create_member(
             user_id=user_id,
-            state_key=target_user_id,
             room_id=room_id,
-            membership=Membership.JOIN,
-            content=content,
         )
 
-        joined = ["red", "green"]
-
-        self.state_handler.handle_new_event.return_value = defer.succeed(True)
-
-        def get_joined(*args):
-            return defer.succeed(joined)
-
-        self.datastore.get_joined_hosts_for_room.side_effect = get_joined
-
+        self.auth.check_host_in_room.return_value = defer.succeed(True)
 
         store_id = "store_id_fooo"
         self.datastore.persist_event.return_value = defer.succeed(store_id)
@@ -178,12 +179,17 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
         join_signal_observer = Mock()
         self.distributor.observe("user_joined_room", join_signal_observer)
 
+        event.state_events = {
+            (RoomMemberEvent.TYPE, "@alice:green"): self._create_member(
+                user_id="@alice:green",
+                room_id=room_id,
+            ),
+            (RoomMemberEvent.TYPE, user_id): event,
+        }
+
         # Actual invocation
         yield self.room_member_handler.change_membership(event)
 
-        self.state_handler.handle_new_event.assert_called_once_with(
-            event, self.snapshot
-        )
         self.federation.handle_new_event.assert_called_once_with(
             event, self.snapshot
         )
@@ -197,138 +203,32 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
             event
         )
         self.notifier.on_new_room_event.assert_called_once_with(
-                event, extra_users=[user])
-
-        join_signal_observer.assert_called_with(
-                user=user, room_id=room_id)
-
-    @defer.inlineCallbacks
-    def STALE_test_invite_join(self):
-        room_id = "foo"
-        user_id = "@bob:red"
-        target_user_id = "@bob:red"
-        content = {"membership": Membership.JOIN}
-
-        event = self.hs.get_event_factory().create_event(
-            etype=RoomMemberEvent.TYPE,
-            user_id=user_id,
-            target_user_id=target_user_id,
-            room_id=room_id,
-            membership=Membership.JOIN,
-            content=content,
-        )
-
-        joined = ["red", "blue", "green"]
-
-        self.state_handler.handle_new_event.return_value = defer.succeed(True)
-        self.datastore.get_joined_hosts_for_room.return_value = (
-            defer.succeed(joined)
-        )
-
-        store_id = "store_id_fooo"
-        self.datastore.store_room_member.return_value = defer.succeed(store_id)
-        self.datastore.get_room.return_value = defer.succeed(None)
-
-        prev_state = NonCallableMock(name="prev_state")
-        prev_state.membership = Membership.INVITE
-        prev_state.sender = "@foo:blue"
-        self.datastore.get_room_member.return_value = defer.succeed(prev_state)
-
-        # Actual invocation
-        yield self.room_member_handler.change_membership(event)
-
-        self.datastore.get_room_member.assert_called_once_with(
-            target_user_id, room_id
-        )
-
-        self.assertTrue(self.federation.handle_new_event.called)
-        args = self.federation.handle_new_event.call_args[0]
-        invite_join_event = args[0]
-
-        self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE)
-        self.assertTrue("blue", invite_join_event.target_host)
-        self.assertTrue(room_id, invite_join_event.room_id)
-        self.assertTrue(user_id, invite_join_event.user_id)
-        self.assertFalse(hasattr(invite_join_event, "state_key"))
-
-        self.assertEquals(
-            set(["blue"]),
-            set(invite_join_event.destinations)
+            event, extra_users=[user]
         )
 
-        self.federation.get_state_for_room.assert_called_once_with(
-            "blue", room_id
+        join_signal_observer.assert_called_with(
+            user=user, room_id=room_id
         )
 
-        self.assertFalse(self.datastore.store_room_member.called)
-
-        self.assertFalse(self.notifier.on_new_room_event.called)
-        self.assertFalse(self.state_handler.handle_new_event.called)
-
-    @defer.inlineCallbacks
-    def STALE_test_invite_join_public(self):
-        room_id = "#foo:blue"
-        user_id = "@bob:red"
-        target_user_id = "@bob:red"
-        content = {"membership": Membership.JOIN}
-
-        event = self.hs.get_event_factory().create_event(
+    def _create_member(self, user_id, room_id):
+        return self.hs.get_event_factory().create_event(
             etype=RoomMemberEvent.TYPE,
             user_id=user_id,
-            target_user_id=target_user_id,
+            state_key=user_id,
             room_id=room_id,
             membership=Membership.JOIN,
-            content=content,
+            content={"membership": Membership.JOIN},
         )
 
-        joined = ["red", "blue", "green"]
-
-        self.state_handler.handle_new_event.return_value = defer.succeed(True)
-        self.datastore.get_joined_hosts_for_room.return_value = (
-            defer.succeed(joined)
-        )
-
-        store_id = "store_id_fooo"
-        self.datastore.store_room_member.return_value = defer.succeed(store_id)
-        self.datastore.get_room.return_value = defer.succeed(None)
-
-        prev_state = NonCallableMock(name="prev_state")
-        prev_state.membership = Membership.INVITE
-        prev_state.sender = "@foo:blue"
-        self.datastore.get_room_member.return_value = defer.succeed(prev_state)
-
-        # Actual invocation
-        yield self.room_member_handler.change_membership(event)
-
-        self.assertTrue(self.federation.handle_new_event.called)
-        args = self.federation.handle_new_event.call_args[0]
-        invite_join_event = args[0]
-
-        self.assertTrue(InviteJoinEvent.TYPE, invite_join_event.TYPE)
-        self.assertTrue("blue", invite_join_event.target_host)
-        self.assertTrue("foo", invite_join_event.room_id)
-        self.assertTrue(user_id, invite_join_event.user_id)
-        self.assertFalse(hasattr(invite_join_event, "state_key"))
-
-        self.assertEquals(
-            set(["blue"]),
-            set(invite_join_event.destinations)
-        )
-
-        self.federation.get_state_for_room.assert_called_once_with(
-            "blue", "foo"
-        )
-
-        self.assertFalse(self.datastore.store_room_member.called)
-
-        self.assertFalse(self.notifier.on_new_room_event.called)
-        self.assertFalse(self.state_handler.handle_new_event.called)
-
 
 class RoomCreationTest(unittest.TestCase):
 
     def setUp(self):
         self.hostname = "red"
+
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer(
             self.hostname,
             db_pool=None,
@@ -345,12 +245,14 @@ class RoomCreationTest(unittest.TestCase):
                 "room_member_handler",
                 "federation_handler",
             ]),
-            auth=NonCallableMock(spec_set=["check"]),
-            state_handler=NonCallableMock(spec_set=["handle_new_event"]),
+            auth=NonCallableMock(spec_set=["check", "add_auth_events"]),
+            state_handler=NonCallableMock(spec_set=[
+                "annotate_event_with_state",
+            ]),
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
 
         self.federation = NonCallableMock(spec_set=[
@@ -373,6 +275,11 @@ class RoomCreationTest(unittest.TestCase):
         ])
         self.room_member_handler = self.handlers.room_member_handler
 
+        def annotate(event):
+            event.state_events = {}
+            return defer.succeed(None)
+        self.state_handler.annotate_event_with_state.side_effect = annotate
+
         def hosts(room):
             return defer.succeed([])
         self.datastore.get_joined_hosts_for_room.side_effect = hosts
@@ -400,6 +307,6 @@ class RoomCreationTest(unittest.TestCase):
         self.assertEquals(user_id, join_event.user_id)
         self.assertEquals(user_id, join_event.state_key)
 
-        self.assertTrue(self.state_handler.handle_new_event.called)
+        self.assertTrue(self.state_handler.annotate_event_with_state.called)
 
         self.assertTrue(self.federation.handle_new_event.called)
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index f1d3b27f74..adb5148351 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -33,13 +33,11 @@ def _expect_edu(destination, edu_type, content, origin="test"):
         "pdus": [],
         "edus": [
             {
-                # TODO: SYN-103: Remove "origin" and "destination" keys.
-                "origin": origin,
-                "destination": destination,
                 "edu_type": edu_type,
                 "content": content,
             }
         ],
+        "pdu_failures": [],
     }