Merge branch 'develop' into request_logging

Conflicts:
	setup.py
	synapse/storage/_base.py
	synapse/util/async.py

3 files changed, 172 insertions, 118 deletions

diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py
index 79b371c04d..4a3234c332 100644
--- a/tests/rest/test_events.py
+++ b/tests/rest/test_events.py
@@ -25,10 +25,7 @@ import synapse.rest.room
 
 from synapse.server import HomeServer
 
-# python imports
-import json
-
-from ..utils import MockHttpResource, MemoryDataStore
+from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -49,7 +46,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase):
     def tearDown(self):
         pass
 
-    def test_long_poll(self):
+    def TODO_test_long_poll(self):
         # stream from 'end' key, send (self+other) message, expect message.
 
         # stream from 'END', send (self+other) message, expect message.
@@ -64,7 +61,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase):
 
         pass
 
-    def test_stream_forward(self):
+    def TODO_test_stream_forward(self):
         # stream from START, expect injected items
 
         # stream from 'start' key, expect same content
@@ -80,14 +77,14 @@ class EventStreamPaginationApiTestCase(unittest.TestCase):
         # returned as end key
         pass
 
-    def test_limits(self):
+    def TODO_test_limits(self):
         # stream from a key, expect limit_num items
 
         # stream from START, expect limit_num items
 
         pass
 
-    def test_range(self):
+    def TODO_test_range(self):
         # stream from key to key, expect X items
 
         # stream from key to END, expect X items
@@ -97,7 +94,7 @@ class EventStreamPaginationApiTestCase(unittest.TestCase):
         # stream from START to END, expect all items
         pass
 
-    def test_direction(self):
+    def TODO_test_direction(self):
         # stream from END to START and fwds, expect newest first
 
         # stream from END to START and bwds, expect oldest first
@@ -116,19 +113,20 @@ class EventStreamPermissionsTestCase(RestTestCase):
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
-
         persistence_service = Mock(spec=["get_latest_pdus_in_context"])
         persistence_service.get_latest_pdus_in_context.return_value = []
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
+
         hs = HomeServer(
             "test",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            state_handler=state_handler,
-            datastore=MemoryDataStore(),
             persistence_service=persistence_service,
             clock=Mock(spec=[
                 "call_later",
@@ -139,7 +137,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -148,6 +146,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
         hs.get_handlers().federation_handler = Mock()
 
         hs.get_clock().time_msec.return_value = 1000000
+        hs.get_clock().time.return_value = 1000
 
         synapse.rest.register.register_servlets(hs, self.mock_resource)
         synapse.rest.events.register_servlets(hs, self.mock_resource)
@@ -172,12 +171,14 @@ class EventStreamPermissionsTestCase(RestTestCase):
     def test_stream_basic_permissions(self):
         # invalid token, expect 403
         (code, response) = yield self.mock_resource.trigger_get(
-                           "/events?access_token=%s" % ("invalid" + self.token))
+            "/events?access_token=%s" % ("invalid" + self.token, )
+        )
         self.assertEquals(403, code, msg=str(response))
 
         # valid token, expect content
         (code, response) = yield self.mock_resource.trigger_get(
-                           "/events?access_token=%s&timeout=0" % (self.token))
+            "/events?access_token=%s&timeout=0" % (self.token,)
+        )
         self.assertEquals(200, code, msg=str(response))
         self.assertTrue("chunk" in response)
         self.assertTrue("start" in response)
@@ -185,15 +186,23 @@ class EventStreamPermissionsTestCase(RestTestCase):
 
     @defer.inlineCallbacks
     def test_stream_room_permissions(self):
-        room_id = yield self.create_room_as(self.other_user,
-                                            tok=self.other_token)
+        room_id = yield self.create_room_as(
+            self.other_user,
+            tok=self.other_token
+        )
         yield self.send(room_id, tok=self.other_token)
 
         # invited to room (expect no content for room)
-        yield self.invite(room_id, src=self.other_user, targ=self.user_id,
-                          tok=self.other_token)
+        yield self.invite(
+            room_id,
+            src=self.other_user,
+            targ=self.user_id,
+            tok=self.other_token
+        )
+
         (code, response) = yield self.mock_resource.trigger_get(
-                           "/events?access_token=%s&timeout=0" % (self.token))
+            "/events?access_token=%s&timeout=0" % (self.token,)
+        )
         self.assertEquals(200, code, msg=str(response))
 
         self.assertEquals(0, len(response["chunk"]))
@@ -203,7 +212,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
 
         # left to room (expect no content for room)
 
-    def test_stream_items(self):
+    def TODO_test_stream_items(self):
         # new user, no content
 
         # join room, expect 1 item (join)
diff --git a/tests/rest/test_profile.py b/tests/rest/test_profile.py
index b0f48e7fd8..3a0d1e700a 100644
--- a/tests/rest/test_profile.py
+++ b/tests/rest/test_profile.py
@@ -18,9 +18,9 @@
 from tests import unittest
 from twisted.internet import defer
 
-from mock import Mock
+from mock import Mock, NonCallableMock
 
-from ..utils import MockHttpResource
+from ..utils import MockHttpResource, MockKey
 
 from synapse.api.errors import SynapseError, AuthError
 from synapse.server import HomeServer
@@ -41,6 +41,9 @@ class ProfileTestCase(unittest.TestCase):
             "set_avatar_url",
         ])
 
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
+
         hs = HomeServer("test",
             db_pool=None,
             http_client=None,
@@ -48,6 +51,7 @@ class ProfileTestCase(unittest.TestCase):
             federation=Mock(),
             replication_layer=Mock(),
             datastore=None,
+            config=self.mock_config,
         )
 
         def _get_user_by_req(request=None):
diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py
index 1ce9b8a83d..e27990dace 100644
--- a/tests/rest/test_rooms.py
+++ b/tests/rest/test_rooms.py
@@ -23,11 +23,14 @@ from synapse.api.constants import Membership
 
 from synapse.server import HomeServer
 
+from tests import unittest
+
 # python imports
 import json
 import urllib
+import types
 
-from ..utils import MockHttpResource, MemoryDataStore
+from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -44,24 +47,21 @@ class RoomPermissionsTestCase(RestTestCase):
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
-        persistence_service = Mock(spec=["get_latest_pdus_in_context"])
-        persistence_service.get_latest_pdus_in_context.return_value = []
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
 
         hs = HomeServer(
             "red",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
-            datastore=MemoryDataStore(),
             replication_layer=Mock(),
-            state_handler=state_handler,
-            persistence_service=persistence_service,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -76,6 +76,10 @@ class RoomPermissionsTestCase(RestTestCase):
             }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
+        def _insert_client_ip(*args, **kwargs):
+            return defer.succeed(None)
+        hs.get_datastore().insert_client_ip = _insert_client_ip
+
         self.auth_user_id = self.rmcreator_id
 
         synapse.rest.room.register_servlets(hs, self.mock_resource)
@@ -147,38 +151,55 @@ class RoomPermissionsTestCase(RestTestCase):
     @defer.inlineCallbacks
     def test_send_message(self):
         msg_content = '{"msgtype":"m.text","body":"hello"}'
-        send_msg_path = ("/rooms/%s/send/m.room.message/mid1" %
-                        (self.created_rmid))
+        send_msg_path = (
+            "/rooms/%s/send/m.room.message/mid1" % (self.created_rmid,)
+        )
 
         # send message in uncreated room, expect 403
         (code, response) = yield self.mock_resource.trigger(
-                           "PUT",
-                           "/rooms/%s/send/m.room.message/mid2" %
-                           (self.uncreated_rmid), msg_content)
+            "PUT",
+            "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,),
+            msg_content
+        )
         self.assertEquals(403, code, msg=str(response))
 
         # send message in created room not joined (no state), expect 403
         (code, response) = yield self.mock_resource.trigger(
-                           "PUT", send_msg_path, msg_content)
+            "PUT",
+            send_msg_path,
+            msg_content
+        )
         self.assertEquals(403, code, msg=str(response))
 
         # send message in created room and invited, expect 403
-        yield self.invite(room=self.created_rmid, src=self.rmcreator_id,
-                          targ=self.user_id)
+        yield self.invite(
+            room=self.created_rmid,
+            src=self.rmcreator_id,
+            targ=self.user_id
+        )
         (code, response) = yield self.mock_resource.trigger(
-                           "PUT", send_msg_path, msg_content)
+            "PUT",
+            send_msg_path,
+            msg_content
+        )
         self.assertEquals(403, code, msg=str(response))
 
         # send message in created room and joined, expect 200
         yield self.join(room=self.created_rmid, user=self.user_id)
         (code, response) = yield self.mock_resource.trigger(
-                           "PUT", send_msg_path, msg_content)
+            "PUT",
+            send_msg_path,
+            msg_content
+        )
         self.assertEquals(200, code, msg=str(response))
 
         # send message in created room and left, expect 403
         yield self.leave(room=self.created_rmid, user=self.user_id)
         (code, response) = yield self.mock_resource.trigger(
-                           "PUT", send_msg_path, msg_content)
+            "PUT",
+            send_msg_path,
+            msg_content
+        )
         self.assertEquals(403, code, msg=str(response))
 
     @defer.inlineCallbacks
@@ -209,15 +230,20 @@ class RoomPermissionsTestCase(RestTestCase):
                            "PUT", topic_path, topic_content)
         self.assertEquals(403, code, msg=str(response))
 
-        # get topic in created PRIVATE room and invited, expect 200 (or 404)
+        # get topic in created PRIVATE room and invited, expect 403
         (code, response) = yield self.mock_resource.trigger_get(topic_path)
-        self.assertEquals(404, code, msg=str(response))
+        self.assertEquals(403, code, msg=str(response))
 
         # set/get topic in created PRIVATE room and joined, expect 200
         yield self.join(room=self.created_rmid, user=self.user_id)
+
+        # Only room ops can set topic by default
+        self.auth_user_id = self.rmcreator_id
         (code, response) = yield self.mock_resource.trigger(
                            "PUT", topic_path, topic_content)
         self.assertEquals(200, code, msg=str(response))
+        self.auth_user_id = self.user_id
+
         (code, response) = yield self.mock_resource.trigger_get(topic_path)
         self.assertEquals(200, code, msg=str(response))
         self.assert_dict(json.loads(topic_content), response)
@@ -230,10 +256,10 @@ class RoomPermissionsTestCase(RestTestCase):
         (code, response) = yield self.mock_resource.trigger_get(topic_path)
         self.assertEquals(403, code, msg=str(response))
 
-        # get topic in PUBLIC room, not joined, expect 200 (or 404)
+        # get topic in PUBLIC room, not joined, expect 403
         (code, response) = yield self.mock_resource.trigger_get(
                            "/rooms/%s/state/m.room.topic" % self.created_public_rmid)
-        self.assertEquals(200, code, msg=str(response))
+        self.assertEquals(403, code, msg=str(response))
 
         # set topic in PUBLIC room, not joined, expect 403
         (code, response) = yield self.mock_resource.trigger(
@@ -300,12 +326,12 @@ class RoomPermissionsTestCase(RestTestCase):
     def test_membership_public_room_perms(self):
         room = self.created_public_rmid
         # get membership of self, get membership of other, public room + invite
-        # expect all 200s - public rooms, you can see who is in them.
+        # expect 403
         yield self.invite(room=room, src=self.rmcreator_id,
                           targ=self.user_id)
         yield self._test_get_membership(
             members=[self.user_id, self.rmcreator_id],
-            room=room, expect_code=200)
+            room=room, expect_code=403)
 
         # get membership of self, get membership of other, public room + joined
         # expect all 200s
@@ -315,11 +341,11 @@ class RoomPermissionsTestCase(RestTestCase):
             room=room, expect_code=200)
 
         # get membership of self, get membership of other, public room + left
-        # expect all 200s - public rooms, you can always see who is in them.
+        # expect 403.
         yield self.leave(room=room, user=self.user_id)
         yield self._test_get_membership(
             members=[self.user_id, self.rmcreator_id],
-            room=room, expect_code=200)
+            room=room, expect_code=403)
 
     @defer.inlineCallbacks
     def test_invited_permissions(self):
@@ -381,45 +407,55 @@ class RoomPermissionsTestCase(RestTestCase):
         # set [invite/join/left] of self, set [invite/join/left] of other,
         # expect all 403s
         for usr in [self.user_id, self.rmcreator_id]:
-            yield self.change_membership(room=room, src=self.user_id,
-                                     targ=usr,
-                                     membership=Membership.INVITE,
-                                     expect_code=403)
-            yield self.change_membership(room=room, src=self.user_id,
-                                     targ=usr,
-                                     membership=Membership.JOIN,
-                                     expect_code=403)
-            yield self.change_membership(room=room, src=self.user_id,
-                                     targ=usr,
-                                     membership=Membership.LEAVE,
-                                     expect_code=403)
+            yield self.change_membership(
+                room=room,
+                src=self.user_id,
+                targ=usr,
+                membership=Membership.INVITE,
+                expect_code=403
+            )
+
+            yield self.change_membership(
+                room=room,
+                src=self.user_id,
+                targ=usr,
+                membership=Membership.JOIN,
+                expect_code=403
+            )
+
+        # It is always valid to LEAVE if you've already left (currently.)
+        yield self.change_membership(
+            room=room,
+            src=self.user_id,
+            targ=self.rmcreator_id,
+            membership=Membership.LEAVE,
+            expect_code=403
+        )
 
 
 class RoomsMemberListTestCase(RestTestCase):
     """ Tests /rooms/$room_id/members/list REST events."""
     user_id = "@sid1:red"
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
-        persistence_service = Mock(spec=["get_latest_pdus_in_context"])
-        persistence_service.get_latest_pdus_in_context.return_value = []
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
 
         hs = HomeServer(
             "red",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
-            datastore=MemoryDataStore(),
             replication_layer=Mock(),
-            state_handler=state_handler,
-            persistence_service=persistence_service,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -436,6 +472,10 @@ class RoomsMemberListTestCase(RestTestCase):
             }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
+        def _insert_client_ip(*args, **kwargs):
+            return defer.succeed(None)
+        hs.get_datastore().insert_client_ip = _insert_client_ip
+
         synapse.rest.room.register_servlets(hs, self.mock_resource)
 
     def tearDown(self):
@@ -487,28 +527,26 @@ class RoomsCreateTestCase(RestTestCase):
     """ Tests /rooms and /rooms/$room_id REST events. """
     user_id = "@sid1:red"
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
-        persistence_service = Mock(spec=["get_latest_pdus_in_context"])
-        persistence_service.get_latest_pdus_in_context.return_value = []
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
 
         hs = HomeServer(
             "red",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
-            datastore=MemoryDataStore(),
             replication_layer=Mock(),
-            state_handler=state_handler,
-            persistence_service=persistence_service,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -523,6 +561,10 @@ class RoomsCreateTestCase(RestTestCase):
             }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
+        def _insert_client_ip(*args, **kwargs):
+            return defer.succeed(None)
+        hs.get_datastore().insert_client_ip = _insert_client_ip
+
         synapse.rest.room.register_servlets(hs, self.mock_resource)
 
     def tearDown(self):
@@ -592,24 +634,21 @@ class RoomTopicTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
-        persistence_service = Mock(spec=["get_latest_pdus_in_context"])
-        persistence_service.get_latest_pdus_in_context.return_value = []
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
 
         hs = HomeServer(
             "red",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
-            datastore=MemoryDataStore(),
             replication_layer=Mock(),
-            state_handler=state_handler,
-            persistence_service=persistence_service,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -622,13 +661,18 @@ class RoomTopicTestCase(RestTestCase):
                 "admin": False,
                 "device_id": None,
             }
+
         hs.get_auth().get_user_by_token = _get_user_by_token
 
+        def _insert_client_ip(*args, **kwargs):
+            return defer.succeed(None)
+        hs.get_datastore().insert_client_ip = _insert_client_ip
+
         synapse.rest.room.register_servlets(hs, self.mock_resource)
 
         # create the room
         self.room_id = yield self.create_room_as(self.user_id)
-        self.path = "/rooms/%s/state/m.room.topic" % self.room_id
+        self.path = "/rooms/%s/state/m.room.topic" % (self.room_id,)
 
     def tearDown(self):
         pass
@@ -706,24 +750,21 @@ class RoomMemberStateTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
-        persistence_service = Mock(spec=["get_latest_pdus_in_context"])
-        persistence_service.get_latest_pdus_in_context.return_value = []
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
 
         hs = HomeServer(
             "red",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
-            datastore=MemoryDataStore(),
             replication_layer=Mock(),
-            state_handler=state_handler,
-            persistence_service=persistence_service,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -736,13 +777,12 @@ class RoomMemberStateTestCase(RestTestCase):
                 "admin": False,
                 "device_id": None,
             }
-            return {
-                "user": hs.parse_userid(self.auth_user_id),
-                "admin": False,
-                "device_id": None,
-            }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
+        def _insert_client_ip(*args, **kwargs):
+            return defer.succeed(None)
+        hs.get_datastore().insert_client_ip = _insert_client_ip
+
         synapse.rest.room.register_servlets(hs, self.mock_resource)
 
         self.room_id = yield self.create_room_as(self.user_id)
@@ -847,24 +887,21 @@ class RoomMessagesTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        state_handler = Mock(spec=["handle_new_event"])
-        state_handler.handle_new_event.return_value = True
+        self.mock_config = NonCallableMock()
+        self.mock_config.signing_key = [MockKey()]
 
-        persistence_service = Mock(spec=["get_latest_pdus_in_context"])
-        persistence_service.get_latest_pdus_in_context.return_value = []
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
 
         hs = HomeServer(
             "red",
-            db_pool=None,
+            db_pool=db_pool,
             http_client=None,
-            datastore=MemoryDataStore(),
             replication_layer=Mock(),
-            state_handler=state_handler,
-            persistence_service=persistence_service,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=NonCallableMock(),
+            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -879,6 +916,10 @@ class RoomMessagesTestCase(RestTestCase):
             }
         hs.get_auth().get_user_by_token = _get_user_by_token
 
+        def _insert_client_ip(*args, **kwargs):
+            return defer.succeed(None)
+        hs.get_datastore().insert_client_ip = _insert_client_ip
+
         synapse.rest.room.register_servlets(hs, self.mock_resource)
 
         self.room_id = yield self.create_room_as(self.user_id)