Merge branch 'develop' of github.com:matrix-org/synapse into erikj-perf

12 files changed, 282 insertions, 113 deletions

diff --git a/tests/storage/TESTS_NEEDED_FOR b/tests/storage/TESTS_NEEDED_FOR
deleted file mode 100644
index 8e5d0cbdc4..0000000000
--- a/tests/storage/TESTS_NEEDED_FOR
+++ /dev/null
@@ -1,5 +0,0 @@
-synapse/storage/feedback.py
-synapse/storage/keys.py
-synapse/storage/pdu.py
-synapse/storage/stream.py
-synapse/storage/transactions.py
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
new file mode 100644
index 0000000000..55d22f665a
--- /dev/null
+++ b/tests/storage/test__base.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from tests import unittest
+from twisted.internet import defer
+
+from synapse.storage._base import cached
+
+
+class CacheDecoratorTestCase(unittest.TestCase):
+
+    @defer.inlineCallbacks
+    def test_passthrough(self):
+        @cached()
+        def func(self, key):
+            return key
+
+        self.assertEquals((yield func(self, "foo")), "foo")
+        self.assertEquals((yield func(self, "bar")), "bar")
+
+    @defer.inlineCallbacks
+    def test_hit(self):
+        callcount = [0]
+
+        @cached()
+        def func(self, key):
+            callcount[0] += 1
+            return key
+
+        yield func(self, "foo")
+
+        self.assertEquals(callcount[0], 1)
+
+        self.assertEquals((yield func(self, "foo")), "foo")
+        self.assertEquals(callcount[0], 1)
+
+    @defer.inlineCallbacks
+    def test_invalidate(self):
+        callcount = [0]
+
+        @cached()
+        def func(self, key):
+            callcount[0] += 1
+            return key
+
+        yield func(self, "foo")
+
+        self.assertEquals(callcount[0], 1)
+
+        func.invalidate("foo")
+
+        yield func(self, "foo")
+
+        self.assertEquals(callcount[0], 2)
+
+    def test_invalidate_missing(self):
+        @cached()
+        def func(self, key):
+            return key
+
+        func.invalidate("what")
+
+    @defer.inlineCallbacks
+    def test_max_entries(self):
+        callcount = [0]
+
+        @cached(max_entries=10)
+        def func(self, key):
+            callcount[0] += 1
+            return key
+
+        for k in range(0,12):
+            yield func(self, k)
+
+        self.assertEquals(callcount[0], 12)
+
+        # There must have been at least 2 evictions, meaning if we calculate
+        # all 12 values again, we must get called at least 2 more times
+        for k in range(0,12):
+            yield func(self, k)
+
+        self.assertTrue(callcount[0] >= 14,
+            msg="Expected callcount >= 14, got %d" % (callcount[0]))
+
+    @defer.inlineCallbacks
+    def test_prefill(self):
+        callcount = [0]
+
+        @cached()
+        def func(self, key):
+            callcount[0] += 1
+            return key
+
+        func.prefill("foo", 123)
+
+        self.assertEquals((yield func(self, "foo")), 123)
+        self.assertEquals(callcount[0], 0)
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
new file mode 100644
index 0000000000..ca5b92ec85
--- /dev/null
+++ b/tests/storage/test_appservice.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from tests import unittest
+from twisted.internet import defer
+
+from synapse.appservice import ApplicationService
+from synapse.server import HomeServer
+from synapse.storage.appservice import ApplicationServiceStore
+
+from mock import Mock
+from tests.utils import SQLiteMemoryDbPool, MockClock
+
+
+class ApplicationServiceStoreTestCase(unittest.TestCase):
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
+        hs = HomeServer(
+            "test", db_pool=db_pool, clock=MockClock(), config=Mock()
+        )
+        self.as_token = "token1"
+        db_pool.runQuery(
+            "INSERT INTO application_services(token) VALUES(?)",
+            (self.as_token,)
+        )
+        db_pool.runQuery(
+            "INSERT INTO application_services(token) VALUES(?)", ("token2",)
+        )
+        db_pool.runQuery(
+            "INSERT INTO application_services(token) VALUES(?)", ("token3",)
+        )
+        # must be done after inserts
+        self.store = ApplicationServiceStore(hs)
+
+    @defer.inlineCallbacks
+    def test_update_and_retrieval_of_service(self):
+        url = "https://matrix.org/appservices/foobar"
+        hs_token = "hstok"
+        user_regex = [
+            {"regex": "@foobar_.*:matrix.org", "exclusive": True}
+        ]
+        alias_regex = [
+            {"regex": "#foobar_.*:matrix.org", "exclusive": False}
+        ]
+        room_regex = [
+
+        ]
+        service = ApplicationService(
+            url=url, hs_token=hs_token, token=self.as_token, namespaces={
+                ApplicationService.NS_USERS: user_regex,
+                ApplicationService.NS_ALIASES: alias_regex,
+                ApplicationService.NS_ROOMS: room_regex
+        })
+        yield self.store.update_app_service(service)
+
+        stored_service = yield self.store.get_app_service_by_token(
+            self.as_token
+        )
+        self.assertEquals(stored_service.token, self.as_token)
+        self.assertEquals(stored_service.url, url)
+        self.assertEquals(
+            stored_service.namespaces[ApplicationService.NS_ALIASES],
+            alias_regex
+        )
+        self.assertEquals(
+            stored_service.namespaces[ApplicationService.NS_ROOMS],
+            room_regex
+        )
+        self.assertEquals(
+            stored_service.namespaces[ApplicationService.NS_USERS],
+            user_regex
+        )
+
+    @defer.inlineCallbacks
+    def test_retrieve_unknown_service_token(self):
+        service = yield self.store.get_app_service_by_token("invalid_token")
+        self.assertEquals(service, None)
+
+    @defer.inlineCallbacks
+    def test_retrieval_of_service(self):
+        stored_service = yield self.store.get_app_service_by_token(
+            self.as_token
+        )
+        self.assertEquals(stored_service.token, self.as_token)
+        self.assertEquals(stored_service.url, None)
+        self.assertEquals(
+            stored_service.namespaces[ApplicationService.NS_ALIASES],
+            []
+        )
+        self.assertEquals(
+            stored_service.namespaces[ApplicationService.NS_ROOMS],
+            []
+        )
+        self.assertEquals(
+            stored_service.namespaces[ApplicationService.NS_USERS],
+            []
+        )
+
+    @defer.inlineCallbacks
+    def test_retrieval_of_all_services(self):
+        services = yield self.store.get_app_services()
+        self.assertEquals(len(services), 3)
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index a6f1d6a333..55fbffa7a2 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -22,6 +22,7 @@ from mock import Mock, call
 from collections import OrderedDict
 
 from synapse.server import HomeServer
+
 from synapse.storage._base import SQLBaseStore
 
 
@@ -37,8 +38,9 @@ class SQLBaseStoreTestCase(unittest.TestCase):
             return defer.succeed(func(self.mock_txn, *args, **kwargs))
         self.db_pool.runInteraction = runInteraction
 
-        hs = HomeServer("test",
-                db_pool=self.db_pool)
+        config = Mock()
+        config.event_cache_size = 1
+        hs = HomeServer("test", db_pool=self.db_pool, config=config)
 
         self.datastore = SQLBaseStore(hs)
 
diff --git a/tests/storage/test_directory.py b/tests/storage/test_directory.py
index e9c242cc07..b9bfbc00e2 100644
--- a/tests/storage/test_directory.py
+++ b/tests/storage/test_directory.py
@@ -17,28 +17,22 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.directory import DirectoryStore
+from synapse.types import RoomID, RoomAlias
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class DirectoryStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-        )
+        hs = yield setup_test_homeserver()
 
         self.store = DirectoryStore(hs)
 
-        self.room = hs.parse_roomid("!abcde:test")
-        self.alias = hs.parse_roomalias("#my-room:test")
+        self.room = RoomID.from_string("!abcde:test")
+        self.alias = RoomAlias.from_string("#my-room:test")
 
     @defer.inlineCallbacks
     def test_room_to_alias(self):
diff --git a/tests/storage/test_presence.py b/tests/storage/test_presence.py
index 9655d3cf42..065eebdbcf 100644
--- a/tests/storage/test_presence.py
+++ b/tests/storage/test_presence.py
@@ -17,28 +17,22 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.presence import PresenceStore
+from synapse.types import UserID
 
-from tests.utils import SQLiteMemoryDbPool, MockClock
+from tests.utils import setup_test_homeserver, MockClock
 
 
 class PresenceStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            clock=MockClock(),
-            db_pool=db_pool,
-        )
+        hs = yield setup_test_homeserver(clock=MockClock())
 
         self.store = PresenceStore(hs)
 
-        self.u_apple = hs.parse_userid("@apple:test")
-        self.u_banana = hs.parse_userid("@banana:test")
+        self.u_apple = UserID.from_string("@apple:test")
+        self.u_banana = UserID.from_string("@banana:test")
 
     @defer.inlineCallbacks
     def test_state(self):
diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py
index 5d36723c28..1fa783f313 100644
--- a/tests/storage/test_profile.py
+++ b/tests/storage/test_profile.py
@@ -17,26 +17,21 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.profile import ProfileStore
+from synapse.types import UserID
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class ProfileStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
+        hs = yield setup_test_homeserver()
 
         self.store = ProfileStore(hs)
 
-        self.u_frank = hs.parse_userid("@frank:test")
+        self.u_frank = UserID.from_string("@frank:test")
 
     @defer.inlineCallbacks
     def test_displayname(self):
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 9806fbc69b..b57006fcb4 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -17,10 +17,10 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes, Membership
+from synapse.types import UserID, RoomID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 from mock import Mock
 
@@ -29,16 +29,7 @@ class RedactionTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-            config=self.mock_config,
+        hs = yield setup_test_homeserver(
             resource_for_federation=Mock(),
             http_client=None,
         )
@@ -48,10 +39,10 @@ class RedactionTestCase(unittest.TestCase):
         self.handlers = hs.get_handlers()
         self.message_handler = self.handlers.message_handler
 
-        self.u_alice = hs.parse_userid("@alice:test")
-        self.u_bob = hs.parse_userid("@bob:test")
+        self.u_alice = UserID.from_string("@alice:test")
+        self.u_bob = UserID.from_string("@bob:test")
 
-        self.room1 = hs.parse_roomid("!abc123:test")
+        self.room1 = RoomID.from_string("!abc123:test")
 
         self.depth = 1
 
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index 84bfde7568..e0b81f2b57 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -17,22 +17,16 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.registration import RegistrationStore
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class RegistrationStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
+        hs = yield setup_test_homeserver()
 
         self.store = RegistrationStore(hs)
 
@@ -53,7 +47,10 @@ class RegistrationStoreTestCase(unittest.TestCase):
         )
 
         self.assertEquals(
-            {"admin": 0, "device_id": None, "name": self.user_id},
+            {"admin": 0,
+             "device_id": None,
+             "name": self.user_id,
+             "token_id": 1},
             (yield self.store.get_user_by_token(self.tokens[0]))
         )
 
@@ -63,7 +60,10 @@ class RegistrationStoreTestCase(unittest.TestCase):
         yield self.store.add_access_token_to_user(self.user_id, self.tokens[1])
 
         self.assertEquals(
-            {"admin": 0, "device_id": None, "name": self.user_id},
+            {"admin": 0,
+             "device_id": None,
+             "name": self.user_id,
+             "token_id": 2},
             (yield self.store.get_user_by_token(self.tokens[1]))
         )
 
diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py
index e7739776ec..c88dd446fb 100644
--- a/tests/storage/test_room.py
+++ b/tests/storage/test_room.py
@@ -17,30 +17,25 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes
+from synapse.types import UserID, RoomID, RoomAlias
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class RoomStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
+        hs = yield setup_test_homeserver()
 
         # We can't test RoomStore on its own without the DirectoryStore, for
         # management of the 'room_aliases' table
         self.store = hs.get_datastore()
 
-        self.room = hs.parse_roomid("!abcde:test")
-        self.alias = hs.parse_roomalias("#a-room-name:test")
-        self.u_creator = hs.parse_userid("@creator:test")
+        self.room = RoomID.from_string("!abcde:test")
+        self.alias = RoomAlias.from_string("#a-room-name:test")
+        self.u_creator = UserID.from_string("@creator:test")
 
         yield self.store.store_room(self.room.to_string(),
             room_creator_user_id=self.u_creator.to_string(),
@@ -85,19 +80,14 @@ class RoomEventsStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
+        hs = setup_test_homeserver()
 
         # Room events need the full datastore, for persist_event() and
         # get_room_state()
         self.store = hs.get_datastore()
         self.event_factory = hs.get_event_factory();
 
-        self.room = hs.parse_roomid("!abcde:test")
+        self.room = RoomID.from_string("!abcde:test")
 
         yield self.store.store_room(self.room.to_string(),
             room_creator_user_id="@creator:text",
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index a23a8189df..811fea544b 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -17,10 +17,10 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes, Membership
+from synapse.types import UserID, RoomID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 from mock import Mock
 
@@ -29,16 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-            config=self.mock_config,
+        hs = yield setup_test_homeserver(
             resource_for_federation=Mock(),
             http_client=None,
         )
@@ -49,13 +40,13 @@ class RoomMemberStoreTestCase(unittest.TestCase):
         self.handlers = hs.get_handlers()
         self.message_handler = self.handlers.message_handler
 
-        self.u_alice = hs.parse_userid("@alice:test")
-        self.u_bob = hs.parse_userid("@bob:test")
+        self.u_alice = UserID.from_string("@alice:test")
+        self.u_bob = UserID.from_string("@bob:test")
 
         # User elsewhere on another host
-        self.u_charlie = hs.parse_userid("@charlie:elsewhere")
+        self.u_charlie = UserID.from_string("@charlie:elsewhere")
 
-        self.room = hs.parse_roomid("!abc123:test")
+        self.room = RoomID.from_string("!abc123:test")
 
     @defer.inlineCallbacks
     def inject_room_member(self, room, user, membership, replaces_state=None):
diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py
index 9247fc579e..0c9b89d765 100644
--- a/tests/storage/test_stream.py
+++ b/tests/storage/test_stream.py
@@ -17,10 +17,10 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes, Membership
+from synapse.types import UserID, RoomID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 from mock import Mock
 
@@ -29,16 +29,7 @@ class StreamStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-            config=self.mock_config,
+        hs = yield setup_test_homeserver(
             resource_for_federation=Mock(),
             http_client=None,
         )
@@ -48,11 +39,11 @@ class StreamStoreTestCase(unittest.TestCase):
         self.handlers = hs.get_handlers()
         self.message_handler = self.handlers.message_handler
 
-        self.u_alice = hs.parse_userid("@alice:test")
-        self.u_bob = hs.parse_userid("@bob:test")
+        self.u_alice = UserID.from_string("@alice:test")
+        self.u_bob = UserID.from_string("@bob:test")
 
-        self.room1 = hs.parse_roomid("!abc123:test")
-        self.room2 = hs.parse_roomid("!xyx987:test")
+        self.room1 = RoomID.from_string("!abc123:test")
+        self.room2 = RoomID.from_string("!xyx987:test")
 
         self.depth = 1