From ebc3db295bfe6d0c43bf45b8fcd7fa6bbc429375 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 19 Feb 2015 18:36:02 +0000 Subject: Take named arguments to @cached() decorator, add a 'max_entries' limit --- tests/storage/test__base.py | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/storage/test__base.py (limited to 'tests/storage') diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py new file mode 100644 index 0000000000..057f798640 --- /dev/null +++ b/tests/storage/test__base.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.storage._base import cached + + +class CacheDecoratorTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def test_passthrough(self): + @cached() + def func(self, key): + return key + + self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals((yield func(self, "bar")), "bar") + + @defer.inlineCallbacks + def test_hit(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + yield func(self, "foo") + + self.assertEquals(callcount[0], 1) + + self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals(callcount[0], 1) + + @defer.inlineCallbacks + def test_invalidate(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + yield func(self, "foo") + + self.assertEquals(callcount[0], 1) + + func.invalidate("foo") + + yield func(self, "foo") + + self.assertEquals(callcount[0], 2) + + @defer.inlineCallbacks + def test_max_entries(self): + callcount = [0] + + @cached(max_entries=10) + def func(self, key): + callcount[0] += 1 + return key + + for k in range(0,12): + yield func(self, k) + + self.assertEquals(callcount[0], 12) + + # There must have been at least 2 evictions, meaning if we calculate + # all 12 values again, we must get called at least 2 more times + for k in range(0,12): + yield func(self, k) + + self.assertTrue(callcount[0] >= 14, + msg="Expected callcount >= 14, got %d" % (callcount[0])) -- cgit 1.5.1 From e76d485e29498fce7412423e7a5b6ac6bc287ec3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 15:41:54 +0000 Subject: Allow @cached-wrapped functions to have a prefill method for setting entries --- synapse/storage/_base.py | 23 +++++++++++++++-------- tests/storage/test__base.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) (limited to 'tests/storage') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 78ba5f25ea..4b1ec687c9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -48,24 +48,30 @@ def cached(max_entries=1000): The wrapped function has an additional member, a callable called "invalidate". This can be used to remove individual entries from the cache. + + The wrapped function has another additional callable, called "prefill", + which can be used to insert values into the cache specifically, without + calling the calculation function. """ def wrap(orig): cache = {} - @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: - defer.returnValue(cache[key]) - - ret = yield orig(self, key) - + def prefill(key, value): while len(cache) > max_entries: # TODO(paul): This feels too biased. However, a random index # would be a bit inefficient, walking the list of keys just # to ignore most of them? del cache[cache.keys()[0]] - cache[key] = ret; + cache[key] = value + + @defer.inlineCallbacks + def wrapped(self, key): + if key in cache: + defer.returnValue(cache[key]) + + ret = yield orig(self, key) + prefill(key, ret) defer.returnValue(ret) def invalidate(key): @@ -73,6 +79,7 @@ def cached(max_entries=1000): del cache[key] wrapped.invalidate = invalidate + wrapped.prefill = prefill return wrapped return wrap diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 057f798640..fb306cb784 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -87,3 +87,17 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertTrue(callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])) + + @defer.inlineCallbacks + def test_prefill(self): + callcount = [0] + + @cached() + def func(self, key): + callcount[0] += 1 + return key + + func.prefill("foo", 123) + + self.assertEquals((yield func(self, "foo")), 123) + self.assertEquals(callcount[0], 0) -- cgit 1.5.1 From f53fcbce9789186c1c42fe2f93ba46e3d8720b1b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Feb 2015 18:29:26 +0000 Subject: Use cache.pop() instead of a separate membership test + del [] --- synapse/storage/_base.py | 3 +-- tests/storage/test__base.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'tests/storage') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 42edb56c36..da698cb3b8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -75,8 +75,7 @@ def cached(max_entries=1000): defer.returnValue(ret) def invalidate(key): - if key in cache: - del cache[key] + cache.pop(key, None) wrapped.invalidate = invalidate wrapped.prefill = prefill diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index fb306cb784..55d22f665a 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -66,6 +66,13 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertEquals(callcount[0], 2) + def test_invalidate_missing(self): + @cached() + def func(self, key): + return key + + func.invalidate("what") + @defer.inlineCallbacks def test_max_entries(self): callcount = [0] -- cgit 1.5.1 From 127efeeb68a4ee6fe6c3bfe417a980878f6e165d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 11:10:48 +0000 Subject: Update unit tests to use new format. --- tests/appservice/test_appservice.py | 39 ++++++++++++++++++++++--------------- tests/storage/test_appservice.py | 12 +++++++++--- 2 files changed, 32 insertions(+), 19 deletions(-) (limited to 'tests/storage') diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index d12e4f2644..c1c2892eb1 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -18,6 +18,13 @@ from mock import Mock, PropertyMock from tests import unittest +def _regex(regex, exclusive=True): + return { + "regex": regex, + exclusive: exclusive + } + + class ApplicationServiceTestCase(unittest.TestCase): def setUp(self): @@ -36,21 +43,21 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_user_id_prefix_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.assertTrue(self.service.is_interested(self.event)) def test_regex_user_id_prefix_no_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@someone_else:matrix.org" self.assertFalse(self.service.is_interested(self.event)) def test_regex_room_member_is_checked(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@someone_else:matrix.org" self.event.type = "m.room.member" @@ -59,21 +66,21 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_room_id_match(self): self.service.namespaces[ApplicationService.NS_ROOMS].append( - "!some_prefix.*some_suffix:matrix.org" + _regex("!some_prefix.*some_suffix:matrix.org") ) self.event.room_id = "!some_prefixs0m3th1nGsome_suffix:matrix.org" self.assertTrue(self.service.is_interested(self.event)) def test_regex_room_id_no_match(self): self.service.namespaces[ApplicationService.NS_ROOMS].append( - "!some_prefix.*some_suffix:matrix.org" + _regex("!some_prefix.*some_suffix:matrix.org") ) self.event.room_id = "!XqBunHwQIXUiqCaoxq:matrix.org" self.assertFalse(self.service.is_interested(self.event)) def test_regex_alias_match(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#irc_.*:matrix.org" + _regex("#irc_.*:matrix.org") ) self.assertTrue(self.service.is_interested( self.event, @@ -82,7 +89,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_alias_no_match(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#irc_.*:matrix.org" + _regex("#irc_.*:matrix.org") ) self.assertFalse(self.service.is_interested( self.event, @@ -91,10 +98,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_regex_multiple_matches(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#irc_.*:matrix.org" + _regex("#irc_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.assertTrue(self.service.is_interested( @@ -104,10 +111,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_restrict_to_rooms(self): self.service.namespaces[ApplicationService.NS_ROOMS].append( - "!flibble_.*:matrix.org" + _regex("!flibble_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.event.room_id = "!wibblewoo:matrix.org" @@ -118,10 +125,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_restrict_to_aliases(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#xmpp_.*:matrix.org" + _regex("#xmpp_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@irc_foobar:matrix.org" self.assertFalse(self.service.is_interested( @@ -132,10 +139,10 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_restrict_to_senders(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( - "#xmpp_.*:matrix.org" + _regex("#xmpp_.*:matrix.org") ) self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) self.event.sender = "@xmpp_foobar:matrix.org" self.assertFalse(self.service.is_interested( @@ -146,7 +153,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def test_member_list_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( - "@irc_.*" + _regex("@irc_.*") ) join_list = [ Mock( diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index fc733d4c79..9a646fa6eb 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -50,9 +50,15 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): def test_update_and_retrieval_of_service(self): url = "https://matrix.org/appservices/foobar" hs_token = "hstok" - user_regex = ["@foobar_.*:matrix.org"] - alias_regex = ["#foobar_.*:matrix.org"] - room_regex = [] + user_regex = [ + {"regex": "@foobar_.*:matrix.org", "exclusive": True} + ] + alias_regex = [ + {"regex": "#foobar_.*:matrix.org", "exclusive": True} + ] + room_regex = [ + + ] service = ApplicationService( url=url, hs_token=hs_token, token=self.as_token, namespaces={ ApplicationService.NS_USERS: user_regex, -- cgit 1.5.1 From de190e49d5711edb5503e5574296a3c2a02ca06c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 27 Feb 2015 11:51:06 +0000 Subject: Add more unit tests for exclusive namespaces. --- tests/appservice/test_appservice.py | 50 ++++++++++++++++++++++++++++++++++++- tests/storage/test_appservice.py | 2 +- 2 files changed, 50 insertions(+), 2 deletions(-) (limited to 'tests/storage') diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index c1c2892eb1..eb7becf725 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -21,7 +21,7 @@ from tests import unittest def _regex(regex, exclusive=True): return { "regex": regex, - exclusive: exclusive + "exclusive": exclusive } @@ -87,6 +87,54 @@ class ApplicationServiceTestCase(unittest.TestCase): aliases_for_event=["#irc_foobar:matrix.org", "#athing:matrix.org"] )) + def test_non_exclusive_alias(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + _regex("#irc_.*:matrix.org", exclusive=False) + ) + self.assertFalse(self.service.is_exclusive_alias( + "#irc_foobar:matrix.org" + )) + + def test_non_exclusive_room(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + _regex("!irc_.*:matrix.org", exclusive=False) + ) + self.assertFalse(self.service.is_exclusive_room( + "!irc_foobar:matrix.org" + )) + + def test_non_exclusive_user(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + _regex("@irc_.*:matrix.org", exclusive=False) + ) + self.assertFalse(self.service.is_exclusive_user( + "@irc_foobar:matrix.org" + )) + + def test_exclusive_alias(self): + self.service.namespaces[ApplicationService.NS_ALIASES].append( + _regex("#irc_.*:matrix.org", exclusive=True) + ) + self.assertTrue(self.service.is_exclusive_alias( + "#irc_foobar:matrix.org" + )) + + def test_exclusive_user(self): + self.service.namespaces[ApplicationService.NS_USERS].append( + _regex("@irc_.*:matrix.org", exclusive=True) + ) + self.assertTrue(self.service.is_exclusive_user( + "@irc_foobar:matrix.org" + )) + + def test_exclusive_room(self): + self.service.namespaces[ApplicationService.NS_ROOMS].append( + _regex("!irc_.*:matrix.org", exclusive=True) + ) + self.assertTrue(self.service.is_exclusive_room( + "!irc_foobar:matrix.org" + )) + def test_regex_alias_no_match(self): self.service.namespaces[ApplicationService.NS_ALIASES].append( _regex("#irc_.*:matrix.org") diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 9a646fa6eb..ca5b92ec85 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -54,7 +54,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): {"regex": "@foobar_.*:matrix.org", "exclusive": True} ] alias_regex = [ - {"regex": "#foobar_.*:matrix.org", "exclusive": True} + {"regex": "#foobar_.*:matrix.org", "exclusive": False} ] room_regex = [ -- cgit 1.5.1