Merge branch 'cross-signing_hidden' into cross-signing_keys

16 files changed, 279 insertions, 106 deletions

diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py
index 795703967d..c4f0bbd3dd 100644
--- a/tests/crypto/test_keyring.py
+++ b/tests/crypto/test_keyring.py
@@ -86,35 +86,6 @@ class KeyringTestCase(unittest.HomeserverTestCase):
             getattr(LoggingContext.current_context(), "request", None), expected
         )
 
-    def test_wait_for_previous_lookups(self):
-        kr = keyring.Keyring(self.hs)
-
-        lookup_1_deferred = defer.Deferred()
-        lookup_2_deferred = defer.Deferred()
-
-        # we run the lookup in a logcontext so that the patched inlineCallbacks can check
-        # it is doing the right thing with logcontexts.
-        wait_1_deferred = run_in_context(
-            kr.wait_for_previous_lookups, {"server1": lookup_1_deferred}
-        )
-
-        # there were no previous lookups, so the deferred should be ready
-        self.successResultOf(wait_1_deferred)
-
-        # set off another wait. It should block because the first lookup
-        # hasn't yet completed.
-        wait_2_deferred = run_in_context(
-            kr.wait_for_previous_lookups, {"server1": lookup_2_deferred}
-        )
-
-        self.assertFalse(wait_2_deferred.called)
-
-        # let the first lookup complete (in the sentinel context)
-        lookup_1_deferred.callback(None)
-
-        # now the second wait should complete.
-        self.successResultOf(wait_2_deferred)
-
     def test_verify_json_objects_for_server_awaits_previous_requests(self):
         key1 = signedjson.key.generate_signing_key(1)
 
@@ -136,7 +107,7 @@ class KeyringTestCase(unittest.HomeserverTestCase):
             self.assertEquals(LoggingContext.current_context().request, "11")
             with PreserveLoggingContext():
                 yield persp_deferred
-            defer.returnValue(persp_resp)
+            return persp_resp
 
         self.http_client.post_json.side_effect = get_perspectives
 
@@ -583,7 +554,7 @@ def run_in_context(f, *args, **kwargs):
         # logs.
         ctx.request = "testctx"
         rv = yield f(*args, **kwargs)
-    defer.returnValue(rv)
+    return rv
 
 
 def _verify_json_for_server(kr, *args):
@@ -594,6 +565,6 @@ def _verify_json_for_server(kr, *args):
     @defer.inlineCallbacks
     def v():
         rv1 = yield kr.verify_json_for_server(*args)
-        defer.returnValue(rv1)
+        return rv1
 
     return run_in_context(v)
diff --git a/tests/federation/test_complexity.py b/tests/federation/test_complexity.py
index a5b03005d7..51714a2b06 100644
--- a/tests/federation/test_complexity.py
+++ b/tests/federation/test_complexity.py
@@ -13,12 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from mock import Mock
+
 from twisted.internet import defer
 
+from synapse.api.errors import Codes, SynapseError
 from synapse.config.ratelimiting import FederationRateLimitConfig
 from synapse.federation.transport import server
 from synapse.rest import admin
 from synapse.rest.client.v1 import login, room
+from synapse.types import UserID
 from synapse.util.ratelimitutils import FederationRateLimiter
 
 from tests import unittest
@@ -33,9 +37,8 @@ class RoomComplexityTests(unittest.HomeserverTestCase):
     ]
 
     def default_config(self, name="test"):
-        config = super(RoomComplexityTests, self).default_config(name=name)
-        config["limit_large_remote_room_joins"] = True
-        config["limit_large_remote_room_complexity"] = 0.05
+        config = super().default_config(name=name)
+        config["limit_remote_rooms"] = {"enabled": True, "complexity": 0.05}
         return config
 
     def prepare(self, reactor, clock, homeserver):
@@ -88,3 +91,71 @@ class RoomComplexityTests(unittest.HomeserverTestCase):
         self.assertEquals(200, channel.code)
         complexity = channel.json_body["v1"]
         self.assertEqual(complexity, 1.23)
+
+    def test_join_too_large(self):
+
+        u1 = self.register_user("u1", "pass")
+
+        handler = self.hs.get_room_member_handler()
+        fed_transport = self.hs.get_federation_transport_client()
+
+        # Mock out some things, because we don't want to test the whole join
+        fed_transport.client.get_json = Mock(return_value=defer.succeed({"v1": 9999}))
+        handler.federation_handler.do_invite_join = Mock(return_value=defer.succeed(1))
+
+        d = handler._remote_join(
+            None,
+            ["otherserver.example"],
+            "roomid",
+            UserID.from_string(u1),
+            {"membership": "join"},
+        )
+
+        self.pump()
+
+        # The request failed with a SynapseError saying the resource limit was
+        # exceeded.
+        f = self.get_failure(d, SynapseError)
+        self.assertEqual(f.value.code, 400, f.value)
+        self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+    def test_join_too_large_once_joined(self):
+
+        u1 = self.register_user("u1", "pass")
+        u1_token = self.login("u1", "pass")
+
+        # Ok, this might seem a bit weird -- I want to test that we actually
+        # leave the room, but I don't want to simulate two servers. So, we make
+        # a local room, which we say we're joining remotely, even if there's no
+        # remote, because we mock that out. Then, we'll leave the (actually
+        # local) room, which will be propagated over federation in a real
+        # scenario.
+        room_1 = self.helper.create_room_as(u1, tok=u1_token)
+
+        handler = self.hs.get_room_member_handler()
+        fed_transport = self.hs.get_federation_transport_client()
+
+        # Mock out some things, because we don't want to test the whole join
+        fed_transport.client.get_json = Mock(return_value=defer.succeed(None))
+        handler.federation_handler.do_invite_join = Mock(return_value=defer.succeed(1))
+
+        # Artificially raise the complexity
+        self.hs.get_datastore().get_current_state_event_counts = lambda x: defer.succeed(
+            600
+        )
+
+        d = handler._remote_join(
+            None,
+            ["otherserver.example"],
+            room_1,
+            UserID.from_string(u1),
+            {"membership": "join"},
+        )
+
+        self.pump()
+
+        # The request failed with a SynapseError saying the resource limit was
+        # exceeded.
+        f = self.get_failure(d, SynapseError)
+        self.assertEqual(f.value.code, 400)
+        self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 90d0129374..0ad0a88165 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -44,7 +44,7 @@ class RegistrationTestCase(unittest.HomeserverTestCase):
         hs_config["max_mau_value"] = 50
         hs_config["limit_usage_by_mau"] = True
 
-        hs = self.setup_test_homeserver(config=hs_config, expire_access_token=True)
+        hs = self.setup_test_homeserver(config=hs_config)
         return hs
 
     def prepare(self, reactor, clock, hs):
@@ -283,4 +283,4 @@ class RegistrationTestCase(unittest.HomeserverTestCase):
                 user, requester, displayname, by_admin=True
             )
 
-        defer.returnValue((user_id, token))
+        return (user_id, token)
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index a49f9b3224..b906686b49 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -145,7 +145,7 @@ class MatrixFederationAgentTests(TestCase):
 
             try:
                 fetch_res = yield fetch_d
-                defer.returnValue(fetch_res)
+                return fetch_res
             except Exception as e:
                 logger.info("Fetch of %s failed: %s", uri.decode("ascii"), e)
                 raise
@@ -936,7 +936,7 @@ class MatrixFederationAgentTests(TestCase):
         except Exception as e:
             logger.warning("Error fetching well-known: %s", e)
             raise
-        defer.returnValue(result)
+        return result
 
     def test_well_known_cache(self):
         self.reactor.lookups["testserv"] = "1.2.3.4"
diff --git a/tests/http/federation/test_srv_resolver.py b/tests/http/federation/test_srv_resolver.py
index 65b51dc981..3b885ef64b 100644
--- a/tests/http/federation/test_srv_resolver.py
+++ b/tests/http/federation/test_srv_resolver.py
@@ -61,7 +61,7 @@ class SrvResolverTestCase(unittest.TestCase):
                 # should have restored our context
                 self.assertIs(LoggingContext.current_context(), ctx)
 
-                defer.returnValue(result)
+                return result
 
         test_d = do_lookup()
         self.assertNoResult(test_d)
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index b9d6d7ad1c..2b01f40a42 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -68,7 +68,7 @@ class FederationClientTests(HomeserverTestCase):
 
                 try:
                     fetch_res = yield fetch_d
-                    defer.returnValue(fetch_res)
+                    return fetch_res
                 finally:
                     check_logcontext(context)
 
diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py
index a8adc9a61d..a3d7e3c046 100644
--- a/tests/rest/client/test_transactions.py
+++ b/tests/rest/client/test_transactions.py
@@ -46,7 +46,7 @@ class HttpTransactionCacheTestCase(unittest.TestCase):
         @defer.inlineCallbacks
         def cb():
             yield Clock(reactor).sleep(0)
-            defer.returnValue("yay")
+            return "yay"
 
         @defer.inlineCallbacks
         def test():
diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py
index 984feb623f..cdf89e3383 100644
--- a/tests/server_notices/test_resource_limits_server_notices.py
+++ b/tests/server_notices/test_resource_limits_server_notices.py
@@ -36,7 +36,7 @@ class TestResourceLimitsServerNotices(unittest.HomeserverTestCase):
             "room_name": "Server Notices",
         }
 
-        hs = self.setup_test_homeserver(config=hs_config, expire_access_token=True)
+        hs = self.setup_test_homeserver(config=hs_config)
         return hs
 
     def prepare(self, reactor, clock, hs):
diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py
index fbb9302694..9fabe3fbc0 100644
--- a/tests/storage/test_background_update.py
+++ b/tests/storage/test_background_update.py
@@ -43,7 +43,7 @@ class BackgroundUpdateTestCase(unittest.TestCase):
                 "test_update",
                 progress,
             )
-            defer.returnValue(count)
+            return count
 
         self.update_handler.side_effect = update
 
@@ -60,7 +60,7 @@ class BackgroundUpdateTestCase(unittest.TestCase):
         @defer.inlineCallbacks
         def update(progress, count):
             yield self.store._end_background_update("test_update")
-            defer.returnValue(count)
+            return count
 
         self.update_handler.side_effect = update
         self.update_handler.reset_mock()
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 732a778fab..8488b6edc8 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,23 +17,21 @@
 
 from mock import Mock
 
-from twisted.internet import defer
-
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.room_versions import RoomVersions
 from synapse.types import RoomID, UserID
 
 from tests import unittest
-from tests.utils import create_room, setup_test_homeserver
+from tests.utils import create_room
 
 
-class RedactionTestCase(unittest.TestCase):
-    @defer.inlineCallbacks
-    def setUp(self):
-        hs = yield setup_test_homeserver(
-            self.addCleanup, resource_for_federation=Mock(), http_client=None
+class RedactionTestCase(unittest.HomeserverTestCase):
+    def make_homeserver(self, reactor, clock):
+        return self.setup_test_homeserver(
+            resource_for_federation=Mock(), http_client=None
         )
 
+    def prepare(self, reactor, clock, hs):
         self.store = hs.get_datastore()
         self.event_builder_factory = hs.get_event_builder_factory()
         self.event_creation_handler = hs.get_event_creation_handler()
@@ -42,11 +41,12 @@ class RedactionTestCase(unittest.TestCase):
 
         self.room1 = RoomID.from_string("!abc123:test")
 
-        yield create_room(hs, self.room1.to_string(), self.u_alice.to_string())
+        self.get_success(
+            create_room(hs, self.room1.to_string(), self.u_alice.to_string())
+        )
 
         self.depth = 1
 
-    @defer.inlineCallbacks
     def inject_room_member(
         self, room, user, membership, replaces_state=None, extra_content={}
     ):
@@ -63,15 +63,14 @@ class RedactionTestCase(unittest.TestCase):
             },
         )
 
-        event, context = yield self.event_creation_handler.create_new_client_event(
-            builder
+        event, context = self.get_success(
+            self.event_creation_handler.create_new_client_event(builder)
         )
 
-        yield self.store.persist_event(event, context)
+        self.get_success(self.store.persist_event(event, context))
 
-        defer.returnValue(event)
+        return event
 
-    @defer.inlineCallbacks
     def inject_message(self, room, user, body):
         self.depth += 1
 
@@ -86,15 +85,14 @@ class RedactionTestCase(unittest.TestCase):
             },
         )
 
-        event, context = yield self.event_creation_handler.create_new_client_event(
-            builder
+        event, context = self.get_success(
+            self.event_creation_handler.create_new_client_event(builder)
         )
 
-        yield self.store.persist_event(event, context)
+        self.get_success(self.store.persist_event(event, context))
 
-        defer.returnValue(event)
+        return event
 
-    @defer.inlineCallbacks
     def inject_redaction(self, room, event_id, user, reason):
         builder = self.event_builder_factory.for_room_version(
             RoomVersions.V1,
@@ -108,20 +106,21 @@ class RedactionTestCase(unittest.TestCase):
             },
         )
 
-        event, context = yield self.event_creation_handler.create_new_client_event(
-            builder
+        event, context = self.get_success(
+            self.event_creation_handler.create_new_client_event(builder)
         )
 
-        yield self.store.persist_event(event, context)
+        self.get_success(self.store.persist_event(event, context))
 
-    @defer.inlineCallbacks
     def test_redact(self):
-        yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
+        self.get_success(
+            self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
+        )
 
-        msg_event = yield self.inject_message(self.room1, self.u_alice, "t")
+        msg_event = self.get_success(self.inject_message(self.room1, self.u_alice, "t"))
 
         # Check event has not been redacted:
-        event = yield self.store.get_event(msg_event.event_id)
+        event = self.get_success(self.store.get_event(msg_event.event_id))
 
         self.assertObjectHasAttributes(
             {
@@ -136,11 +135,11 @@ class RedactionTestCase(unittest.TestCase):
 
         # Redact event
         reason = "Because I said so"
-        yield self.inject_redaction(
-            self.room1, msg_event.event_id, self.u_alice, reason
+        self.get_success(
+            self.inject_redaction(self.room1, msg_event.event_id, self.u_alice, reason)
         )
 
-        event = yield self.store.get_event(msg_event.event_id)
+        event = self.get_success(self.store.get_event(msg_event.event_id))
 
         self.assertEqual(msg_event.event_id, event.event_id)
 
@@ -164,15 +163,18 @@ class RedactionTestCase(unittest.TestCase):
             event.unsigned["redacted_because"],
         )
 
-    @defer.inlineCallbacks
     def test_redact_join(self):
-        yield self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
+        self.get_success(
+            self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
+        )
 
-        msg_event = yield self.inject_room_member(
-            self.room1, self.u_bob, Membership.JOIN, extra_content={"blue": "red"}
+        msg_event = self.get_success(
+            self.inject_room_member(
+                self.room1, self.u_bob, Membership.JOIN, extra_content={"blue": "red"}
+            )
         )
 
-        event = yield self.store.get_event(msg_event.event_id)
+        event = self.get_success(self.store.get_event(msg_event.event_id))
 
         self.assertObjectHasAttributes(
             {
@@ -187,13 +189,13 @@ class RedactionTestCase(unittest.TestCase):
 
         # Redact event
         reason = "Because I said so"
-        yield self.inject_redaction(
-            self.room1, msg_event.event_id, self.u_alice, reason
+        self.get_success(
+            self.inject_redaction(self.room1, msg_event.event_id, self.u_alice, reason)
         )
 
         # Check redaction
 
-        event = yield self.store.get_event(msg_event.event_id)
+        event = self.get_success(self.store.get_event(msg_event.event_id))
 
         self.assertTrue("redacted_because" in event.unsigned)
 
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 73ed943f5a..64cb294c37 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -20,7 +20,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.room_versions import RoomVersions
-from synapse.types import RoomID, UserID
+from synapse.types import Requester, RoomID, UserID
 
 from tests import unittest
 from tests.utils import create_room, setup_test_homeserver
@@ -67,7 +67,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
 
         yield self.store.persist_event(event, context)
 
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def test_one_member(self):
@@ -84,3 +84,38 @@ class RoomMemberStoreTestCase(unittest.TestCase):
                 )
             ],
         )
+
+
+class CurrentStateMembershipUpdateTestCase(unittest.HomeserverTestCase):
+    def prepare(self, reactor, clock, homeserver):
+        self.store = homeserver.get_datastore()
+        self.room_creator = homeserver.get_room_creation_handler()
+
+    def test_can_rerun_update(self):
+        # First make sure we have completed all updates.
+        while not self.get_success(self.store.has_completed_background_updates()):
+            self.get_success(self.store.do_next_background_update(100), by=0.1)
+
+        # Now let's create a room, which will insert a membership
+        user = UserID("alice", "test")
+        requester = Requester(user, None, False, None, None)
+        self.get_success(self.room_creator.create_room(requester, {}))
+
+        # Register the background update to run again.
+        self.get_success(
+            self.store._simple_insert(
+                table="background_updates",
+                values={
+                    "update_name": "current_state_events_membership",
+                    "progress_json": "{}",
+                    "depends_on": None,
+                },
+            )
+        )
+
+        # ... and tell the DataStore that it hasn't finished all updates yet
+        self.store._all_done = False
+
+        # Now let's actually drive the updates to completion
+        while not self.get_success(self.store.has_completed_background_updates()):
+            self.get_success(self.store.do_next_background_update(100), by=0.1)
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index 212a7ae765..5c2cf3c2db 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -65,7 +65,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
 
         yield self.store.persist_event(event, context)
 
-        defer.returnValue(event)
+        return event
 
     def assertStateMapEqual(self, s1, s2):
         for t in s1:
diff --git a/tests/test_server.py b/tests/test_server.py
index ba08483a4b..2a7d407c98 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -61,7 +61,10 @@ class JsonResourceTests(unittest.TestCase):
 
         res = JsonResource(self.homeserver)
         res.register_paths(
-            "GET", [re.compile("^/_matrix/foo/(?P<room_id>[^/]*)$")], _callback
+            "GET",
+            [re.compile("^/_matrix/foo/(?P<room_id>[^/]*)$")],
+            _callback,
+            "test_servlet",
         )
 
         request, channel = make_request(
@@ -82,7 +85,9 @@ class JsonResourceTests(unittest.TestCase):
             raise Exception("boo")
 
         res = JsonResource(self.homeserver)
-        res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
+        res.register_paths(
+            "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet"
+        )
 
         request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo")
         render(request, res, self.reactor)
@@ -105,7 +110,9 @@ class JsonResourceTests(unittest.TestCase):
             return make_deferred_yieldable(d)
 
         res = JsonResource(self.homeserver)
-        res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
+        res.register_paths(
+            "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet"
+        )
 
         request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo")
         render(request, res, self.reactor)
@@ -122,7 +129,9 @@ class JsonResourceTests(unittest.TestCase):
             raise SynapseError(403, "Forbidden!!one!", Codes.FORBIDDEN)
 
         res = JsonResource(self.homeserver)
-        res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
+        res.register_paths(
+            "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet"
+        )
 
         request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo")
         render(request, res, self.reactor)
@@ -143,7 +152,9 @@ class JsonResourceTests(unittest.TestCase):
             self.fail("shouldn't ever get here")
 
         res = JsonResource(self.homeserver)
-        res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback)
+        res.register_paths(
+            "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet"
+        )
 
         request, channel = make_request(self.reactor, b"GET", b"/_matrix/foobar")
         render(request, res, self.reactor)
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 118c3bd238..e0605dac2f 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -139,7 +139,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
             builder
         )
         yield self.hs.get_datastore().persist_event(event, context)
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def inject_room_member(self, user_id, membership="join", extra_content={}):
@@ -161,7 +161,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
         )
 
         yield self.hs.get_datastore().persist_event(event, context)
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def inject_message(self, user_id, content=None):
@@ -182,7 +182,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
         )
 
         yield self.hs.get_datastore().persist_event(event, context)
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def test_large_room(self):
diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py
index 7807328e2f..5713870f48 100644
--- a/tests/util/caches/test_descriptors.py
+++ b/tests/util/caches/test_descriptors.py
@@ -27,6 +27,7 @@ from synapse.logging.context import (
     make_deferred_yieldable,
 )
 from synapse.util.caches import descriptors
+from synapse.util.caches.descriptors import cached
 
 from tests import unittest
 
@@ -55,12 +56,15 @@ class CacheTestCase(unittest.TestCase):
         d2 = defer.Deferred()
         cache.set("key2", d2, partial(record_callback, 1))
 
-        # lookup should return the deferreds
-        self.assertIs(cache.get("key1"), d1)
-        self.assertIs(cache.get("key2"), d2)
+        # lookup should return observable deferreds
+        self.assertFalse(cache.get("key1").has_called())
+        self.assertFalse(cache.get("key2").has_called())
 
         # let one of the lookups complete
         d2.callback("result2")
+
+        # for now at least, the cache will return real results rather than an
+        # observabledeferred
         self.assertEqual(cache.get("key2"), "result2")
 
         # now do the invalidation
@@ -146,6 +150,28 @@ class DescriptorTestCase(unittest.TestCase):
         self.assertEqual(r, "chips")
         obj.mock.assert_not_called()
 
+    def test_cache_with_sync_exception(self):
+        """If the wrapped function throws synchronously, things should continue to work
+        """
+
+        class Cls(object):
+            @cached()
+            def fn(self, arg1):
+                raise SynapseError(100, "mai spoon iz too big!!1")
+
+        obj = Cls()
+
+        # this should fail immediately
+        d = obj.fn(1)
+        self.failureResultOf(d, SynapseError)
+
+        # ... leaving the cache empty
+        self.assertEqual(len(obj.fn.cache.cache), 0)
+
+        # and a second call should result in a second exception
+        d = obj.fn(1)
+        self.failureResultOf(d, SynapseError)
+
     def test_cache_logcontexts(self):
         """Check that logcontexts are set and restored correctly when
         using the cache."""
@@ -159,7 +185,7 @@ class DescriptorTestCase(unittest.TestCase):
                 def inner_fn():
                     with PreserveLoggingContext():
                         yield complete_lookup
-                    defer.returnValue(1)
+                    return 1
 
                 return inner_fn()
 
@@ -169,7 +195,7 @@ class DescriptorTestCase(unittest.TestCase):
                 c1.name = "c1"
                 r = yield obj.fn(1)
                 self.assertEqual(LoggingContext.current_context(), c1)
-            defer.returnValue(r)
+            return r
 
         def check_result(r):
             self.assertEqual(r, 1)
@@ -222,6 +248,9 @@ class DescriptorTestCase(unittest.TestCase):
 
                 self.assertEqual(LoggingContext.current_context(), c1)
 
+            # the cache should now be empty
+            self.assertEqual(len(obj.fn.cache.cache), 0)
+
         obj = Cls()
 
         # set off a deferred which will do a cache lookup
@@ -268,6 +297,61 @@ class DescriptorTestCase(unittest.TestCase):
         self.assertEqual(r, "chips")
         obj.mock.assert_not_called()
 
+    def test_cache_iterable(self):
+        class Cls(object):
+            def __init__(self):
+                self.mock = mock.Mock()
+
+            @descriptors.cached(iterable=True)
+            def fn(self, arg1, arg2):
+                return self.mock(arg1, arg2)
+
+        obj = Cls()
+
+        obj.mock.return_value = ["spam", "eggs"]
+        r = obj.fn(1, 2)
+        self.assertEqual(r, ["spam", "eggs"])
+        obj.mock.assert_called_once_with(1, 2)
+        obj.mock.reset_mock()
+
+        # a call with different params should call the mock again
+        obj.mock.return_value = ["chips"]
+        r = obj.fn(1, 3)
+        self.assertEqual(r, ["chips"])
+        obj.mock.assert_called_once_with(1, 3)
+        obj.mock.reset_mock()
+
+        # the two values should now be cached
+        self.assertEqual(len(obj.fn.cache.cache), 3)
+
+        r = obj.fn(1, 2)
+        self.assertEqual(r, ["spam", "eggs"])
+        r = obj.fn(1, 3)
+        self.assertEqual(r, ["chips"])
+        obj.mock.assert_not_called()
+
+    def test_cache_iterable_with_sync_exception(self):
+        """If the wrapped function throws synchronously, things should continue to work
+        """
+
+        class Cls(object):
+            @descriptors.cached(iterable=True)
+            def fn(self, arg1):
+                raise SynapseError(100, "mai spoon iz too big!!1")
+
+        obj = Cls()
+
+        # this should fail immediately
+        d = obj.fn(1)
+        self.failureResultOf(d, SynapseError)
+
+        # ... leaving the cache empty
+        self.assertEqual(len(obj.fn.cache.cache), 0)
+
+        # and a second call should result in a second exception
+        d = obj.fn(1)
+        self.failureResultOf(d, SynapseError)
+
 
 class CachedListDescriptorTestCase(unittest.TestCase):
     @defer.inlineCallbacks
@@ -286,7 +370,7 @@ class CachedListDescriptorTestCase(unittest.TestCase):
                 # we want this to behave like an asynchronous function
                 yield run_on_reactor()
                 assert LoggingContext.current_context().request == "c1"
-                defer.returnValue(self.mock(args1, arg2))
+                return self.mock(args1, arg2)
 
         with LoggingContext() as c1:
             c1.request = "c1"
@@ -334,7 +418,7 @@ class CachedListDescriptorTestCase(unittest.TestCase):
             def list_fn(self, args1, arg2):
                 # we want this to behave like an asynchronous function
                 yield run_on_reactor()
-                defer.returnValue(self.mock(args1, arg2))
+                return self.mock(args1, arg2)
 
         obj = Cls()
         invalidate0 = mock.Mock()
diff --git a/tests/utils.py b/tests/utils.py
index 8a94ce0b47..f1eb9a545c 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -126,7 +126,6 @@ def default_config(name, parse=False):
         "enable_registration": True,
         "enable_registration_captcha": False,
         "macaroon_secret_key": "not even a little secret",
-        "expire_access_token": False,
         "trusted_third_party_id_servers": [],
         "room_invite_state_types": [],
         "password_providers": [],
@@ -361,7 +360,7 @@ def setup_test_homeserver(
     if fed:
         register_federation_servlets(hs, fed)
 
-    defer.returnValue(hs)
+    return hs
 
 
 def register_federation_servlets(hs, resource):
@@ -465,13 +464,13 @@ class MockHttpResource(HttpServer):
                     args = [urlparse.unquote(u) for u in matcher.groups()]
 
                     (code, response) = yield func(mock_request, *args)
-                    defer.returnValue((code, response))
+                    return (code, response)
                 except CodeMessageException as e:
-                    defer.returnValue((e.code, cs_error(e.msg, code=e.errcode)))
+                    return (e.code, cs_error(e.msg, code=e.errcode))
 
         raise KeyError("No event can handle %s" % path)
 
-    def register_paths(self, method, path_patterns, callback):
+    def register_paths(self, method, path_patterns, callback, servlet_name):
         for path_pattern in path_patterns:
             self.callbacks.append((method, path_pattern, callback))