Merge branch 'develop' of github.com:matrix-org/synapse into erikj/fix_get_missing_events_error

7 files changed, 539 insertions, 269 deletions

diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py
index 5b2105bc76..917548bb31 100644
--- a/tests/handlers/test_directory.py
+++ b/tests/handlers/test_directory.py
@@ -115,11 +115,7 @@ class TestCreateAliasACL(unittest.HomeserverTestCase):
         # We cheekily override the config to add custom alias creation rules
         config = {}
         config["alias_creation_rules"] = [
-            {
-                "user_id": "*",
-                "alias": "#unofficial_*",
-                "action": "allow",
-            }
+            {"user_id": "*", "alias": "#unofficial_*", "action": "allow"}
         ]
         config["room_list_publication_rules"] = []
 
@@ -162,9 +158,7 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase):
         room_id = self.helper.create_room_as(self.user_id)
 
         request, channel = self.make_request(
-            "PUT",
-            b"directory/list/room/%s" % (room_id.encode('ascii'),),
-            b'{}',
+            "PUT", b"directory/list/room/%s" % (room_id.encode('ascii'),), b'{}'
         )
         self.render(request)
         self.assertEquals(200, channel.code, channel.result)
@@ -179,10 +173,7 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase):
         self.directory_handler.enable_room_list_search = True
 
         # Room list is enabled so we should get some results
-        request, channel = self.make_request(
-            "GET",
-            b"publicRooms",
-        )
+        request, channel = self.make_request("GET", b"publicRooms")
         self.render(request)
         self.assertEquals(200, channel.code, channel.result)
         self.assertTrue(len(channel.json_body["chunk"]) > 0)
@@ -191,10 +182,7 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase):
         self.directory_handler.enable_room_list_search = False
 
         # Room list disabled so we should get no results
-        request, channel = self.make_request(
-            "GET",
-            b"publicRooms",
-        )
+        request, channel = self.make_request("GET", b"publicRooms")
         self.render(request)
         self.assertEquals(200, channel.code, channel.result)
         self.assertTrue(len(channel.json_body["chunk"]) == 0)
@@ -202,9 +190,7 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase):
         # Room list disabled so we shouldn't be allowed to publish rooms
         room_id = self.helper.create_room_as(self.user_id)
         request, channel = self.make_request(
-            "PUT",
-            b"directory/list/room/%s" % (room_id.encode('ascii'),),
-            b'{}',
+            "PUT", b"directory/list/room/%s" % (room_id.encode('ascii'),), b'{}'
         )
         self.render(request)
         self.assertEquals(403, channel.code, channel.result)
diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py
index 1c49bbbc3c..2e72a1dd23 100644
--- a/tests/handlers/test_e2e_room_keys.py
+++ b/tests/handlers/test_e2e_room_keys.py
@@ -36,7 +36,7 @@ room_keys = {
                     "first_message_index": 1,
                     "forwarded_count": 1,
                     "is_verified": False,
-                    "session_data": "SSBBTSBBIEZJU0gK"
+                    "session_data": "SSBBTSBBIEZJU0gK",
                 }
             }
         }
@@ -47,15 +47,13 @@ room_keys = {
 class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs)
-        self.hs = None       # type: synapse.server.HomeServer
+        self.hs = None  # type: synapse.server.HomeServer
         self.handler = None  # type: synapse.handlers.e2e_keys.E2eRoomKeysHandler
 
     @defer.inlineCallbacks
     def setUp(self):
         self.hs = yield utils.setup_test_homeserver(
-            self.addCleanup,
-            handlers=None,
-            replication_layer=mock.Mock(),
+            self.addCleanup, handlers=None, replication_layer=mock.Mock()
         )
         self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs)
         self.local_user = "@boris:" + self.hs.hostname
@@ -88,67 +86,86 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_create_version(self):
         """Check that we can create and then retrieve versions.
         """
-        res = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        res = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(res, "1")
 
         # check we can retrieve it as the current version
         res = yield self.handler.get_version_info(self.local_user)
-        self.assertDictEqual(res, {
-            "version": "1",
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        self.assertDictEqual(
+            res,
+            {
+                "version": "1",
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "first_version_auth_data",
+            },
+        )
 
         # check we can retrieve it as a specific version
         res = yield self.handler.get_version_info(self.local_user, "1")
-        self.assertDictEqual(res, {
-            "version": "1",
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        self.assertDictEqual(
+            res,
+            {
+                "version": "1",
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "first_version_auth_data",
+            },
+        )
 
         # upload a new one...
-        res = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "second_version_auth_data",
-        })
+        res = yield self.handler.create_version(
+            self.local_user,
+            {
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "second_version_auth_data",
+            },
+        )
         self.assertEqual(res, "2")
 
         # check we can retrieve it as the current version
         res = yield self.handler.get_version_info(self.local_user)
-        self.assertDictEqual(res, {
-            "version": "2",
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "second_version_auth_data",
-        })
+        self.assertDictEqual(
+            res,
+            {
+                "version": "2",
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "second_version_auth_data",
+            },
+        )
 
     @defer.inlineCallbacks
     def test_update_version(self):
         """Check that we can update versions.
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
-        res = yield self.handler.update_version(self.local_user, version, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "revised_first_version_auth_data",
-            "version": version
-        })
+        res = yield self.handler.update_version(
+            self.local_user,
+            version,
+            {
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "revised_first_version_auth_data",
+                "version": version,
+            },
+        )
         self.assertDictEqual(res, {})
 
         # check we can retrieve it as the current version
         res = yield self.handler.get_version_info(self.local_user)
-        self.assertDictEqual(res, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "revised_first_version_auth_data",
-            "version": version
-        })
+        self.assertDictEqual(
+            res,
+            {
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "revised_first_version_auth_data",
+                "version": version,
+            },
+        )
 
     @defer.inlineCallbacks
     def test_update_missing_version(self):
@@ -156,11 +173,15 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
         """
         res = None
         try:
-            yield self.handler.update_version(self.local_user, "1", {
-                "algorithm": "m.megolm_backup.v1",
-                "auth_data": "revised_first_version_auth_data",
-                "version": "1"
-            })
+            yield self.handler.update_version(
+                self.local_user,
+                "1",
+                {
+                    "algorithm": "m.megolm_backup.v1",
+                    "auth_data": "revised_first_version_auth_data",
+                    "version": "1",
+                },
+            )
         except errors.SynapseError as e:
             res = e.code
         self.assertEqual(res, 404)
@@ -170,29 +191,37 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
         """Check that we get a 400 if the version in the body is missing or
         doesn't match
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
         res = None
         try:
-            yield self.handler.update_version(self.local_user, version, {
-                "algorithm": "m.megolm_backup.v1",
-                "auth_data": "revised_first_version_auth_data"
-            })
+            yield self.handler.update_version(
+                self.local_user,
+                version,
+                {
+                    "algorithm": "m.megolm_backup.v1",
+                    "auth_data": "revised_first_version_auth_data",
+                },
+            )
         except errors.SynapseError as e:
             res = e.code
         self.assertEqual(res, 400)
 
         res = None
         try:
-            yield self.handler.update_version(self.local_user, version, {
-                "algorithm": "m.megolm_backup.v1",
-                "auth_data": "revised_first_version_auth_data",
-                "version": "incorrect"
-            })
+            yield self.handler.update_version(
+                self.local_user,
+                version,
+                {
+                    "algorithm": "m.megolm_backup.v1",
+                    "auth_data": "revised_first_version_auth_data",
+                    "version": "incorrect",
+                },
+            )
         except errors.SynapseError as e:
             res = e.code
         self.assertEqual(res, 400)
@@ -223,10 +252,10 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_delete_version(self):
         """Check that we can create and then delete versions.
         """
-        res = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        res = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(res, "1")
 
         # check we can delete it
@@ -255,16 +284,14 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_get_missing_room_keys(self):
         """Check we get an empty response from an empty backup
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
         res = yield self.handler.get_room_keys(self.local_user, version)
-        self.assertDictEqual(res, {
-            "rooms": {}
-        })
+        self.assertDictEqual(res, {"rooms": {}})
 
     # TODO: test the locking semantics when uploading room_keys,
     # although this is probably best done in sytest
@@ -275,7 +302,9 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
         """
         res = None
         try:
-            yield self.handler.upload_room_keys(self.local_user, "no_version", room_keys)
+            yield self.handler.upload_room_keys(
+                self.local_user, "no_version", room_keys
+            )
         except errors.SynapseError as e:
             res = e.code
         self.assertEqual(res, 404)
@@ -285,10 +314,10 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
         """Check that we get a 404 on uploading keys when an nonexistent version
         is specified
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
         res = None
@@ -304,16 +333,19 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_upload_room_keys_wrong_version(self):
         """Check that we get a 403 on uploading keys for an old version
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "second_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {
+                "algorithm": "m.megolm_backup.v1",
+                "auth_data": "second_version_auth_data",
+            },
+        )
         self.assertEqual(version, "2")
 
         res = None
@@ -327,10 +359,10 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_upload_room_keys_insert(self):
         """Check that we can insert and retrieve keys for a session
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
         yield self.handler.upload_room_keys(self.local_user, version, room_keys)
@@ -340,18 +372,13 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
 
         # check getting room_keys for a given room
         res = yield self.handler.get_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org"
+            self.local_user, version, room_id="!abc:matrix.org"
         )
         self.assertDictEqual(res, room_keys)
 
         # check getting room_keys for a given session_id
         res = yield self.handler.get_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org",
-            session_id="c0ff33",
+            self.local_user, version, room_id="!abc:matrix.org", session_id="c0ff33"
         )
         self.assertDictEqual(res, room_keys)
 
@@ -359,10 +386,10 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_upload_room_keys_merge(self):
         """Check that we can upload a new room_key for an existing session and
         have it correctly merged"""
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
         yield self.handler.upload_room_keys(self.local_user, version, room_keys)
@@ -378,7 +405,7 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
         res = yield self.handler.get_room_keys(self.local_user, version)
         self.assertEqual(
             res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'],
-            "SSBBTSBBIEZJU0gK"
+            "SSBBTSBBIEZJU0gK",
         )
 
         # test that marking the session as verified however /does/ replace it
@@ -387,8 +414,7 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
 
         res = yield self.handler.get_room_keys(self.local_user, version)
         self.assertEqual(
-            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'],
-            "new"
+            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], "new"
         )
 
         # test that a session with a higher forwarded_count doesn't replace one
@@ -399,8 +425,7 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
 
         res = yield self.handler.get_room_keys(self.local_user, version)
         self.assertEqual(
-            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'],
-            "new"
+            res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], "new"
         )
 
         # TODO: check edge cases as well as the common variations here
@@ -409,56 +434,36 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase):
     def test_delete_room_keys(self):
         """Check that we can insert and delete keys for a session
         """
-        version = yield self.handler.create_version(self.local_user, {
-            "algorithm": "m.megolm_backup.v1",
-            "auth_data": "first_version_auth_data",
-        })
+        version = yield self.handler.create_version(
+            self.local_user,
+            {"algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data"},
+        )
         self.assertEqual(version, "1")
 
         # check for bulk-delete
         yield self.handler.upload_room_keys(self.local_user, version, room_keys)
         yield self.handler.delete_room_keys(self.local_user, version)
         res = yield self.handler.get_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org",
-            session_id="c0ff33",
+            self.local_user, version, room_id="!abc:matrix.org", session_id="c0ff33"
         )
-        self.assertDictEqual(res, {
-            "rooms": {}
-        })
+        self.assertDictEqual(res, {"rooms": {}})
 
         # check for bulk-delete per room
         yield self.handler.upload_room_keys(self.local_user, version, room_keys)
         yield self.handler.delete_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org",
+            self.local_user, version, room_id="!abc:matrix.org"
         )
         res = yield self.handler.get_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org",
-            session_id="c0ff33",
+            self.local_user, version, room_id="!abc:matrix.org", session_id="c0ff33"
         )
-        self.assertDictEqual(res, {
-            "rooms": {}
-        })
+        self.assertDictEqual(res, {"rooms": {}})
 
         # check for bulk-delete per session
         yield self.handler.upload_room_keys(self.local_user, version, room_keys)
         yield self.handler.delete_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org",
-            session_id="c0ff33",
+            self.local_user, version, room_id="!abc:matrix.org", session_id="c0ff33"
         )
         res = yield self.handler.get_room_keys(
-            self.local_user,
-            version,
-            room_id="!abc:matrix.org",
-            session_id="c0ff33",
+            self.local_user, version, room_id="!abc:matrix.org", session_id="c0ff33"
         )
-        self.assertDictEqual(res, {
-            "rooms": {}
-        })
+        self.assertDictEqual(res, {"rooms": {}})
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index 94c6080e34..f70c6e7d65 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -424,8 +424,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
 
     def make_homeserver(self, reactor, clock):
         hs = self.setup_test_homeserver(
-            "server", http_client=None,
-            federation_sender=Mock(),
+            "server", http_client=None, federation_sender=Mock()
         )
         return hs
 
@@ -457,7 +456,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
 
         # Mark test2 as online, test will be offline with a last_active of 0
         self.presence_handler.set_state(
-            UserID.from_string("@test2:server"), {"presence": PresenceState.ONLINE},
+            UserID.from_string("@test2:server"), {"presence": PresenceState.ONLINE}
         )
         self.reactor.pump([0])  # Wait for presence updates to be handled
 
@@ -506,13 +505,13 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
 
         # Mark test as online
         self.presence_handler.set_state(
-            UserID.from_string("@test:server"), {"presence": PresenceState.ONLINE},
+            UserID.from_string("@test:server"), {"presence": PresenceState.ONLINE}
         )
 
         # Mark test2 as online, test will be offline with a last_active of 0.
         # Note we don't join them to the room yet
         self.presence_handler.set_state(
-            UserID.from_string("@test2:server"), {"presence": PresenceState.ONLINE},
+            UserID.from_string("@test2:server"), {"presence": PresenceState.ONLINE}
         )
 
         # Add servers to the room
@@ -541,8 +540,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
         )
         self.assertEqual(expected_state.state, PresenceState.ONLINE)
         self.federation_sender.send_presence_to_destinations.assert_called_once_with(
-            destinations=set(("server2", "server3")),
-            states=[expected_state]
+            destinations=set(("server2", "server3")), states=[expected_state]
         )
 
     def _add_new_user(self, room_id, user_id):
@@ -565,7 +563,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase):
             type=EventTypes.Member,
             sender=user_id,
             state_key=user_id,
-            content={"membership": Membership.JOIN}
+            content={"membership": Membership.JOIN},
         )
 
         prev_event_ids = self.get_success(
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index 017ea0385e..5ffba2ca7a 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -37,8 +37,12 @@ class RegistrationTestCase(unittest.HomeserverTestCase):
         hs_config = self.default_config("test")
 
         # some of the tests rely on us having a user consent version
-        hs_config.user_consent_version = "test_consent_version"
-        hs_config.max_mau_value = 50
+        hs_config["user_consent"] = {
+            "version": "test_consent_version",
+            "template_dir": ".",
+        }
+        hs_config["max_mau_value"] = 50
+        hs_config["limit_usage_by_mau"] = True
 
         hs = self.setup_test_homeserver(config=hs_config, expire_access_token=True)
         return hs
@@ -224,3 +228,10 @@ class RegistrationTestCase(unittest.HomeserverTestCase):
     def test_register_not_support_user(self):
         res = self.get_success(self.handler.register(localpart='user'))
         self.assertFalse(self.store.is_support_user(res[0]))
+
+    def test_invalid_user_id_length(self):
+        invalid_user_id = "x" * 256
+        self.get_failure(
+            self.handler.register(localpart=invalid_user_id),
+            SynapseError
+        )
diff --git a/tests/handlers/test_stats.py b/tests/handlers/test_stats.py
new file mode 100644
index 0000000000..2710c991cf
--- /dev/null
+++ b/tests/handlers/test_stats.py
@@ -0,0 +1,307 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mock import Mock
+
+from twisted.internet import defer
+
+from synapse.api.constants import EventTypes, Membership
+from synapse.rest import admin
+from synapse.rest.client.v1 import login, room
+
+from tests import unittest
+
+
+class StatsRoomTests(unittest.HomeserverTestCase):
+
+    servlets = [
+        admin.register_servlets_for_client_rest_resource,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor, clock, hs):
+
+        self.store = hs.get_datastore()
+        self.handler = self.hs.get_stats_handler()
+
+    def _add_background_updates(self):
+        """
+        Add the background updates we need to run.
+        """
+        # Ugh, have to reset this flag
+        self.store._all_done = False
+
+        self.get_success(
+            self.store._simple_insert(
+                "background_updates",
+                {"update_name": "populate_stats_createtables", "progress_json": "{}"},
+            )
+        )
+        self.get_success(
+            self.store._simple_insert(
+                "background_updates",
+                {
+                    "update_name": "populate_stats_process_rooms",
+                    "progress_json": "{}",
+                    "depends_on": "populate_stats_createtables",
+                },
+            )
+        )
+        self.get_success(
+            self.store._simple_insert(
+                "background_updates",
+                {
+                    "update_name": "populate_stats_cleanup",
+                    "progress_json": "{}",
+                    "depends_on": "populate_stats_process_rooms",
+                },
+            )
+        )
+
+    def test_initial_room(self):
+        """
+        The background updates will build the table from scratch.
+        """
+        r = self.get_success(self.store.get_all_room_state())
+        self.assertEqual(len(r), 0)
+
+        # Disable stats
+        self.hs.config.stats_enabled = False
+        self.handler.stats_enabled = False
+
+        u1 = self.register_user("u1", "pass")
+        u1_token = self.login("u1", "pass")
+
+        room_1 = self.helper.create_room_as(u1, tok=u1_token)
+        self.helper.send_state(
+            room_1, event_type="m.room.topic", body={"topic": "foo"}, tok=u1_token
+        )
+
+        # Stats disabled, shouldn't have done anything
+        r = self.get_success(self.store.get_all_room_state())
+        self.assertEqual(len(r), 0)
+
+        # Enable stats
+        self.hs.config.stats_enabled = True
+        self.handler.stats_enabled = True
+
+        # Do the initial population of the user directory via the background update
+        self._add_background_updates()
+
+        while not self.get_success(self.store.has_completed_background_updates()):
+            self.get_success(self.store.do_next_background_update(100), by=0.1)
+
+        r = self.get_success(self.store.get_all_room_state())
+
+        self.assertEqual(len(r), 1)
+        self.assertEqual(r[0]["topic"], "foo")
+
+    def test_initial_earliest_token(self):
+        """
+        Ingestion via notify_new_event will ignore tokens that the background
+        update have already processed.
+        """
+        self.reactor.advance(86401)
+
+        self.hs.config.stats_enabled = False
+        self.handler.stats_enabled = False
+
+        u1 = self.register_user("u1", "pass")
+        u1_token = self.login("u1", "pass")
+
+        u2 = self.register_user("u2", "pass")
+        u2_token = self.login("u2", "pass")
+
+        u3 = self.register_user("u3", "pass")
+        u3_token = self.login("u3", "pass")
+
+        room_1 = self.helper.create_room_as(u1, tok=u1_token)
+        self.helper.send_state(
+            room_1, event_type="m.room.topic", body={"topic": "foo"}, tok=u1_token
+        )
+
+        # Begin the ingestion by creating the temp tables. This will also store
+        # the position that the deltas should begin at, once they take over.
+        self.hs.config.stats_enabled = True
+        self.handler.stats_enabled = True
+        self.store._all_done = False
+        self.get_success(self.store.update_stats_stream_pos(None))
+
+        self.get_success(
+            self.store._simple_insert(
+                "background_updates",
+                {"update_name": "populate_stats_createtables", "progress_json": "{}"},
+            )
+        )
+
+        while not self.get_success(self.store.has_completed_background_updates()):
+            self.get_success(self.store.do_next_background_update(100), by=0.1)
+
+        # Now, before the table is actually ingested, add some more events.
+        self.helper.invite(room=room_1, src=u1, targ=u2, tok=u1_token)
+        self.helper.join(room=room_1, user=u2, tok=u2_token)
+
+        # Now do the initial ingestion.
+        self.get_success(
+            self.store._simple_insert(
+                "background_updates",
+                {"update_name": "populate_stats_process_rooms", "progress_json": "{}"},
+            )
+        )
+        self.get_success(
+            self.store._simple_insert(
+                "background_updates",
+                {
+                    "update_name": "populate_stats_cleanup",
+                    "progress_json": "{}",
+                    "depends_on": "populate_stats_process_rooms",
+                },
+            )
+        )
+
+        self.store._all_done = False
+        while not self.get_success(self.store.has_completed_background_updates()):
+            self.get_success(self.store.do_next_background_update(100), by=0.1)
+
+        self.reactor.advance(86401)
+
+        # Now add some more events, triggering ingestion. Because of the stream
+        # position being set to before the events sent in the middle, a simpler
+        # implementation would reprocess those events, and say there were four
+        # users, not three.
+        self.helper.invite(room=room_1, src=u1, targ=u3, tok=u1_token)
+        self.helper.join(room=room_1, user=u3, tok=u3_token)
+
+        # Get the deltas! There should be two -- day 1, and day 2.
+        r = self.get_success(self.store.get_deltas_for_room(room_1, 0))
+
+        # The oldest has 2 joined members
+        self.assertEqual(r[-1]["joined_members"], 2)
+
+        # The newest has 3
+        self.assertEqual(r[0]["joined_members"], 3)
+
+    def test_incorrect_state_transition(self):
+        """
+        If the state transition is not one of (JOIN, INVITE, LEAVE, BAN) to
+        (JOIN, INVITE, LEAVE, BAN), an error is raised.
+        """
+        events = {
+            "a1": {"membership": Membership.LEAVE},
+            "a2": {"membership": "not a real thing"},
+        }
+
+        def get_event(event_id, allow_none=True):
+            m = Mock()
+            m.content = events[event_id]
+            d = defer.Deferred()
+            self.reactor.callLater(0.0, d.callback, m)
+            return d
+
+        def get_received_ts(event_id):
+            return defer.succeed(1)
+
+        self.store.get_received_ts = get_received_ts
+        self.store.get_event = get_event
+
+        deltas = [
+            {
+                "type": EventTypes.Member,
+                "state_key": "some_user",
+                "room_id": "room",
+                "event_id": "a1",
+                "prev_event_id": "a2",
+                "stream_id": 60,
+            }
+        ]
+
+        f = self.get_failure(self.handler._handle_deltas(deltas), ValueError)
+        self.assertEqual(
+            f.value.args[0], "'not a real thing' is not a valid prev_membership"
+        )
+
+        # And the other way...
+        deltas = [
+            {
+                "type": EventTypes.Member,
+                "state_key": "some_user",
+                "room_id": "room",
+                "event_id": "a2",
+                "prev_event_id": "a1",
+                "stream_id": 100,
+            }
+        ]
+
+        f = self.get_failure(self.handler._handle_deltas(deltas), ValueError)
+        self.assertEqual(
+            f.value.args[0], "'not a real thing' is not a valid membership"
+        )
+
+    def test_redacted_prev_event(self):
+        """
+        If the prev_event does not exist, then it is assumed to be a LEAVE.
+        """
+        u1 = self.register_user("u1", "pass")
+        u1_token = self.login("u1", "pass")
+
+        room_1 = self.helper.create_room_as(u1, tok=u1_token)
+
+        # Do the initial population of the user directory via the background update
+        self._add_background_updates()
+
+        while not self.get_success(self.store.has_completed_background_updates()):
+            self.get_success(self.store.do_next_background_update(100), by=0.1)
+
+        events = {
+            "a1": None,
+            "a2": {"membership": Membership.JOIN},
+        }
+
+        def get_event(event_id, allow_none=True):
+            if events.get(event_id):
+                m = Mock()
+                m.content = events[event_id]
+            else:
+                m = None
+            d = defer.Deferred()
+            self.reactor.callLater(0.0, d.callback, m)
+            return d
+
+        def get_received_ts(event_id):
+            return defer.succeed(1)
+
+        self.store.get_received_ts = get_received_ts
+        self.store.get_event = get_event
+
+        deltas = [
+            {
+                "type": EventTypes.Member,
+                "state_key": "some_user:test",
+                "room_id": room_1,
+                "event_id": "a2",
+                "prev_event_id": "a1",
+                "stream_id": 100,
+            }
+        ]
+
+        # Handle our fake deltas, which has a user going from LEAVE -> JOIN.
+        self.get_success(self.handler._handle_deltas(deltas))
+
+        # One delta, with two joined members -- the room creator, and our fake
+        # user.
+        r = self.get_success(self.store.get_deltas_for_room(room_1, 0))
+        self.assertEqual(len(r), 1)
+        self.assertEqual(r[0]["joined_members"], 2)
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index 5a0b6c201c..cb8b4d2913 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -64,20 +64,22 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
         mock_federation_client.put_json.return_value = defer.succeed((200, "OK"))
 
         hs = self.setup_test_homeserver(
-            datastore=(Mock(
-                spec=[
-                    # Bits that Federation needs
-                    "prep_send_transaction",
-                    "delivered_txn",
-                    "get_received_txn_response",
-                    "set_received_txn_response",
-                    "get_destination_retry_timings",
-                    "get_devices_by_remote",
-                    # Bits that user_directory needs
-                    "get_user_directory_stream_pos",
-                    "get_current_state_deltas",
-                ]
-            )),
+            datastore=(
+                Mock(
+                    spec=[
+                        # Bits that Federation needs
+                        "prep_send_transaction",
+                        "delivered_txn",
+                        "get_received_txn_response",
+                        "set_received_txn_response",
+                        "get_destination_retry_timings",
+                        "get_devices_by_remote",
+                        # Bits that user_directory needs
+                        "get_user_directory_stream_pos",
+                        "get_current_state_deltas",
+                    ]
+                )
+            ),
             notifier=Mock(),
             http_client=mock_federation_client,
             keyring=mock_keyring,
@@ -87,7 +89,7 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
 
     def prepare(self, reactor, clock, hs):
         # the tests assume that we are starting at unix time 1000
-        reactor.pump((1000, ))
+        reactor.pump((1000,))
 
         mock_notifier = hs.get_notifier()
         self.on_new_event = mock_notifier.on_new_event
@@ -114,6 +116,7 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
         def check_joined_room(room_id, user_id):
             if user_id not in [u.to_string() for u in self.room_members]:
                 raise AuthError(401, "User is not in the room")
+
         hs.get_auth().check_joined_room = check_joined_room
 
         def get_joined_hosts_for_room(room_id):
@@ -123,6 +126,7 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
 
         def get_current_users_in_room(room_id):
             return set(str(u) for u in self.room_members)
+
         hs.get_state_handler().get_current_users_in_room = get_current_users_in_room
 
         self.datastore.get_user_directory_stream_pos.return_value = (
@@ -141,21 +145,16 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
 
         self.assertEquals(self.event_source.get_current_key(), 0)
 
-        self.successResultOf(self.handler.started_typing(
-            target_user=U_APPLE,
-            auth_user=U_APPLE,
-            room_id=ROOM_ID,
-            timeout=20000,
-        ))
-
-        self.on_new_event.assert_has_calls(
-            [call('typing_key', 1, rooms=[ROOM_ID])]
+        self.successResultOf(
+            self.handler.started_typing(
+                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=20000
+            )
         )
 
+        self.on_new_event.assert_has_calls([call('typing_key', 1, rooms=[ROOM_ID])])
+
         self.assertEquals(self.event_source.get_current_key(), 1)
-        events = self.event_source.get_new_events(
-            room_ids=[ROOM_ID], from_key=0
-        )
+        events = self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
         self.assertEquals(
             events[0],
             [
@@ -170,12 +169,11 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
     def test_started_typing_remote_send(self):
         self.room_members = [U_APPLE, U_ONION]
 
-        self.successResultOf(self.handler.started_typing(
-            target_user=U_APPLE,
-            auth_user=U_APPLE,
-            room_id=ROOM_ID,
-            timeout=20000,
-        ))
+        self.successResultOf(
+            self.handler.started_typing(
+                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=20000
+            )
+        )
 
         put_json = self.hs.get_http_client().put_json
         put_json.assert_called_once_with(
@@ -216,14 +214,10 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
         self.render(request)
         self.assertEqual(channel.code, 200)
 
-        self.on_new_event.assert_has_calls(
-            [call('typing_key', 1, rooms=[ROOM_ID])]
-        )
+        self.on_new_event.assert_has_calls([call('typing_key', 1, rooms=[ROOM_ID])])
 
         self.assertEquals(self.event_source.get_current_key(), 1)
-        events = self.event_source.get_new_events(
-            room_ids=[ROOM_ID], from_key=0
-        )
+        events = self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
         self.assertEquals(
             events[0],
             [
@@ -247,14 +241,14 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
 
         self.assertEquals(self.event_source.get_current_key(), 0)
 
-        self.successResultOf(self.handler.stopped_typing(
-            target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID
-        ))
-
-        self.on_new_event.assert_has_calls(
-            [call('typing_key', 1, rooms=[ROOM_ID])]
+        self.successResultOf(
+            self.handler.stopped_typing(
+                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID
+            )
         )
 
+        self.on_new_event.assert_has_calls([call('typing_key', 1, rooms=[ROOM_ID])])
+
         put_json = self.hs.get_http_client().put_json
         put_json.assert_called_once_with(
             "farm",
@@ -274,18 +268,10 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
         )
 
         self.assertEquals(self.event_source.get_current_key(), 1)
-        events = self.event_source.get_new_events(
-            room_ids=[ROOM_ID], from_key=0
-        )
+        events = self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
         self.assertEquals(
             events[0],
-            [
-                {
-                    "type": "m.typing",
-                    "room_id": ROOM_ID,
-                    "content": {"user_ids": []},
-                }
-            ],
+            [{"type": "m.typing", "room_id": ROOM_ID, "content": {"user_ids": []}}],
         )
 
     def test_typing_timeout(self):
@@ -293,22 +279,17 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
 
         self.assertEquals(self.event_source.get_current_key(), 0)
 
-        self.successResultOf(self.handler.started_typing(
-            target_user=U_APPLE,
-            auth_user=U_APPLE,
-            room_id=ROOM_ID,
-            timeout=10000,
-        ))
-
-        self.on_new_event.assert_has_calls(
-            [call('typing_key', 1, rooms=[ROOM_ID])]
+        self.successResultOf(
+            self.handler.started_typing(
+                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=10000
+            )
         )
+
+        self.on_new_event.assert_has_calls([call('typing_key', 1, rooms=[ROOM_ID])])
         self.on_new_event.reset_mock()
 
         self.assertEquals(self.event_source.get_current_key(), 1)
-        events = self.event_source.get_new_events(
-            room_ids=[ROOM_ID], from_key=0
-        )
+        events = self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
         self.assertEquals(
             events[0],
             [
@@ -320,45 +301,30 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase):
             ],
         )
 
-        self.reactor.pump([16, ])
+        self.reactor.pump([16])
 
-        self.on_new_event.assert_has_calls(
-            [call('typing_key', 2, rooms=[ROOM_ID])]
-        )
+        self.on_new_event.assert_has_calls([call('typing_key', 2, rooms=[ROOM_ID])])
 
         self.assertEquals(self.event_source.get_current_key(), 2)
-        events = self.event_source.get_new_events(
-            room_ids=[ROOM_ID], from_key=1
-        )
+        events = self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=1)
         self.assertEquals(
             events[0],
-            [
-                {
-                    "type": "m.typing",
-                    "room_id": ROOM_ID,
-                    "content": {"user_ids": []},
-                }
-            ],
+            [{"type": "m.typing", "room_id": ROOM_ID, "content": {"user_ids": []}}],
         )
 
         # SYN-230 - see if we can still set after timeout
 
-        self.successResultOf(self.handler.started_typing(
-            target_user=U_APPLE,
-            auth_user=U_APPLE,
-            room_id=ROOM_ID,
-            timeout=10000,
-        ))
-
-        self.on_new_event.assert_has_calls(
-            [call('typing_key', 3, rooms=[ROOM_ID])]
+        self.successResultOf(
+            self.handler.started_typing(
+                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=10000
+            )
         )
+
+        self.on_new_event.assert_has_calls([call('typing_key', 3, rooms=[ROOM_ID])])
         self.on_new_event.reset_mock()
 
         self.assertEquals(self.event_source.get_current_key(), 3)
-        events = self.event_source.get_new_events(
-            room_ids=[ROOM_ID], from_key=0
-        )
+        events = self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
         self.assertEquals(
             events[0],
             [
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index f1d0aa42b6..9021e647fe 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -14,8 +14,9 @@
 # limitations under the License.
 from mock import Mock
 
+import synapse.rest.admin
 from synapse.api.constants import UserTypes
-from synapse.rest.client.v1 import admin, login, room
+from synapse.rest.client.v1 import login, room
 from synapse.rest.client.v2_alpha import user_directory
 from synapse.storage.roommember import ProfileInfo
 
@@ -29,14 +30,14 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
 
     servlets = [
         login.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
         room.register_servlets,
     ]
 
     def make_homeserver(self, reactor, clock):
 
         config = self.default_config()
-        config.update_user_directory = True
+        config["update_user_directory"] = True
         return self.setup_test_homeserver(config=config)
 
     def prepare(self, reactor, clock, hs):
@@ -327,12 +328,12 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase):
         user_directory.register_servlets,
         room.register_servlets,
         login.register_servlets,
-        admin.register_servlets,
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
     ]
 
     def make_homeserver(self, reactor, clock):
         config = self.default_config()
-        config.update_user_directory = True
+        config["update_user_directory"] = True
         hs = self.setup_test_homeserver(config=config)
 
         self.config = hs.config
@@ -351,9 +352,7 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase):
 
         # Assert user directory is not empty
         request, channel = self.make_request(
-            "POST",
-            b"user_directory/search",
-            b'{"search_term":"user2"}',
+            "POST", b"user_directory/search", b'{"search_term":"user2"}'
         )
         self.render(request)
         self.assertEquals(200, channel.code, channel.result)
@@ -362,9 +361,7 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase):
         # Disable user directory and check search returns nothing
         self.config.user_directory_search_enabled = False
         request, channel = self.make_request(
-            "POST",
-            b"user_directory/search",
-            b'{"search_term":"user2"}',
+            "POST", b"user_directory/search", b'{"search_term":"user2"}'
         )
         self.render(request)
         self.assertEquals(200, channel.code, channel.result)