Move to integration tests

2 files changed, 116 insertions, 395 deletions

diff --git a/tests/handlers/test_sliding_sync.py b/tests/handlers/test_sliding_sync.py
index 5e91be0026..a7aa9bb8af 100644
--- a/tests/handlers/test_sliding_sync.py
+++ b/tests/handlers/test_sliding_sync.py
@@ -19,7 +19,7 @@
 #
 import logging
 from copy import deepcopy
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional
 from unittest.mock import patch
 
 from parameterized import parameterized
@@ -46,13 +46,7 @@ from synapse.rest import admin
 from synapse.rest.client import knock, login, room
 from synapse.server import HomeServer
 from synapse.storage.util.id_generators import MultiWriterIdGenerator
-from synapse.types import (
-    JsonDict,
-    SlidingSyncStreamToken,
-    StreamToken,
-    UserID,
-    create_requester,
-)
+from synapse.types import JsonDict, StreamToken, UserID
 from synapse.types.handlers import SlidingSyncConfig
 from synapse.util import Clock
 
@@ -3765,390 +3759,3 @@ class SortRoomsTestCase(HomeserverTestCase):
             # We only care about the *latest* event in the room.
             [room_id1, room_id2],
         )
-
-
-class GetRoomSyncDataTestCase(HomeserverTestCase):
-    """
-    Tests for Sliding Sync handler `get_room_sync_data()`
-    """
-
-    servlets = [
-        admin.register_servlets,
-        knock.register_servlets,
-        login.register_servlets,
-        room.register_servlets,
-    ]
-
-    def default_config(self) -> JsonDict:
-        config = super().default_config()
-        # Enable sliding sync
-        config["experimental_features"] = {"msc3575_enabled": True}
-        return config
-
-    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-        self.sliding_sync_handler = self.hs.get_sliding_sync_handler()
-        self.store = self.hs.get_datastores().main
-        self.event_sources = hs.get_event_sources()
-
-    def _create_sync_configs(
-        self,
-        user_id: str,
-        room_id: str,
-        timeline_limit: int = 5,
-    ) -> Tuple[SlidingSyncConfig, RoomSyncConfig, _RoomMembershipForUser]:
-        """Create the configs necessary to call `get_room_sync_data`"""
-        requester = create_requester(user_id, device_id="foo_device")
-        sync_config = SlidingSyncConfig(
-            user=requester.user,
-            requester=requester,
-            conn_id="conn_id",
-            lists={
-                "list": SlidingSyncConfig.SlidingSyncList(
-                    timeline_limit=timeline_limit,
-                    required_state=[
-                        (EventTypes.Name, ""),
-                    ],
-                ),
-            },
-            room_subscriptions={},
-            extensions=None,
-        )
-
-        room_sync_config = RoomSyncConfig(timeline_limit, {EventTypes.Name: {""}})
-
-        rooms = self.get_success(
-            self.store.get_rooms_for_local_user_where_membership_is(
-                user_id, membership_list=[Membership.JOIN]
-            )
-        )
-        room_for_user = rooms[0]
-        assert room_for_user.room_id == room_id
-
-        room_membership_for_user_at_to_token = _RoomMembershipForUser(
-            room_id=room_id,
-            event_id=room_for_user.event_id,
-            event_pos=room_for_user.event_pos,
-            membership=Membership.JOIN,
-            sender=user_id,
-            newly_joined=False,
-            newly_left=False,
-            is_dm=False,
-        )
-
-        return (sync_config, room_sync_config, room_membership_for_user_at_to_token)
-
-    def test_room_sync_data_initial(self) -> None:
-        """Tests getting room sync data with no from token"""
-
-        user1_id = self.register_user("user1", "pass")
-        user1_tok = self.login(user1_id, "pass")
-
-        room_id1 = self.helper.create_room_as(
-            user1_id,
-            tok=user1_tok,
-        )
-
-        sync_config, room_sync_config, room_membership_for_user_at_to_token = (
-            self._create_sync_configs(user1_id, room_id1, timeline_limit=5)
-        )
-
-        # Timeline limit is 5 so let's send 5 messages that we'll expect to get
-        # back.
-        expected_timeline = []
-        for _ in range(5):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        to_token = self.event_sources.get_current_token()
-
-        result = self.get_success(
-            self.sliding_sync_handler.get_room_sync_data(
-                sync_config,
-                room_id=room_id1,
-                room_sync_config=room_sync_config,
-                room_membership_for_user_at_to_token=room_membership_for_user_at_to_token,
-                from_token=None,
-                to_token=to_token,
-            )
-        )
-
-        self.assertTrue(result.initial)
-        self.assertTrue(result.limited)
-        self.assertEqual(
-            [e.event_id for e in result.timeline_events], expected_timeline
-        )
-
-    def test_room_sync_data_incremental_live(self) -> None:
-        """Test getting room data where we have previously sent down the room
-        and its state is considered LIVE"""
-        user1_id = self.register_user("user1", "pass")
-        user1_tok = self.login(user1_id, "pass")
-
-        room_id1 = self.helper.create_room_as(
-            user1_id,
-            tok=user1_tok,
-        )
-
-        sync_config, room_sync_config, room_membership_for_user_at_to_token = (
-            self._create_sync_configs(user1_id, room_id1)
-        )
-
-        # These messages are sent before the `from_token`, so we don't expect to
-        # see these messages.
-        for _ in range(5):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-
-        # Mark the room as having been sent down, and create an appropriate
-        # `from_token`.
-        connection_token = self.get_success(
-            self.sliding_sync_handler.connection_store.record_rooms(
-                sync_config, None, sent_room_ids=[room_id1], unsent_room_ids=[]
-            )
-        )
-        from_token = SlidingSyncStreamToken(
-            self.event_sources.get_current_token(), connection_token
-        )
-
-        # These messages are sent after the `from_token`, so we expect to only
-        # see these messages.
-        expected_timeline = []
-        for _ in range(2):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        to_token = self.event_sources.get_current_token()
-
-        result = self.get_success(
-            self.sliding_sync_handler.get_room_sync_data(
-                sync_config,
-                room_id=room_id1,
-                room_sync_config=room_sync_config,
-                room_membership_for_user_at_to_token=room_membership_for_user_at_to_token,
-                from_token=from_token,
-                to_token=to_token,
-            )
-        )
-
-        self.assertFalse(result.initial)
-        self.assertFalse(result.limited)
-        self.assertEqual(
-            [e.event_id for e in result.timeline_events], expected_timeline
-        )
-
-    def test_room_sync_data_incremental_previously_not_limited(self) -> None:
-        """Test getting room data where we have previously sent down the room,
-        but we missed sending down some data previously and so its state is
-        considered PREVIOUSLY.
-
-        In this case there has been more than timeline limit events.
-        """
-        user1_id = self.register_user("user1", "pass")
-        user1_tok = self.login(user1_id, "pass")
-
-        room_id1 = self.helper.create_room_as(
-            user1_id,
-            tok=user1_tok,
-        )
-
-        sync_config, room_sync_config, room_membership_for_user_at_to_token = (
-            self._create_sync_configs(user1_id, room_id1)
-        )
-
-        # These messages are sent before the initial `from_token`, so we don't
-        # expect to see these messages.
-        for _ in range(5):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-
-        # Mark the room as having been sent down, and create an appropriate
-        # `from_token`.
-        connection_token = self.get_success(
-            self.sliding_sync_handler.connection_store.record_rooms(
-                sync_config, None, sent_room_ids=[room_id1], unsent_room_ids=[]
-            )
-        )
-        from_token = SlidingSyncStreamToken(
-            self.event_sources.get_current_token(), connection_token
-        )
-
-        # These messages are sent after the initial `from_token`, so we expect
-        # to see these messages.
-        expected_timeline = []
-        for _ in range(2):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        # Mark the room as *not* having been sent down, and create a new
-        # `from_token`.
-        connection_token = self.get_success(
-            self.sliding_sync_handler.connection_store.record_rooms(
-                sync_config, from_token, sent_room_ids=[], unsent_room_ids=[room_id1]
-            )
-        )
-
-        from_token = SlidingSyncStreamToken(
-            self.event_sources.get_current_token(), connection_token
-        )
-
-        # We should also receive new messages
-        for _ in range(2):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        to_token = self.event_sources.get_current_token()
-
-        result = self.get_success(
-            self.sliding_sync_handler.get_room_sync_data(
-                sync_config,
-                room_id=room_id1,
-                room_sync_config=room_sync_config,
-                room_membership_for_user_at_to_token=room_membership_for_user_at_to_token,
-                from_token=from_token,
-                to_token=to_token,
-            )
-        )
-
-        self.assertFalse(result.initial)
-        self.assertFalse(result.limited)
-        self.assertEqual(
-            [e.event_id for e in result.timeline_events], expected_timeline
-        )
-
-    def test_room_sync_data_incremental_previously_limited(self) -> None:
-        """Test getting room data where we have previously sent down the room,
-        but we missed sending down some data previously and so its state is
-        considered PREVIOUSLY.
-
-        In this case there has been fewer than timeline limit events.
-        """
-        user1_id = self.register_user("user1", "pass")
-        user1_tok = self.login(user1_id, "pass")
-
-        room_id1 = self.helper.create_room_as(
-            user1_id,
-            tok=user1_tok,
-        )
-
-        sync_config, room_sync_config, room_membership_for_user_at_to_token = (
-            self._create_sync_configs(user1_id, room_id1)
-        )
-
-        # These messages are sent before the initial `from_token`, so we don't
-        # expect to see these messages.
-        for _ in range(5):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-
-        # Mark the room as having been sent down, and create an appropriate
-        # `from_token`.
-        connection_token = self.get_success(
-            self.sliding_sync_handler.connection_store.record_rooms(
-                sync_config, None, sent_room_ids=[room_id1], unsent_room_ids=[]
-            )
-        )
-        from_token = SlidingSyncStreamToken(
-            self.event_sources.get_current_token(), connection_token
-        )
-
-        # These messages are sent after the initial `from_token`, but are before
-        # the timeline limit, so we don't expect to see these messages.
-        for _ in range(5):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-
-        # ... but these messages are within the timeline limit, so we do expect
-        # to see them
-        expected_timeline = []
-        for _ in range(3):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        # Mark the room as *not* having been sent down, and create a new
-        # `from_token`.
-        connection_token = self.get_success(
-            self.sliding_sync_handler.connection_store.record_rooms(
-                sync_config, from_token, sent_room_ids=[], unsent_room_ids=[room_id1]
-            )
-        )
-
-        from_token = SlidingSyncStreamToken(
-            self.event_sources.get_current_token(), connection_token
-        )
-
-        # We should also receive new messages
-        for _ in range(2):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        to_token = self.event_sources.get_current_token()
-
-        result = self.get_success(
-            self.sliding_sync_handler.get_room_sync_data(
-                sync_config,
-                room_id=room_id1,
-                room_sync_config=room_sync_config,
-                room_membership_for_user_at_to_token=room_membership_for_user_at_to_token,
-                from_token=from_token,
-                to_token=to_token,
-            )
-        )
-
-        self.assertFalse(result.initial)
-        self.assertTrue(result.limited)
-        self.assertEqual(
-            [e.event_id for e in result.timeline_events], expected_timeline
-        )
-
-    def test_room_sync_data_incremental_never(self) -> None:
-        """Test getting room data where we have not previously sent down the room,
-        so its state is considered NEVER.
-        """
-        user1_id = self.register_user("user1", "pass")
-        user1_tok = self.login(user1_id, "pass")
-
-        room_id1 = self.helper.create_room_as(
-            user1_id,
-            tok=user1_tok,
-        )
-
-        sync_config, room_sync_config, room_membership_for_user_at_to_token = (
-            self._create_sync_configs(user1_id, room_id1)
-        )
-
-        # We expect to see these messages even though they're before the
-        # `from_token`, as the room has not been sent down.
-        expected_timeline = []
-        for _ in range(2):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        # Create a new `from_token`.
-        connection_token = self.get_success(
-            self.sliding_sync_handler.connection_store.record_rooms(
-                sync_config, None, sent_room_ids=[], unsent_room_ids=[]
-            )
-        )
-        from_token = SlidingSyncStreamToken(
-            self.event_sources.get_current_token(), connection_token
-        )
-
-        # We should also receive new messages
-        for _ in range(3):
-            r = self.helper.send(room_id1, "message", tok=user1_tok)
-            expected_timeline.append(r["event_id"])
-
-        to_token = self.event_sources.get_current_token()
-
-        result = self.get_success(
-            self.sliding_sync_handler.get_room_sync_data(
-                sync_config,
-                room_id=room_id1,
-                room_sync_config=room_sync_config,
-                room_membership_for_user_at_to_token=room_membership_for_user_at_to_token,
-                from_token=from_token,
-                to_token=to_token,
-            )
-        )
-
-        self.assertTrue(result.initial)
-        self.assertTrue(result.limited)
-        self.assertEqual(
-            [e.event_id for e in result.timeline_events], expected_timeline
-        )
diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py
index bce16a4eb3..05a9da1101 100644
--- a/tests/rest/client/test_sync.py
+++ b/tests/rest/client/test_sync.py
@@ -59,6 +59,7 @@ from synapse.types import (
     StreamToken,
     UserID,
 )
+from synapse.types.handlers import SlidingSyncConfig
 from synapse.util import Clock
 from synapse.util.stringutils import random_string
 
@@ -4606,6 +4607,119 @@ class SlidingSyncTestCase(SlidingSyncBase):
             exact=True,
         )
 
+    @parameterized.expand([(False,), (True,)])
+    def test_incremental_sync_full_state_previously(self, limited: bool) -> None:
+        """
+        Test getting room data where we have previously sent down the room, but
+        we missed sending down some data previously and so its state is
+        considered PREVIOUSLY.
+
+        There are two versions of this test, one where there are more messages
+        than the timeline limit, and one where there isn't.
+        """
+
+        user1_id = self.register_user("user1", "pass")
+        user1_tok = self.login(user1_id, "pass")
+
+        room_id1 = self.helper.create_room_as(user1_id, tok=user1_tok)
+        room_id2 = self.helper.create_room_as(user1_id, tok=user1_tok)
+
+        self.helper.send(room_id1, "msg", tok=user1_tok)
+
+        timeline_limit = 5
+        conn_id = "conn_id"
+        sync_body = {
+            "lists": {
+                "foo-list": {
+                    "ranges": [[0, 0]],
+                    "required_state": [
+                        [EventTypes.Create, ""],
+                        [EventTypes.RoomHistoryVisibility, ""],
+                        # This one doesn't exist in the room
+                        [EventTypes.Name, ""],
+                    ],
+                    "timeline_limit": timeline_limit,
+                }
+            },
+            "conn_id": "conn_id",
+        }
+
+        # The first room gets sent down the initial sync
+        _, initial_from_token = self.do_sync(sync_body, tok=user1_tok)
+
+        # We now send down some events in room1 (depending on the test param).
+        expected_events = []  # The set of events in the timeline
+        if limited:
+            for _ in range(10):
+                resp = self.helper.send(room_id1, "msg1", tok=user1_tok)
+                expected_events.append(resp["event_id"])
+        else:
+            resp = self.helper.send(room_id1, "msg1", tok=user1_tok)
+            expected_events.append(resp["event_id"])
+
+        # A second messages happens in the other room, so room1 won't get sent down.
+        self.helper.send(room_id2, "msg", tok=user1_tok)
+
+        # Only the second room gets sent down sync.
+        _, from_token = self.do_sync(sync_body, since=initial_from_token, tok=user1_tok)
+
+        # FIXME: This is a hack to record that the first room wasn't sent down
+        # sync, as we don't implement that currently.
+        sliding_sync_handler = self.hs.get_sliding_sync_handler()
+        requester = self.get_success(
+            self.hs.get_auth().get_user_by_access_token(user1_tok)
+        )
+        sync_config = SlidingSyncConfig(
+            user=requester.user,
+            requester=requester,
+            conn_id=conn_id,
+            lists={
+                "list": SlidingSyncConfig.SlidingSyncList(
+                    timeline_limit=1,
+                    required_state=[
+                        (EventTypes.Name, ""),
+                    ],
+                ),
+            },
+            room_subscriptions={},
+            extensions=None,
+        )
+
+        parsed_initial_from_token = self.get_success(
+            SlidingSyncStreamToken.from_string(self.store, initial_from_token)
+        )
+        connection_position = self.get_success(
+            sliding_sync_handler.connection_store.record_rooms(
+                sync_config,
+                parsed_initial_from_token,
+                sent_room_ids=[],
+                unsent_room_ids=[room_id1],
+            )
+        )
+
+        # FIXME: Now fix up `from_token` with new connect position above.
+        parsed_from_token = self.get_success(
+            SlidingSyncStreamToken.from_string(self.store, from_token)
+        )
+        parsed_from_token = SlidingSyncStreamToken(
+            stream_token=parsed_from_token.stream_token,
+            connection_position=connection_position,
+        )
+        from_token = self.get_success(parsed_from_token.to_string(self.store))
+
+        # We now send another event to room1, so we should sync all the missing events.
+        resp = self.helper.send(room_id1, "msg2", tok=user1_tok)
+        expected_events.append(resp["event_id"])
+
+        # This sync should contain the messages from room1 not yet sent down.
+        response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok)
+
+        self.assertEqual(
+            [ev["event_id"] for ev in response_body["rooms"][room_id1]["timeline"]],
+            expected_events[-timeline_limit:],
+        )
+        self.assertEqual(response_body["rooms"][room_id1]["limited"], limited)
+
 
 class SlidingSyncToDeviceExtensionTestCase(SlidingSyncBase):
     """Tests for the to-device sliding sync extension"""