summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--synapse/api/filtering.py11
-rw-r--r--synapse/handlers/events.py87
-rw-r--r--synapse/handlers/sync.py112
-rw-r--r--synapse/rest/client/v2_alpha/sync.py147
4 files changed, 181 insertions, 176 deletions
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 4d570b74f8..2d5431ba60 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -54,7 +54,7 @@ class Filtering(object):
         ]
 
         room_level_definitions = [
-            "state", "events", "ephemeral"
+            "state", "timeline", "ephemeral"
         ]
 
         for key in top_level_definitions:
@@ -135,6 +135,15 @@ class Filter(object):
     def __init__(self, filter_json):
         self.filter_json = filter_json
 
+    def timeline_limit(self):
+        return self.filter_json.get("room", {}).get("timeline", {}).get("limit", 10)
+
+    def presence_limit(self):
+        return self.filter_json.get("presence", {}).get("limit", 10)
+
+    def ephemeral_limit(self):
+        return self.filter_json.get("room", {}).get("ephemeral", {}).get("limit", 10)
+
     def filter_public_user_data(self, events):
         return self._filter_on_key(events, ["public_user_data"])
 
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index 891502c04f..92afa35d57 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -47,6 +47,56 @@ class EventStreamHandler(BaseHandler):
         self.notifier = hs.get_notifier()
 
     @defer.inlineCallbacks
+    def started_stream(self, user):
+        """Tells the presence handler that we have started an eventstream for
+        the user:
+
+        Args:
+            user (User): The user who started a stream.
+        Returns:
+            A deferred that completes once their presence has been updated.
+        """
+        if user not in self._streams_per_user:
+            self._streams_per_user[user] = 0
+            if user in self._stop_timer_per_user:
+                try:
+                    self.clock.cancel_call_later(
+                        self._stop_timer_per_user.pop(user)
+                    )
+                except:
+                    logger.exception("Failed to cancel event timer")
+            else:
+                yield self.distributor.fire("started_user_eventstream", user)
+
+        self._streams_per_user[user] += 1
+
+    def stopped_stream(self, user):
+        """If there are no streams for a user this starts a timer that will
+        notify the presence handler that we haven't got an event stream for
+        the user unless the user starts a new stream in 30 seconds.
+
+        Args:
+            user (User): The user who stopped a stream.
+        """
+        self._streams_per_user[user] -= 1
+        if not self._streams_per_user[user]:
+            del self._streams_per_user[user]
+
+            # 30 seconds of grace to allow the client to reconnect again
+            #   before we think they're gone
+            def _later():
+                logger.debug("_later stopped_user_eventstream %s", user)
+
+                self._stop_timer_per_user.pop(user, None)
+
+                return self.distributor.fire("stopped_user_eventstream", user)
+
+            logger.debug("Scheduling _later: for %s", user)
+            self._stop_timer_per_user[user] = (
+                self.clock.call_later(30, _later)
+            )
+
+    @defer.inlineCallbacks
     @log_function
     def get_stream(self, auth_user_id, pagin_config, timeout=0,
                    as_client_event=True, affect_presence=True,
@@ -59,20 +109,7 @@ class EventStreamHandler(BaseHandler):
 
         try:
             if affect_presence:
-                if auth_user not in self._streams_per_user:
-                    self._streams_per_user[auth_user] = 0
-                    if auth_user in self._stop_timer_per_user:
-                        try:
-                            self.clock.cancel_call_later(
-                                self._stop_timer_per_user.pop(auth_user)
-                            )
-                        except:
-                            logger.exception("Failed to cancel event timer")
-                    else:
-                        yield self.distributor.fire(
-                            "started_user_eventstream", auth_user
-                        )
-                self._streams_per_user[auth_user] += 1
+                yield self.started_stream(auth_user)
 
             rm_handler = self.hs.get_handlers().room_member_handler
 
@@ -114,27 +151,7 @@ class EventStreamHandler(BaseHandler):
 
         finally:
             if affect_presence:
-                self._streams_per_user[auth_user] -= 1
-                if not self._streams_per_user[auth_user]:
-                    del self._streams_per_user[auth_user]
-
-                    # 10 seconds of grace to allow the client to reconnect again
-                    #   before we think they're gone
-                    def _later():
-                        logger.debug(
-                            "_later stopped_user_eventstream %s", auth_user
-                        )
-
-                        self._stop_timer_per_user.pop(auth_user, None)
-
-                        return self.distributor.fire(
-                            "stopped_user_eventstream", auth_user
-                        )
-
-                    logger.debug("Scheduling _later: for %s", auth_user)
-                    self._stop_timer_per_user[auth_user] = (
-                        self.clock.call_later(30, _later)
-                    )
+                self.stopped_stream(auth_user)
 
 
 class EventHandler(BaseHandler):
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 9914ff6f9c..76cca7c621 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -28,21 +28,28 @@ logger = logging.getLogger(__name__)
 
 SyncConfig = collections.namedtuple("SyncConfig", [
     "user",
-    "limit",
-    "gap",
-    "sort",
-    "backfill",
     "filter",
 ])
 
 
+class TimelineBatch(collections.namedtuple("TimelineBatch", [
+    "prev_batch",
+    "events",
+    "limited",
+])):
+    __slots__ = []
+
+    def __nonzero__(self):
+        """Make the result appear empty if there are no updates. This is used
+        to tell if room needs to be part of the sync result.
+        """
+        return bool(self.events)
+
+
 class RoomSyncResult(collections.namedtuple("RoomSyncResult", [
     "room_id",
-    "limited",
-    "published",
-    "events",
+    "timeline",
     "state",
-    "prev_batch",
     "ephemeral",
 ])):
     __slots__ = []
@@ -51,13 +58,12 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [
         """Make the result appear empty if there are no updates. This is used
         to tell if room needs to be part of the sync result.
         """
-        return bool(self.events or self.state or self.ephemeral)
+        return bool(self.timeline or self.state or self.ephemeral)
 
 
 class SyncResult(collections.namedtuple("SyncResult", [
     "next_batch",  # Token for the next sync
-    "private_user_data",  # List of private events for the user.
-    "public_user_data",  # List of public events for all users.
+    "presence",  # List of presence events for the user.
     "rooms",  # RoomSyncResult for each room.
 ])):
     __slots__ = []
@@ -121,11 +127,7 @@ class SyncHandler(BaseHandler):
         if since_token is None:
             return self.initial_sync(sync_config)
         else:
-            if sync_config.gap:
-                return self.incremental_sync_with_gap(sync_config, since_token)
-            else:
-                # TODO(mjark): Handle gapless sync
-                raise NotImplementedError()
+            return self.incremental_sync_with_gap(sync_config, since_token)
 
     @defer.inlineCallbacks
     def initial_sync(self, sync_config):
@@ -133,12 +135,6 @@ class SyncHandler(BaseHandler):
         Returns:
             A Deferred SyncResult.
         """
-        if sync_config.sort == "timeline,desc":
-            # TODO(mjark): Handle going through events in reverse order?.
-            # What does "most recent events" mean when applying the limits mean
-            # in this case?
-            raise NotImplementedError()
-
         now_token = yield self.event_sources.get_current_token()
 
         presence_stream = self.event_sources.sources["presence"]
@@ -155,33 +151,27 @@ class SyncHandler(BaseHandler):
             membership_list=[Membership.INVITE, Membership.JOIN]
         )
 
-        # TODO (mjark): Does public mean "published"?
-        published_rooms = yield self.store.get_rooms(is_public=True)
-        published_room_ids = set(r["room_id"] for r in published_rooms)
-
         rooms = []
         for event in room_list:
             room_sync = yield self.initial_sync_for_room(
-                event.room_id, sync_config, now_token, published_room_ids
+                event.room_id, sync_config, now_token,
             )
             rooms.append(room_sync)
 
         defer.returnValue(SyncResult(
-            public_user_data=presence,
-            private_user_data=[],
+            presence=presence,
             rooms=rooms,
             next_batch=now_token,
         ))
 
     @defer.inlineCallbacks
-    def initial_sync_for_room(self, room_id, sync_config, now_token,
-                              published_room_ids):
+    def initial_sync_for_room(self, room_id, sync_config, now_token):
         """Sync a room for a client which is starting without any state
         Returns:
             A Deferred RoomSyncResult.
         """
 
-        recents, prev_batch_token, limited = yield self.load_filtered_recents(
+        batch = yield self.load_filtered_recents(
             room_id, sync_config, now_token,
         )
 
@@ -192,11 +182,8 @@ class SyncHandler(BaseHandler):
 
         defer.returnValue(RoomSyncResult(
             room_id=room_id,
-            published=room_id in published_room_ids,
-            events=recents,
-            prev_batch=prev_batch_token,
+            timeline=batch,
             state=current_state_events,
-            limited=limited,
             ephemeral=[],
         ))
 
@@ -207,19 +194,13 @@ class SyncHandler(BaseHandler):
         Returns:
             A Deferred SyncResult.
         """
-        if sync_config.sort == "timeline,desc":
-            # TODO(mjark): Handle going through events in reverse order?.
-            # What does "most recent events" mean when applying the limits mean
-            # in this case?
-            raise NotImplementedError()
-
         now_token = yield self.event_sources.get_current_token()
 
         presence_source = self.event_sources.sources["presence"]
         presence, presence_key = yield presence_source.get_new_events_for_user(
             user=sync_config.user,
             from_key=since_token.presence_key,
-            limit=sync_config.limit,
+            limit=sync_config.filter.presence_limit(),
         )
         now_token = now_token.copy_and_replace("presence_key", presence_key)
 
@@ -227,7 +208,7 @@ class SyncHandler(BaseHandler):
         typing, typing_key = yield typing_source.get_new_events_for_user(
             user=sync_config.user,
             from_key=since_token.typing_key,
-            limit=sync_config.limit,
+            limit=sync_config.filter.ephemeral_limit(),
         )
         now_token = now_token.copy_and_replace("typing_key", typing_key)
 
@@ -248,20 +229,18 @@ class SyncHandler(BaseHandler):
                 sync_config.user
             )
 
-        # TODO (mjark): Does public mean "published"?
-        published_rooms = yield self.store.get_rooms(is_public=True)
-        published_room_ids = set(r["room_id"] for r in published_rooms)
+        timeline_limit = sync_config.filter.timeline_limit()
 
         room_events, _ = yield self.store.get_room_events_stream(
             sync_config.user.to_string(),
             from_key=since_token.room_key,
             to_key=now_token.room_key,
             room_id=None,
-            limit=sync_config.limit + 1,
+            limit=timeline_limit + 1,
         )
 
         rooms = []
-        if len(room_events) <= sync_config.limit:
+        if len(room_events) <= timeline_limit:
             # There is no gap in any of the rooms. Therefore we can just
             # partition the new events by room and return them.
             events_by_room_id = {}
@@ -284,11 +263,12 @@ class SyncHandler(BaseHandler):
 
                 room_sync = RoomSyncResult(
                     room_id=room_id,
-                    published=room_id in published_room_ids,
-                    events=recents,
-                    prev_batch=prev_batch,
+                    timeline=TimelineBatch(
+                        events=recents,
+                        prev_batch=prev_batch,
+                        limited=False,
+                    ),
                     state=state,
-                    limited=False,
                     ephemeral=typing_by_room.get(room_id, [])
                 )
                 if room_sync:
@@ -303,8 +283,7 @@ class SyncHandler(BaseHandler):
                     rooms.append(room_sync)
 
         defer.returnValue(SyncResult(
-            public_user_data=presence,
-            private_user_data=[],
+            presence=presence,
             rooms=rooms,
             next_batch=now_token,
         ))
@@ -361,12 +340,13 @@ class SyncHandler(BaseHandler):
         limited = True
         recents = []
         filtering_factor = 2
-        load_limit = max(sync_config.limit * filtering_factor, 100)
+        timeline_limit = sync_config.filter.timeline_limit()
+        load_limit = max(timeline_limit * filtering_factor, 100)
         max_repeat = 3  # Only try a few times per room, otherwise
         room_key = now_token.room_key
         end_key = room_key
 
-        while limited and len(recents) < sync_config.limit and max_repeat:
+        while limited and len(recents) < timeline_limit and max_repeat:
             events, keys = yield self.store.get_recent_events_for_room(
                 room_id,
                 limit=load_limit + 1,
@@ -385,15 +365,18 @@ class SyncHandler(BaseHandler):
                 limited = False
             max_repeat -= 1
 
-        if len(recents) > sync_config.limit:
-            recents = recents[-sync_config.limit:]
+        if len(recents) > timeline_limit:
+            limited = True
+            recents = recents[-timeline_limit:]
             room_key = recents[0].internal_metadata.before
 
         prev_batch_token = now_token.copy_and_replace(
             "room_key", room_key
         )
 
-        defer.returnValue((recents, prev_batch_token, limited))
+        defer.returnValue(TimelineBatch(
+            events=recents, prev_batch=prev_batch_token, limited=limited
+        ))
 
     @defer.inlineCallbacks
     def incremental_sync_with_gap_for_room(self, room_id, sync_config,
@@ -408,11 +391,11 @@ class SyncHandler(BaseHandler):
 
         # TODO(mjark): Check for redactions we might have missed.
 
-        recents, prev_batch_token, limited = yield self.load_filtered_recents(
+        batch = yield self.load_filtered_recents(
             room_id, sync_config, now_token, since_token,
         )
 
-        logging.debug("Recents %r", recents)
+        logging.debug("Recents %r", batch)
 
         # TODO(mjark): This seems racy since this isn't being passed a
         # token to indicate what point in the stream this is
@@ -437,11 +420,8 @@ class SyncHandler(BaseHandler):
 
         room_sync = RoomSyncResult(
             room_id=room_id,
-            published=room_id in published_room_ids,
-            events=recents,
-            prev_batch=prev_batch_token,
+            timeline=batch,
             state=state_events_delta,
-            limited=limited,
             ephemeral=typing_by_room.get(room_id, [])
         )
 
diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py
index cac28b47b6..3348b46c14 100644
--- a/synapse/rest/client/v2_alpha/sync.py
+++ b/synapse/rest/client/v2_alpha/sync.py
@@ -16,7 +16,7 @@
 from twisted.internet import defer
 
 from synapse.http.servlet import (
-    RestServlet, parse_string, parse_integer, parse_boolean
+    RestServlet, parse_string, parse_integer
 )
 from synapse.handlers.sync import SyncConfig
 from synapse.types import StreamToken
@@ -26,6 +26,7 @@ from synapse.events.utils import (
 from synapse.api.filtering import Filter
 from ._base import client_v2_pattern
 
+import copy
 import logging
 
 logger = logging.getLogger(__name__)
@@ -36,51 +37,49 @@ class SyncRestServlet(RestServlet):
 
     GET parameters::
         timeout(int): How long to wait for new events in milliseconds.
-        limit(int): Maxiumum number of events per room to return.
-        gap(bool): Create gaps the message history if limit is exceeded to
-            ensure that the client has the most recent messages. Defaults to
-            "true".
-        sort(str,str): tuple of sort key (e.g. "timeline") and direction
-            (e.g. "asc", "desc"). Defaults to "timeline,asc".
         since(batch_token): Batch token when asking for incremental deltas.
         set_presence(str): What state the device presence should be set to.
             default is "online".
-        backfill(bool): Should the HS request message history from other
-            servers. This may take a long time making it unsuitable for clients
-            expecting a prompt response. Defaults to "true".
         filter(filter_id): A filter to apply to the events returned.
-        filter_*: Filter override parameters.
 
     Response JSON::
         {
-            "next_batch": // batch token for the next /sync
-            "private_user_data": // private events for this user.
-            "public_user_data": // public events for all users including the
-                                // public events for this user.
-            "rooms": [{ // List of rooms with updates.
-                "room_id": // Id of the room being updated
-                "limited": // Was the per-room event limit exceeded?
-                "published": // Is the room published by our HS?
+          "next_batch": // batch token for the next /sync
+          "presence": // presence data for the user.
+               "invited": [], // Ids of invited rooms being updated.
+               "joined": [], // Ids of joined rooms being updated.
+               "archived": [] // Ids of archived rooms being updated.
+            }
+          }
+          "rooms": {
+            "joined": { // Joined rooms being updated.
+              "${room_id}": { // Id of the room being updated
                 "event_map": // Map of EventID -> event JSON.
-                "events": { // The recent events in the room if gap is "true"
-                            // otherwise the next events in the room.
-                    "batch": [] // list of EventIDs in the "event_map".
-                    "prev_batch": // back token for getting previous events.
+                "timeline": { // The recent events in the room if gap is "true"
+                  "limited": // Was the per-room event limit exceeded?
+                             // otherwise the next events in the room.
+                  "events": [] // list of EventIDs in the "event_map".
+                  "prev_batch": // back token for getting previous events.
                 }
-                "state": [] // list of EventIDs updating the current state to
-                            // be what it should be at the end of the batch.
-                "ephemeral": []
-            }]
+                "state": {"events": []} // list of EventIDs updating the
+                                        // current state to be what it should
+                                        // be at the end of the batch.
+                "ephemeral": {"events": []} // list of event objects
+              }
+            },
+            "invited": {}, // Ids of invited rooms being updated.
+            "archived": {} // Ids of archived rooms being updated.
+          }
         }
     """
 
     PATTERN = client_v2_pattern("/sync$")
-    ALLOWED_SORT = set(["timeline,asc", "timeline,desc"])
-    ALLOWED_PRESENCE = set(["online", "offline", "idle"])
+    ALLOWED_PRESENCE = set(["online", "offline"])
 
     def __init__(self, hs):
         super(SyncRestServlet, self).__init__()
         self.auth = hs.get_auth()
+        self.event_stream_handler = hs.get_handlers().event_stream_handler
         self.sync_handler = hs.get_handlers().sync_handler
         self.clock = hs.get_clock()
         self.filtering = hs.get_filtering()
@@ -90,45 +89,29 @@ class SyncRestServlet(RestServlet):
         user, token_id = yield self.auth.get_user_by_req(request)
 
         timeout = parse_integer(request, "timeout", default=0)
-        limit = parse_integer(request, "limit", required=True)
-        gap = parse_boolean(request, "gap", default=True)
-        sort = parse_string(
-            request, "sort", default="timeline,asc",
-            allowed_values=self.ALLOWED_SORT
-        )
         since = parse_string(request, "since")
         set_presence = parse_string(
             request, "set_presence", default="online",
             allowed_values=self.ALLOWED_PRESENCE
         )
-        backfill = parse_boolean(request, "backfill", default=False)
         filter_id = parse_string(request, "filter", default=None)
 
         logger.info(
-            "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r,"
-            " set_presence=%r, backfill=%r, filter_id=%r" % (
-                user, timeout, limit, gap, sort, since, set_presence,
-                backfill, filter_id
+            "/sync: user=%r, timeout=%r, since=%r,"
+            " set_presence=%r, filter_id=%r" % (
+                user, timeout, since, set_presence, filter_id
             )
         )
 
-        # TODO(mjark): Load filter and apply overrides.
         try:
             filter = yield self.filtering.get_user_filter(
                 user.localpart, filter_id
             )
         except:
             filter = Filter({})
-        # filter = filter.apply_overrides(http_request)
-        # if filter.matches(event):
-        #   # stuff
 
         sync_config = SyncConfig(
             user=user,
-            gap=gap,
-            limit=limit,
-            sort=sort,
-            backfill=backfill,
             filter=filter,
         )
 
@@ -137,41 +120,59 @@ class SyncRestServlet(RestServlet):
         else:
             since_token = None
 
-        sync_result = yield self.sync_handler.wait_for_sync_for_user(
-            sync_config, since_token=since_token, timeout=timeout
-        )
+        if set_presence == "online":
+            yield self.event_stream_handler.started_stream(user)
+
+        try:
+            sync_result = yield self.sync_handler.wait_for_sync_for_user(
+                sync_config, since_token=since_token, timeout=timeout
+            )
+        finally:
+            if set_presence == "online":
+                self.event_stream_handler.stopped_stream(user)
 
         time_now = self.clock.time_msec()
 
+        rooms = self.encode_rooms(
+            sync_result.rooms, filter, time_now, token_id
+        )
+
         response_content = {
-            "public_user_data": self.encode_user_data(
-                sync_result.public_user_data, filter, time_now
-            ),
-            "private_user_data": self.encode_user_data(
-                sync_result.private_user_data, filter, time_now
-            ),
-            "rooms": self.encode_rooms(
-                sync_result.rooms, filter, time_now, token_id
+            "presence": self.encode_presence(
+                sync_result.presence, filter, time_now
             ),
+            "rooms": rooms,
             "next_batch": sync_result.next_batch.to_string(),
         }
 
         defer.returnValue((200, response_content))
 
-    def encode_user_data(self, events, filter, time_now):
-        return events
+    def encode_presence(self, events, filter, time_now):
+        formatted = []
+        for event in events:
+            event = copy.deepcopy(event)
+            event['sender'] = event['content'].pop('user_id');
+            formatted.append(event)
+        return {"events": formatted}
 
     def encode_rooms(self, rooms, filter, time_now, token_id):
-        return [
-            self.encode_room(room, filter, time_now, token_id)
-            for room in rooms
-        ]
+        joined = {}
+        for room in rooms:
+            joined[room.room_id] = self.encode_room(
+                room, filter, time_now, token_id
+            )
+
+        return {
+            "joined": joined,
+            "invited": {},
+            "archived": {},
+        }
 
     @staticmethod
     def encode_room(room, filter, time_now, token_id):
         event_map = {}
         state_events = filter.filter_room_state(room.state)
-        recent_events = filter.filter_room_events(room.events)
+        recent_events = filter.filter_room_events(room.timeline.events)
         state_event_ids = []
         recent_event_ids = []
         for event in state_events:
@@ -190,16 +191,14 @@ class SyncRestServlet(RestServlet):
             )
             recent_event_ids.append(event.event_id)
         result = {
-            "room_id": room.room_id,
             "event_map": event_map,
-            "events": {
-                "batch": recent_event_ids,
-                "prev_batch": room.prev_batch.to_string(),
+            "timeline": {
+                "events": recent_event_ids,
+                "prev_batch": room.timeline.prev_batch.to_string(),
+                "limited": room.timeline.limited,
             },
-            "state": state_event_ids,
-            "limited": room.limited,
-            "published": room.published,
-            "ephemeral": room.ephemeral,
+            "state": {"events": state_event_ids},
+            "ephemeral": {"events": room.ephemeral},
         }
         return result