diff options
| -rw-r--r-- | synapse/api/filtering.py | 62 | ||||
| -rw-r--r-- | synapse/handlers/sync.py | 172 | ||||
| -rw-r--r-- | synapse/rest/client/v2_alpha/sync.py | 151 | ||||
| -rw-r--r-- | tests/api/test_filtering.py | 12 |
4 files changed, 210 insertions, 187 deletions
diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 4d570b74f8..e79e91e7eb 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -54,7 +54,7 @@ class Filtering(object): ] room_level_definitions = [ - "state", "events", "ephemeral" + "state", "timeline", "ephemeral" ] for key in top_level_definitions: @@ -135,17 +135,23 @@ class Filter(object): def __init__(self, filter_json): self.filter_json = filter_json - def filter_public_user_data(self, events): - return self._filter_on_key(events, ["public_user_data"]) + def timeline_limit(self): + return self.filter_json.get("room", {}).get("timeline", {}).get("limit", 10) - def filter_private_user_data(self, events): - return self._filter_on_key(events, ["private_user_data"]) + def presence_limit(self): + return self.filter_json.get("presence", {}).get("limit", 10) + + def ephemeral_limit(self): + return self.filter_json.get("room", {}).get("ephemeral", {}).get("limit", 10) + + def filter_presence(self, events): + return self._filter_on_key(events, ["presence"]) def filter_room_state(self, events): return self._filter_on_key(events, ["room", "state"]) - def filter_room_events(self, events): - return self._filter_on_key(events, ["room", "events"]) + def filter_room_timeline(self, events): + return self._filter_on_key(events, ["room", "timeline"]) def filter_room_ephemeral(self, events): return self._filter_on_key(events, ["room", "ephemeral"]) @@ -169,11 +175,34 @@ class Filter(object): return [e for e in events if self._passes_definition(definition, e)] def _passes_definition(self, definition, event): + """Check if the event passes the filter definition + Args: + definition(dict): The filter definition to check against + event(dict or Event): The event to check + Returns: + True if the event passes the filter in the definition + """ + if type(event) is dict: + room_id = event.get("room_id") + sender = event.get("sender") + event_type = event["type"] + else: + room_id = getattr(event, "room_id", None) + sender = getattr(event, "sender", None) + event_type = event.type + return self._event_passes_definition( + definition, room_id, sender, event_type + ) + + def _event_passes_definition(self, definition, room_id, sender, + event_type): """Check if the event passes through the given definition. Args: definition(dict): The definition to check against. - event(Event): The event to check. + room_id(str): The id of the room this event is in or None. + sender(str): The sender of the event + event_type(str): The type of the event. Returns: True if the event passes through the filter. """ @@ -185,8 +214,7 @@ class Filter(object): # and 'not_types' then it is treated as only being in 'not_types') # room checks - if hasattr(event, "room_id"): - room_id = event.room_id + if room_id is not None: allow_rooms = definition.get("rooms", None) reject_rooms = definition.get("not_rooms", None) if reject_rooms and room_id in reject_rooms: @@ -195,9 +223,7 @@ class Filter(object): return False # sender checks - if hasattr(event, "sender"): - # Should we be including event.state_key for some event types? - sender = event.sender + if sender is not None: allow_senders = definition.get("senders", None) reject_senders = definition.get("not_senders", None) if reject_senders and sender in reject_senders: @@ -208,12 +234,12 @@ class Filter(object): # type checks if "not_types" in definition: for def_type in definition["not_types"]: - if self._event_matches_type(event, def_type): + if self._event_matches_type(event_type, def_type): return False if "types" in definition: included = False for def_type in definition["types"]: - if self._event_matches_type(event, def_type): + if self._event_matches_type(event_type, def_type): included = True break if not included: @@ -221,9 +247,9 @@ class Filter(object): return True - def _event_matches_type(self, event, def_type): + def _event_matches_type(self, event_type, def_type): if def_type.endswith("*"): type_prefix = def_type[:-1] - return event.type.startswith(type_prefix) + return event_type.startswith(type_prefix) else: - return event.type == def_type + return event_type == def_type diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9914ff6f9c..e693e7c80e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,21 +28,28 @@ logger = logging.getLogger(__name__) SyncConfig = collections.namedtuple("SyncConfig", [ "user", - "limit", - "gap", - "sort", - "backfill", "filter", ]) -class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ - "room_id", - "limited", - "published", +class TimelineBatch(collections.namedtuple("TimelineBatch", [ + "prev_batch", "events", + "limited", +])): + __slots__ = [] + + def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ + return bool(self.events) + + +class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [ + "room_id", + "timeline", "state", - "prev_batch", "ephemeral", ])): __slots__ = [] @@ -51,14 +58,27 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [ """Make the result appear empty if there are no updates. This is used to tell if room needs to be part of the sync result. """ - return bool(self.events or self.state or self.ephemeral) + return bool(self.timeline or self.state or self.ephemeral) + + +class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [ + "room_id", + "invite_state", +])): + __slots__ = [] + + def __nonzero__(self): + """Make the result appear empty if there are no updates. This is used + to tell if room needs to be part of the sync result. + """ + return bool(self.invite_state) class SyncResult(collections.namedtuple("SyncResult", [ "next_batch", # Token for the next sync - "private_user_data", # List of private events for the user. - "public_user_data", # List of public events for all users. - "rooms", # RoomSyncResult for each room. + "presence", # List of presence events for the user. + "joined", # JoinedSyncResult for each joined room. + "invited", # InvitedSyncResult for each invited room. ])): __slots__ = [] @@ -121,11 +141,7 @@ class SyncHandler(BaseHandler): if since_token is None: return self.initial_sync(sync_config) else: - if sync_config.gap: - return self.incremental_sync_with_gap(sync_config, since_token) - else: - # TODO(mjark): Handle gapless sync - raise NotImplementedError() + return self.incremental_sync_with_gap(sync_config, since_token) @defer.inlineCallbacks def initial_sync(self, sync_config): @@ -133,12 +149,6 @@ class SyncHandler(BaseHandler): Returns: A Deferred SyncResult. """ - if sync_config.sort == "timeline,desc": - # TODO(mjark): Handle going through events in reverse order?. - # What does "most recent events" mean when applying the limits mean - # in this case? - raise NotImplementedError() - now_token = yield self.event_sources.get_current_token() presence_stream = self.event_sources.sources["presence"] @@ -155,33 +165,34 @@ class SyncHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - # TODO (mjark): Does public mean "published"? - published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in published_rooms) - - rooms = [] + joined = [] for event in room_list: - room_sync = yield self.initial_sync_for_room( - event.room_id, sync_config, now_token, published_room_ids - ) - rooms.append(room_sync) + if event.membership == Membership.JOIN: + room_sync = yield self.initial_sync_for_room( + event.room_id, sync_config, now_token, + ) + joined.append(room_sync) + elif event.membership == Membership.INVITE: + invited.append(InvitedSyncResult( + room_id=event.room_id, + invited_state=[event], + ) defer.returnValue(SyncResult( - public_user_data=presence, - private_user_data=[], - rooms=rooms, + presence=presence, + joined=joined, + invited=[], next_batch=now_token, )) @defer.inlineCallbacks - def initial_sync_for_room(self, room_id, sync_config, now_token, - published_room_ids): + def initial_sync_for_joined_room(self, room_id, sync_config, now_token): """Sync a room for a client which is starting without any state Returns: - A Deferred RoomSyncResult. + A Deferred JoinedSyncResult. """ - recents, prev_batch_token, limited = yield self.load_filtered_recents( + batch = yield self.load_filtered_recents( room_id, sync_config, now_token, ) @@ -190,13 +201,10 @@ class SyncHandler(BaseHandler): ) current_state_events = current_state.values() - defer.returnValue(RoomSyncResult( + defer.returnValue(JoinedSyncResult( room_id=room_id, - published=room_id in published_room_ids, - events=recents, - prev_batch=prev_batch_token, + timeline=batch, state=current_state_events, - limited=limited, ephemeral=[], )) @@ -207,19 +215,13 @@ class SyncHandler(BaseHandler): Returns: A Deferred SyncResult. """ - if sync_config.sort == "timeline,desc": - # TODO(mjark): Handle going through events in reverse order?. - # What does "most recent events" mean when applying the limits mean - # in this case? - raise NotImplementedError() - now_token = yield self.event_sources.get_current_token() presence_source = self.event_sources.sources["presence"] presence, presence_key = yield presence_source.get_new_events_for_user( user=sync_config.user, from_key=since_token.presence_key, - limit=sync_config.limit, + limit=sync_config.filter.presence_limit(), ) now_token = now_token.copy_and_replace("presence_key", presence_key) @@ -227,7 +229,7 @@ class SyncHandler(BaseHandler): typing, typing_key = yield typing_source.get_new_events_for_user( user=sync_config.user, from_key=since_token.typing_key, - limit=sync_config.limit, + limit=sync_config.filter.ephemeral_limit(), ) now_token = now_token.copy_and_replace("typing_key", typing_key) @@ -248,20 +250,18 @@ class SyncHandler(BaseHandler): sync_config.user ) - # TODO (mjark): Does public mean "published"? - published_rooms = yield self.store.get_rooms(is_public=True) - published_room_ids = set(r["room_id"] for r in published_rooms) + timeline_limit = sync_config.filter.timeline_limit() room_events, _ = yield self.store.get_room_events_stream( sync_config.user.to_string(), from_key=since_token.room_key, to_key=now_token.room_key, room_id=None, - limit=sync_config.limit + 1, + limit=timeline_limit + 1, ) - rooms = [] - if len(room_events) <= sync_config.limit: + joined = [] + if len(room_events) <= timeline_limit: # There is no gap in any of the rooms. Therefore we can just # partition the new events by room and return them. events_by_room_id = {} @@ -282,30 +282,31 @@ class SyncHandler(BaseHandler): sync_config, room_id, state ) - room_sync = RoomSyncResult( + room_sync = JoinedSyncResult( room_id=room_id, - published=room_id in published_room_ids, - events=recents, - prev_batch=prev_batch, + timeline=TimelineBatch( + events=recents, + prev_batch=prev_batch, + limited=False, + ), state=state, - limited=False, ephemeral=typing_by_room.get(room_id, []) ) if room_sync: - rooms.append(room_sync) + joined.append(room_sync) else: for room_id in room_ids: room_sync = yield self.incremental_sync_with_gap_for_room( room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room + typing_by_room ) if room_sync: - rooms.append(room_sync) + joined.append(room_sync) defer.returnValue(SyncResult( - public_user_data=presence, - private_user_data=[], - rooms=rooms, + presence=presence, + joined=joined, + invited=[], next_batch=now_token, )) @@ -361,12 +362,13 @@ class SyncHandler(BaseHandler): limited = True recents = [] filtering_factor = 2 - load_limit = max(sync_config.limit * filtering_factor, 100) + timeline_limit = sync_config.filter.timeline_limit() + load_limit = max(timeline_limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key end_key = room_key - while limited and len(recents) < sync_config.limit and max_repeat: + while limited and len(recents) < timeline_limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, @@ -375,7 +377,7 @@ class SyncHandler(BaseHandler): ) (room_key, _) = keys end_key = "s" + room_key.split('-')[-1] - loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents = sync_config.filter.filter_room_timeline(events) loaded_recents = yield self._filter_events_for_client( sync_config.user.to_string(), room_id, loaded_recents, ) @@ -385,34 +387,37 @@ class SyncHandler(BaseHandler): limited = False max_repeat -= 1 - if len(recents) > sync_config.limit: - recents = recents[-sync_config.limit:] + if len(recents) > timeline_limit: + limited = True + recents = recents[-timeline_limit:] room_key = recents[0].internal_metadata.before prev_batch_token = now_token.copy_and_replace( "room_key", room_key ) - defer.returnValue((recents, prev_batch_token, limited)) + defer.returnValue(TimelineBatch( + events=recents, prev_batch=prev_batch_token, limited=limited + )) @defer.inlineCallbacks def incremental_sync_with_gap_for_room(self, room_id, sync_config, since_token, now_token, - published_room_ids, typing_by_room): + typing_by_room): """ Get the incremental delta needed to bring the client up to date for the room. Gives the client the most recent events and the changes to state. Returns: - A Deferred RoomSyncResult + A Deferred JoinedSyncResult """ # TODO(mjark): Check for redactions we might have missed. - recents, prev_batch_token, limited = yield self.load_filtered_recents( + batch = yield self.load_filtered_recents( room_id, sync_config, now_token, since_token, ) - logging.debug("Recents %r", recents) + logging.debug("Recents %r", batch) # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is @@ -435,13 +440,10 @@ class SyncHandler(BaseHandler): sync_config, room_id, state_events_delta ) - room_sync = RoomSyncResult( + room_sync = JoinedSyncResult( room_id=room_id, - published=room_id in published_room_ids, - events=recents, - prev_batch=prev_batch_token, + timeline=batch, state=state_events_delta, - limited=limited, ephemeral=typing_by_room.get(room_id, []) ) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index cac28b47b6..9b87879f51 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.http.servlet import ( - RestServlet, parse_string, parse_integer, parse_boolean + RestServlet, parse_string, parse_integer ) from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken @@ -26,6 +26,7 @@ from synapse.events.utils import ( from synapse.api.filtering import Filter from ._base import client_v2_pattern +import copy import logging logger = logging.getLogger(__name__) @@ -36,51 +37,44 @@ class SyncRestServlet(RestServlet): GET parameters:: timeout(int): How long to wait for new events in milliseconds. - limit(int): Maxiumum number of events per room to return. - gap(bool): Create gaps the message history if limit is exceeded to - ensure that the client has the most recent messages. Defaults to - "true". - sort(str,str): tuple of sort key (e.g. "timeline") and direction - (e.g. "asc", "desc"). Defaults to "timeline,asc". since(batch_token): Batch token when asking for incremental deltas. set_presence(str): What state the device presence should be set to. default is "online". - backfill(bool): Should the HS request message history from other - servers. This may take a long time making it unsuitable for clients - expecting a prompt response. Defaults to "true". filter(filter_id): A filter to apply to the events returned. - filter_*: Filter override parameters. Response JSON:: { - "next_batch": // batch token for the next /sync - "private_user_data": // private events for this user. - "public_user_data": // public events for all users including the - // public events for this user. - "rooms": [{ // List of rooms with updates. - "room_id": // Id of the room being updated - "limited": // Was the per-room event limit exceeded? - "published": // Is the room published by our HS? + "next_batch": // batch token for the next /sync + "presence": // presence data for the user. + "rooms": { + "joined": { // Joined rooms being updated. + "${room_id}": { // Id of the room being updated "event_map": // Map of EventID -> event JSON. - "events": { // The recent events in the room if gap is "true" - // otherwise the next events in the room. - "batch": [] // list of EventIDs in the "event_map". - "prev_batch": // back token for getting previous events. + "timeline": { // The recent events in the room if gap is "true" + "limited": // Was the per-room event limit exceeded? + // otherwise the next events in the room. + "events": [] // list of EventIDs in the "event_map". + "prev_batch": // back token for getting previous events. } - "state": [] // list of EventIDs updating the current state to - // be what it should be at the end of the batch. - "ephemeral": [] - }] + "state": {"events": []} // list of EventIDs updating the + // current state to be what it should + // be at the end of the batch. + "ephemeral": {"events": []} // list of event objects + } + }, + "invited": {}, // Invited rooms being updated. + "archived": {} // Archived rooms being updated. + } } """ PATTERN = client_v2_pattern("/sync$") - ALLOWED_SORT = set(["timeline,asc", "timeline,desc"]) - ALLOWED_PRESENCE = set(["online", "offline", "idle"]) + ALLOWED_PRESENCE = set(["online", "offline"]) def __init__(self, hs): super(SyncRestServlet, self).__init__() self.auth = hs.get_auth() + self.event_stream_handler = hs.get_handlers().event_stream_handler self.sync_handler = hs.get_handlers().sync_handler self.clock = hs.get_clock() self.filtering = hs.get_filtering() @@ -90,45 +84,29 @@ class SyncRestServlet(RestServlet): user, token_id = yield self.auth.get_user_by_req(request) timeout = parse_integer(request, "timeout", default=0) - limit = parse_integer(request, "limit", required=True) - gap = parse_boolean(request, "gap", default=True) - sort = parse_string( - request, "sort", default="timeline,asc", - allowed_values=self.ALLOWED_SORT - ) since = parse_string(request, "since") set_presence = parse_string( request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = parse_boolean(request, "backfill", default=False) filter_id = parse_string(request, "filter", default=None) logger.info( - "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," - " set_presence=%r, backfill=%r, filter_id=%r" % ( - user, timeout, limit, gap, sort, since, set_presence, - backfill, filter_id + "/sync: user=%r, timeout=%r, since=%r," + " set_presence=%r, filter_id=%r" % ( + user, timeout, since, set_presence, filter_id ) ) - # TODO(mjark): Load filter and apply overrides. try: filter = yield self.filtering.get_user_filter( user.localpart, filter_id ) except: filter = Filter({}) - # filter = filter.apply_overrides(http_request) - # if filter.matches(event): - # # stuff sync_config = SyncConfig( user=user, - gap=gap, - limit=limit, - sort=sort, - backfill=backfill, filter=filter, ) @@ -137,43 +115,62 @@ class SyncRestServlet(RestServlet): else: since_token = None - sync_result = yield self.sync_handler.wait_for_sync_for_user( - sync_config, since_token=since_token, timeout=timeout - ) + if set_presence == "online": + yield self.event_stream_handler.started_stream(user) + + try: + sync_result = yield self.sync_handler.wait_for_sync_for_user( + sync_config, since_token=since_token, timeout=timeout + ) + finally: + if set_presence == "online": + self.event_stream_handler.stopped_stream(user) time_now = self.clock.time_msec() + joined = self.encode_joined( + sync_result.joined, filter, time_now, token_id + ) + response_content = { - "public_user_data": self.encode_user_data( - sync_result.public_user_data, filter, time_now - ), - "private_user_data": self.encode_user_data( - sync_result.private_user_data, filter, time_now - ), - "rooms": self.encode_rooms( - sync_result.rooms, filter, time_now, token_id + "presence": self.encode_presence( + sync_result.presence, filter, time_now ), + "rooms": { + "joined": joined, + "invited": {}, + "archived": {}, + }, "next_batch": sync_result.next_batch.to_string(), } defer.returnValue((200, response_content)) - def encode_user_data(self, events, filter, time_now): - return events + def encode_presence(self, events, filter, time_now): + formatted = [] + for event in events: + event = copy.deepcopy(event) + event['sender'] = event['content'].pop('user_id') + formatted.append(event) + return {"events": filter.filter_presence(formatted)} + + def encode_joined(self, rooms, filter, time_now, token_id): + joined = {} + for room in rooms: + joined[room.room_id] = self.encode_room( + room, filter, time_now, token_id + ) - def encode_rooms(self, rooms, filter, time_now, token_id): - return [ - self.encode_room(room, filter, time_now, token_id) - for room in rooms - ] + return joined @staticmethod def encode_room(room, filter, time_now, token_id): event_map = {} state_events = filter.filter_room_state(room.state) - recent_events = filter.filter_room_events(room.events) + timeline_events = filter.filter_room_timeline(room.timeline.events) + ephemeral_events = filter.filter_room_ephemeral(room.ephemeral) state_event_ids = [] - recent_event_ids = [] + timeline_event_ids = [] for event in state_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( @@ -182,24 +179,22 @@ class SyncRestServlet(RestServlet): ) state_event_ids.append(event.event_id) - for event in recent_events: + for event in timeline_events: # TODO(mjark): Respect formatting requirements in the filter. event_map[event.event_id] = serialize_event( event, time_now, token_id=token_id, event_format=format_event_for_client_v2_without_event_id, ) - recent_event_ids.append(event.event_id) + timeline_event_ids.append(event.event_id) result = { - "room_id": room.room_id, "event_map": event_map, - "events": { - "batch": recent_event_ids, - "prev_batch": room.prev_batch.to_string(), + "timeline": { + "events": timeline_event_ids, + "prev_batch": room.timeline.prev_batch.to_string(), + "limited": room.timeline.limited, }, - "state": state_event_ids, - "limited": room.limited, - "published": room.published, - "ephemeral": room.ephemeral, + "state": {"events": state_event_ids}, + "ephemeral": {"events": ephemeral_events}, } return result diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index 65b2f590c8..6942cdac51 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -345,9 +345,9 @@ class FilteringTestCase(unittest.TestCase): ) @defer.inlineCallbacks - def test_filter_public_user_data_match(self): + def test_filter_presence_match(self): user_filter_json = { - "public_user_data": { + "presence": { "types": ["m.*"] } } @@ -368,13 +368,13 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - results = user_filter.filter_public_user_data(events=events) + results = user_filter.filter_presence(events=events) self.assertEquals(events, results) @defer.inlineCallbacks - def test_filter_public_user_data_no_match(self): + def test_filter_presence_no_match(self): user_filter_json = { - "public_user_data": { + "presence": { "types": ["m.*"] } } @@ -395,7 +395,7 @@ class FilteringTestCase(unittest.TestCase): filter_id=filter_id, ) - results = user_filter.filter_public_user_data(events=events) + results = user_filter.filter_presence(events=events) self.assertEquals([], results) @defer.inlineCallbacks |