From e5999bfb1a4aab56acecb59ed6d068442f5b11a0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Feb 2016 17:10:40 +0000 Subject: Initial cut --- synapse/storage/presence.py | 170 +++++++++++++++++++++++++++++++------------- 1 file changed, 120 insertions(+), 50 deletions(-) (limited to 'synapse/storage/presence.py') diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index ef525f34c5..b133979102 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -14,73 +14,128 @@ # limitations under the License. from ._base import SQLBaseStore -from synapse.util.caches.descriptors import cached, cachedList +from synapse.api.constants import PresenceState +from synapse.util.caches.descriptors import cached, cachedInlineCallbacks +from collections import namedtuple from twisted.internet import defer -class PresenceStore(SQLBaseStore): - def create_presence(self, user_localpart): - res = self._simple_insert( - table="presence", - values={"user_id": user_localpart}, - desc="create_presence", +class UserPresenceState(namedtuple("UserPresenceState", + ("user_id", "state", "last_active", "last_federation_update", + "last_user_sync", "status_msg", "currently_active"))): + """Represents the current presence state of the user. + + user_id (str) + last_active (int): Time in msec that the user last interacted with server. + last_federation_update (int): Time in msec since either a) we sent a presence + update to other servers or b) we received a presence update, depending + on if is a local user or not. + last_user_sync (int): Time in msec that the user last *completed* a sync + (or event stream). + status_msg (str): User set status message. + """ + + def copy_and_replace(self, **kwargs): + return self._replace(**kwargs) + + @classmethod + def default(cls, user_id): + """Returns a default presence state. + """ + return cls( + user_id=user_id, + state=PresenceState.OFFLINE, + last_active=0, + last_federation_update=0, + last_user_sync=0, + status_msg=None, + currently_active=False, ) - self.get_presence_state.invalidate((user_localpart,)) - return res - def has_presence_state(self, user_localpart): - return self._simple_select_one( - table="presence", - keyvalues={"user_id": user_localpart}, - retcols=["user_id"], - allow_none=True, - desc="has_presence_state", +class PresenceStore(SQLBaseStore): + @defer.inlineCallbacks + def update_presence(self, presence_states): + stream_id_manager = yield self._presence_id_gen.get_next(self) + with stream_id_manager as stream_id: + yield self.runInteraction( + "update_presence", + self._update_presence_txn, stream_id, presence_states, + ) + + defer.returnValue((stream_id, self._presence_id_gen.get_max_token())) + + def _update_presence_txn(self, txn, stream_id, presence_states): + for state in presence_states: + txn.call_after( + self.presence_stream_cache.entity_has_changed, + state.user_id, stream_id, + ) + + # Actually insert new rows + self._simple_insert_many_txn( + txn, + table="presence_stream", + values=[ + { + "stream_id": stream_id, + "user_id": state.user_id, + "state": state.state, + "last_active": state.last_active, + "last_federation_update": state.last_federation_update, + "last_user_sync": state.last_user_sync, + "status_msg": state.status_msg, + "currently_active": state.currently_active, + } + for state in presence_states + ], ) - @cached(max_entries=2000) - def get_presence_state(self, user_localpart): - return self._simple_select_one( - table="presence", - keyvalues={"user_id": user_localpart}, - retcols=["state", "status_msg", "mtime"], - desc="get_presence_state", + # Delete old rows to stop database from getting really big + sql = ( + "DELETE FROM presence_stream WHERE" + " stream_id < ?" + " AND user_id IN (%s)" ) - @cachedList(get_presence_state.cache, list_name="user_localparts", - inlineCallbacks=True) - def get_presence_states(self, user_localparts): + batches = ( + presence_states[i:i + 50] + for i in xrange(0, len(presence_states), 50) + ) + for states in batches: + args = [stream_id] + args.extend(s.user_id for s in states) + txn.execute( + sql % (",".join("?" for _ in states),), + args + ) + + @defer.inlineCallbacks + def get_presence_for_users(self, user_ids): rows = yield self._simple_select_many_batch( - table="presence", + table="presence_stream", column="user_id", - iterable=user_localparts, - retcols=("user_id", "state", "status_msg", "mtime",), - desc="get_presence_states", + iterable=user_ids, + keyvalues={}, + retcols=( + "user_id", + "state", + "last_active", + "last_federation_update", + "last_user_sync", + "status_msg", + "currently_active", + ), ) - defer.returnValue({ - row["user_id"]: { - "state": row["state"], - "status_msg": row["status_msg"], - "mtime": row["mtime"], - } - for row in rows - }) + for row in rows: + row["currently_active"] = bool(row["currently_active"]) - @defer.inlineCallbacks - def set_presence_state(self, user_localpart, new_state): - res = yield self._simple_update_one( - table="presence", - keyvalues={"user_id": user_localpart}, - updatevalues={"state": new_state["state"], - "status_msg": new_state["status_msg"], - "mtime": self._clock.time_msec()}, - desc="set_presence_state", - ) + defer.returnValue([UserPresenceState(**row) for row in rows]) - self.get_presence_state.invalidate((user_localpart,)) - defer.returnValue(res) + def get_current_presence_token(self): + return self._presence_id_gen.get_max_token() def allow_presence_visible(self, observed_localpart, observer_userid): return self._simple_insert( @@ -128,6 +183,7 @@ class PresenceStore(SQLBaseStore): desc="set_presence_list_accepted", ) self.get_presence_list_accepted.invalidate((observer_localpart,)) + self.get_presence_list_observers_accepted.invalidate((observed_userid,)) defer.returnValue(result) def get_presence_list(self, observer_localpart, accepted=None): @@ -154,6 +210,19 @@ class PresenceStore(SQLBaseStore): desc="get_presence_list_accepted", ) + @cachedInlineCallbacks() + def get_presence_list_observers_accepted(self, observed_userid): + user_localparts = yield self._simple_select_onecol( + table="presence_list", + keyvalues={"observed_user_id": observed_userid, "accepted": True}, + retcol="user_id", + desc="get_presence_list_accepted", + ) + + defer.returnValue([ + "@%s:%s" % (u, self.hs.hostname,) for u in user_localparts + ]) + @defer.inlineCallbacks def del_presence_list(self, observer_localpart, observed_userid): yield self._simple_delete_one( @@ -163,3 +232,4 @@ class PresenceStore(SQLBaseStore): desc="del_presence_list", ) self.get_presence_list_accepted.invalidate((observer_localpart,)) + self.get_presence_list_observers_accepted.invalidate((observed_userid,)) -- cgit 1.4.1 From 112283e23005bdaa17b6184cb55fd786facff47d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Feb 2016 10:11:43 +0000 Subject: Prefix TS fields with _ts --- synapse/handlers/presence.py | 54 +++++++++++----------- synapse/storage/__init__.py | 4 +- synapse/storage/presence.py | 23 ++++----- .../storage/schema/delta/30/presence_stream.sql | 6 +-- 4 files changed, 44 insertions(+), 43 deletions(-) (limited to 'synapse/storage/presence.py') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 3137c23509..d296953651 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -127,24 +127,24 @@ class PresenceHandler(BaseHandler): self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_active + IDLE_TIMER, + then=state.last_active_ts + IDLE_TIMER, ) self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_user_sync + SYNC_ONLINE_TIMEOUT, + then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, ) if self.hs.is_mine_id(state.user_id): self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_federation_update + FEDERATION_PING_INTERVAL, + then=state.last_federation_update_ts + FEDERATION_PING_INTERVAL, ) else: self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_federation_update + FEDERATION_TIMEOUT, + then=state.last_federation_update_ts + FEDERATION_TIMEOUT, ) # Set of users who have presence in the `user_to_current_state` that @@ -225,7 +225,7 @@ class PresenceHandler(BaseHandler): self.wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_active + IDLE_TIMER + then=new_state.last_active_ts + IDLE_TIMER ) if new_state.state != PresenceState.OFFLINE: @@ -233,14 +233,14 @@ class PresenceHandler(BaseHandler): self.wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_user_sync + SYNC_ONLINE_TIMEOUT + then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT ) - last_federate = new_state.last_federation_update + last_federate = new_state.last_federation_update_ts if now - last_federate > FEDERATION_PING_INTERVAL: # Been a while since we've poked remote servers new_state = new_state.copy_and_replace( - last_federation_update=now, + last_federation_update_ts=now, ) to_federation_ping[user_id] = new_state @@ -248,11 +248,11 @@ class PresenceHandler(BaseHandler): self.wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_federation_update + FEDERATION_TIMEOUT + then=new_state.last_federation_update_ts + FEDERATION_TIMEOUT ) if new_state.state == PresenceState.ONLINE: - currently_active = now - new_state.last_active < LAST_ACTIVE_GRANULARITY + currently_active = now - new_state.last_active_ts < LAST_ACTIVE_GRANULARITY new_state = new_state.copy_and_replace( currently_active=currently_active, ) @@ -260,7 +260,7 @@ class PresenceHandler(BaseHandler): # Check whether the change was something worth notifying about if should_notify(prev_state, new_state): new_state.copy_and_replace( - last_federation_update=now, + last_federation_update_ts=now, ) to_notify[user_id] = new_state @@ -309,18 +309,18 @@ class PresenceHandler(BaseHandler): if self.hs.is_mine_id(user_id): if state.state == PresenceState.ONLINE: - if now - state.last_active > IDLE_TIMER: + if now - state.last_active_ts > IDLE_TIMER: # Currently online, but last activity ages ago so auto # idle changes[user_id] = state.copy_and_replace( state=PresenceState.UNAVAILABLE, ) - elif now - state.last_active > LAST_ACTIVE_GRANULARITY: + elif now - state.last_active_ts > LAST_ACTIVE_GRANULARITY: # So that we send down a notification that we've # stopped updating. changes[user_id] = state - if now - state.last_federation_update > FEDERATION_PING_INTERVAL: + if now - state.last_federation_update_ts > FEDERATION_PING_INTERVAL: # Need to send ping to other servers to ensure they don't # timeout and set us to offline changes[user_id] = state @@ -328,7 +328,7 @@ class PresenceHandler(BaseHandler): # If there are have been no sync for a while (and none ongoing), # set presence to offline if not self.user_to_num_current_syncs.get(user_id, 0): - if now - state.last_user_sync > SYNC_ONLINE_TIMEOUT: + if now - state.last_user_sync_ts > SYNC_ONLINE_TIMEOUT: changes[user_id] = state.copy_and_replace( state=PresenceState.OFFLINE, status_msg=None, @@ -337,7 +337,7 @@ class PresenceHandler(BaseHandler): # We expect to be poked occaisonally by the other side. # This is to protect against forgetful/buggy servers, so that # no one gets stuck online forever. - if now - state.last_federation_update > FEDERATION_TIMEOUT: + if now - state.last_federation_update_ts > FEDERATION_TIMEOUT: # The other side seems to have disappeared. changes[user_id] = state.copy_and_replace( state=PresenceState.OFFLINE, @@ -356,7 +356,7 @@ class PresenceHandler(BaseHandler): prev_state = yield self.current_state_for_user(user_id) new_fields = { - "last_active": self.clock.time_msec(), + "last_active_ts": self.clock.time_msec(), } if prev_state.state == PresenceState.UNAVAILABLE: new_fields["state"] = PresenceState.ONLINE @@ -388,12 +388,12 @@ class PresenceHandler(BaseHandler): # just update the last sync times. yield self._update_states([prev_state.copy_and_replace( state=PresenceState.ONLINE, - last_active=self.clock.time_msec(), - last_user_sync=self.clock.time_msec(), + last_active_ts=self.clock.time_msec(), + last_user_sync_ts=self.clock.time_msec(), )]) else: yield self._update_states([prev_state.copy_and_replace( - last_user_sync=self.clock.time_msec(), + last_user_sync_ts=self.clock.time_msec(), )]) @defer.inlineCallbacks @@ -403,7 +403,7 @@ class PresenceHandler(BaseHandler): prev_state = yield self.current_state_for_user(user_id) yield self._update_states([prev_state.copy_and_replace( - last_user_sync=self.clock.time_msec(), + last_user_sync_ts=self.clock.time_msec(), )]) @contextmanager @@ -553,12 +553,12 @@ class PresenceHandler(BaseHandler): new_fields = { "state": presence_state, - "last_federation_update": now, + "last_federation_update_ts": now, } last_active_ago = push.get("last_active_ago", None) if last_active_ago is not None: - new_fields["last_active"] = now - last_active_ago + new_fields["last_active_ts"] = now - last_active_ago new_fields["status_msg"] = push.get("status_msg", None) @@ -632,7 +632,7 @@ class PresenceHandler(BaseHandler): } if presence == PresenceState.ONLINE: - new_fields["last_active"] = self.clock.time_msec() + new_fields["last_active_ts"] = self.clock.time_msec() yield self._update_states([prev_state.copy_and_replace(**new_fields)]) @@ -823,7 +823,7 @@ def should_notify(old_state, new_state): if new_state.currently_active != old_state.currently_active: return True - if new_state.last_active - old_state.last_active > LAST_ACTIVE_GRANULARITY: + if new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY: # Always notify for a transition where last active gets bumped. return True @@ -841,8 +841,8 @@ def _format_user_presence_state(state, now): "presence": state.state, "user_id": state.user_id, } - if state.last_active: - content["last_active_ago"] = now - state.last_active + if state.last_active_ts: + content["last_active_ago"] = now - state.last_active_ts if state.status_msg and state.state != PresenceState.OFFLINE: content["status_msg"] = state.status_msg if state.state == PresenceState.ONLINE: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8c3cf9e801..fcb968e8f4 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -203,8 +203,8 @@ class DataStore(RoomMemberStore, RoomStore, """ sql = ( - "SELECT user_id, state, last_active, last_federation_update," - " last_user_sync, status_msg, currently_active FROM presence_stream" + "SELECT user_id, state, last_active_ts, last_federation_update_ts," + " last_user_sync_ts, status_msg, currently_active FROM presence_stream" " WHERE state != ?" ) sql = self.database_engine.convert_param_style(sql) diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index b133979102..70ece56548 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -22,8 +22,9 @@ from twisted.internet import defer class UserPresenceState(namedtuple("UserPresenceState", - ("user_id", "state", "last_active", "last_federation_update", - "last_user_sync", "status_msg", "currently_active"))): + ("user_id", "state", "last_active_ts", + "last_federation_update_ts", "last_user_sync_ts", + "status_msg", "currently_active"))): """Represents the current presence state of the user. user_id (str) @@ -46,9 +47,9 @@ class UserPresenceState(namedtuple("UserPresenceState", return cls( user_id=user_id, state=PresenceState.OFFLINE, - last_active=0, - last_federation_update=0, - last_user_sync=0, + last_active_ts=0, + last_federation_update_ts=0, + last_user_sync_ts=0, status_msg=None, currently_active=False, ) @@ -82,9 +83,9 @@ class PresenceStore(SQLBaseStore): "stream_id": stream_id, "user_id": state.user_id, "state": state.state, - "last_active": state.last_active, - "last_federation_update": state.last_federation_update, - "last_user_sync": state.last_user_sync, + "last_active_ts": state.last_active_ts, + "last_federation_update_ts": state.last_federation_update_ts, + "last_user_sync_ts": state.last_user_sync_ts, "status_msg": state.status_msg, "currently_active": state.currently_active, } @@ -121,9 +122,9 @@ class PresenceStore(SQLBaseStore): retcols=( "user_id", "state", - "last_active", - "last_federation_update", - "last_user_sync", + "last_active_ts", + "last_federation_update_ts", + "last_user_sync_ts", "status_msg", "currently_active", ), diff --git a/synapse/storage/schema/delta/30/presence_stream.sql b/synapse/storage/schema/delta/30/presence_stream.sql index 14f5e3d30a..606bbb037d 100644 --- a/synapse/storage/schema/delta/30/presence_stream.sql +++ b/synapse/storage/schema/delta/30/presence_stream.sql @@ -18,9 +18,9 @@ stream_id BIGINT, user_id TEXT, state TEXT, - last_active BIGINT, - last_federation_update BIGINT, - last_user_sync BIGINT, + last_active_ts BIGINT, + last_federation_update_ts BIGINT, + last_user_sync_ts BIGINT, status_msg TEXT, currently_active BOOLEAN ); -- cgit 1.4.1