From 063e1b22e62915ec77bfd3cb9477c29600acb568 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 15:06:00 +0100 Subject: Stop internal keys from getting into SynapseEvents --- synapse/storage/_base.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse/storage') diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 36cc57c1b8..75aab2d3b9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -294,6 +294,11 @@ class SQLBaseStore(object): def _parse_event_from_row(self, row_dict): d = copy.deepcopy({k: v for k, v in row_dict.items() if v}) + + d.pop("stream_ordering", None) + d.pop("topological_ordering", None) + d.pop("processed", None) + d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) del d["unrecognized_keys"] -- cgit 1.4.1 From e7ee0b9fc113b1fd29b8cb96eea7a00641e56887 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 21 Aug 2014 16:40:21 +0100 Subject: Change IM sync api to also return the current presence list. --- synapse/handlers/room.py | 24 +++++++++++++++++++++--- synapse/storage/stream.py | 5 ++--- 2 files changed, 23 insertions(+), 6 deletions(-) (limited to 'synapse/storage') diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d9809bd6de..899b653fb7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomConfigEvent ) from synapse.api.streams.event import EventStream, EventsStreamData +from synapse.handlers.presence import PresenceStreamData from synapse.util import stringutils from ._base import BaseHandler @@ -257,7 +258,19 @@ class MessageHandler(BaseHandler): membership_list=[Membership.INVITE, Membership.JOIN] ) - ret = [] + rooms_ret = [] + + now_rooms_token = yield self.store.get_room_events_max_id() + + # FIXME (erikj): Fix this. + presence_stream = PresenceStreamData(self.hs) + now_presence_token = yield presence_stream.max_token() + presence = yield presence_stream.get_rows( + user_id, 0, now_presence_token, None, None + ) + + # FIXME (erikj): We need to not generate this token, + now_token = "%s_%s" % (now_rooms_token, now_presence_token) for event in room_list: d = { @@ -268,14 +281,15 @@ class MessageHandler(BaseHandler): if event.membership == Membership.INVITE: d["inviter"] = event.user_id - ret.append(d) + rooms_ret.append(d) if event.membership != Membership.JOIN: continue try: messages, token = yield self.store.get_recent_events_for_room( event.room_id, - limit=50, + limit=10, + end_token=now_rooms_token, ) d["messages"] = { @@ -289,6 +303,10 @@ class MessageHandler(BaseHandler): except: logger.exception("Failed to get snapshot") + user = self.hs.parse_userid(user_id) + + ret = {"rooms": rooms_ret, "presence": presence[0], "end": now_token} + logger.debug("snapshot_all_rooms returning: %s", ret) defer.returnValue(ret) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e994017bf2..8bc502483a 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -249,11 +249,10 @@ class StreamStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_recent_events_for_room(self, room_id, limit, with_feedback=False): + def get_recent_events_for_room(self, room_id, limit, end_token, + with_feedback=False): # TODO (erikj): Handle compressed feedback - end_token = yield self.get_room_events_max_id() - sql = ( "SELECT * FROM events " "WHERE room_id = ? AND stream_ordering <= ? " -- cgit 1.4.1 From 808f663ed179dcb16a315810f7f0f8a7eec77e01 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 13:06:07 +0100 Subject: Don't return state event outlier's when paginating. --- synapse/storage/__init__.py | 7 ++++++- synapse/storage/schema/im.sql | 1 + synapse/storage/stream.py | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'synapse/storage') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7732906927..d06033b980 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -105,6 +105,11 @@ class DataStore(RoomMemberStore, RoomStore, "processed": True, } + if hasattr(event, "outlier"): + vals["outlier"] = event.outlier + else: + vals["outlier"] = False + if backfilled: if not self.min_token_deferred.called: yield self.min_token_deferred @@ -123,7 +128,7 @@ class DataStore(RoomMemberStore, RoomStore, except: logger.exception( "Failed to persist, probably duplicate: %s", - event_id + event.event_id ) return diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index ea04261ff0..39a1ed703e 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS events( content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, + outlier BOOL NOT NULL, CONSTRAINT ev_uniq UNIQUE (event_id) ); diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 8bc502483a..87ae961ccd 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -177,6 +177,7 @@ class StreamStore(SQLBaseStore): "((room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering < ? " + "AND e.outlier = 0 " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { "current": current_room_membership_sql, @@ -224,7 +225,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events " - "WHERE room_id = ? AND %(bounds)s " + "WHERE outlier = 0 AND room_id = ? AND %(bounds)s " "ORDER BY topological_ordering %(order)s, stream_ordering %(order)s %(limit)s " ) % {"bounds": bounds, "order": order, "limit": limit_str} -- cgit 1.4.1 From 9f514915afe813cfb531748e252479b6332e5d76 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 18:02:45 +0100 Subject: Add indices to schema --- synapse/handlers/presence.py | 4 ++++ synapse/storage/schema/im.sql | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) (limited to 'synapse/storage') diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 540e114b82..4ce77cbff1 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -187,6 +187,10 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def set_state(self, target_user, auth_user, state): + return + # TODO (erikj): Turn this back on. Why did we end up sending EDUs + # everywhere? + if not target_user.is_mine: raise SynapseError(400, "User is not hosted on this Home Server") diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 39a1ed703e..7f4e758892 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -26,6 +26,11 @@ CREATE TABLE IF NOT EXISTS events( CONSTRAINT ev_uniq UNIQUE (event_id) ); +CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + CREATE TABLE IF NOT EXISTS state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, @@ -34,6 +39,12 @@ CREATE TABLE IF NOT EXISTS state_events( prev_state TEXT ); +CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + CREATE TABLE IF NOT EXISTS current_state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, @@ -42,6 +53,11 @@ CREATE TABLE IF NOT EXISTS current_state_events( CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE ); +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + CREATE TABLE IF NOT EXISTS room_memberships( event_id TEXT NOT NULL, user_id TEXT NOT NULL, @@ -50,6 +66,10 @@ CREATE TABLE IF NOT EXISTS room_memberships( membership TEXT NOT NULL ); +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + CREATE TABLE IF NOT EXISTS feedback( event_id TEXT NOT NULL, feedback_type TEXT, -- cgit 1.4.1 From 537ecd4e9929cc66eec3747aa37ce8c19ad6aa0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 22 Aug 2014 18:12:38 +0100 Subject: Turn off spammy logging --- synapse/storage/roommember.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 89c87290cf..aca5cff737 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -145,7 +145,7 @@ class RoomMemberStore(SQLBaseStore): rows = yield self._execute_and_decode(sql, *where_values) - logger.debug("_get_members_query Got rows %s", rows) + # logger.debug("_get_members_query Got rows %s", rows) results = [self._parse_event_from_row(r) for r in rows] defer.returnValue(results) -- cgit 1.4.1 From 9d86c8c7a68fdbfa49df8bb027a112158df40bb8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sun, 24 Aug 2014 11:28:03 +0100 Subject: Add a unique constraint on the room hosts table --- synapse/storage/schema/im.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 7f4e758892..e92f21ef3b 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -98,5 +98,6 @@ CREATE TABLE IF NOT EXISTS rooms( CREATE TABLE IF NOT EXISTS room_hosts( room_id TEXT NOT NULL, - host TEXT NOT NULL + host TEXT NOT NULL, + CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE ); -- cgit 1.4.1 From be6abdff19ce8dcc5b5d727aba0142f9c1aae2ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Aug 2014 09:22:58 +0100 Subject: Order 'get_recent_events_for_room' correctly. --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 87ae961ccd..3a17a723fe 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -257,7 +257,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events " "WHERE room_id = ? AND stream_ordering <= ? " - "ORDER BY topological_ordering, stream_ordering DESC LIMIT ? " + "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " ) rows = yield self._execute_and_decode( -- cgit 1.4.1