From 84e6b4001f22b0e8c2f806053189fcdb1e85205b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 11 May 2015 18:01:31 +0100 Subject: Initial hack at wiring together pagination and backfill --- synapse/storage/event_federation.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'synapse/storage/event_federation.py') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 74b4e23590..2b5424ced4 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -79,6 +79,28 @@ class EventFederationStore(SQLBaseStore): room_id, ) + def get_oldest_events_with_depth_in_room(self, room_id): + return self.runInteraction( + "get_oldest_events_with_depth_in_room", + self.get_oldest_events_with_depth_in_room_txn, + room_id, + ) + + def get_oldest_events_with_depth_in_room_txn(self, txn, room_id): + sql = ( + "SELECT b.event_id, MAX(e.depth) FROM events as e" + " INNER JOIN event_edges as g" + " ON g.event_id = e.event_id AND g.room_id = e.room_id" + " INNER JOIN event_backward_extremities as b" + " ON g.prev_event_id = b.event_id AND g.room_id = b.room_id" + " WHERE b.room_id = ? AND g.is_state is ?" + " GROUP BY b.event_id" + ) + + txn.execute(sql, (room_id, False,)) + + return dict(txn.fetchall()) + def _get_oldest_events_in_room_txn(self, txn, room_id): return self._simple_select_onecol_txn( txn, @@ -247,11 +269,13 @@ class EventFederationStore(SQLBaseStore): do_insert = depth < min_depth if min_depth else True if do_insert: - self._simple_insert_txn( + self._simple_upsert_txn( txn, table="room_depth", - values={ + keyvalues={ "room_id": room_id, + }, + values={ "min_depth": depth, }, ) -- cgit 1.5.1 From c1779a79bc6da69621d0034e582008e95db02dad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:41:50 +0100 Subject: Fix up _handle_prev_events to not try to insert duplicate rows --- synapse/storage/event_federation.py | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) (limited to 'synapse/storage/event_federation.py') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 2b5424ced4..6773e44688 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -330,31 +330,21 @@ class EventFederationStore(SQLBaseStore): txn.execute(query, (event_id, room_id)) - # Insert all the prev_events as a backwards thing, they'll get - # deleted in a second if they're incorrect anyway. - self._simple_insert_many_txn( - txn, - table="event_backward_extremities", - values=[ - { - "event_id": e_id, - "room_id": room_id, - } - for e_id, _ in prev_events - ], - ) - - # Also delete from the backwards extremities table all ones that - # reference events that we have already seen query = ( - "DELETE FROM event_backward_extremities WHERE EXISTS (" - "SELECT 1 FROM events " - "WHERE " - "event_backward_extremities.event_id = events.event_id " - "AND not events.outlier " - ")" + "INSERT INTO event_backward_extremities (event_id, room_id)" + " SELECT ?, ? WHERE NOT EXISTS (" + " SELECT 1 FROM event_backward_extremities" + " WHERE event_id = ? AND room_id = ?" + " )" + " AND NOT EXISTS (" + " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" + " )" ) - txn.execute(query) + + txn.executemany(query, [ + (e_id, room_id, e_id, room_id, e_id, room_id,) + for e_id, _ in prev_events + ]) txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id -- cgit 1.5.1 From e4eddf9b367bdd0384f9b834cb8ba75db4804ae1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:47:23 +0100 Subject: We do actually want to delete rows out of event_backward_extremities --- synapse/storage/event_federation.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'synapse/storage/event_federation.py') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 6773e44688..f807236eb3 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -336,16 +336,23 @@ class EventFederationStore(SQLBaseStore): " SELECT 1 FROM event_backward_extremities" " WHERE event_id = ? AND room_id = ?" " )" - " AND NOT EXISTS (" - " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" - " )" ) txn.executemany(query, [ - (e_id, room_id, e_id, room_id, e_id, room_id,) + (e_id, room_id, e_id, room_id, ) for e_id, _ in prev_events ]) + + # Also delete from the backwards extremities table all ones that + # reference events that we have already seen + query = ( + "DELETE FROM event_backward_extremities" + " WHERE event_id = ? AND room_id = ?" + ) + txn.executemany(query, [(e_id, room_id) for e_id, _ in prev_events]) + + txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id ) -- cgit 1.5.1 From 30c72d377ef4047f93a6210e25a92dc5272ea0e9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 14:47:40 +0100 Subject: Newlines --- synapse/storage/event_federation.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'synapse/storage/event_federation.py') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index f807236eb3..7ea0ee2323 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -343,7 +343,6 @@ class EventFederationStore(SQLBaseStore): for e_id, _ in prev_events ]) - # Also delete from the backwards extremities table all ones that # reference events that we have already seen query = ( @@ -352,7 +351,6 @@ class EventFederationStore(SQLBaseStore): ) txn.executemany(query, [(e_id, room_id) for e_id, _ in prev_events]) - txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id ) -- cgit 1.5.1 From 8b28209c6050656b998f7eb7fab8ac55ae9b019b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 May 2015 15:02:53 +0100 Subject: Err, delete the right stuff --- synapse/storage/event_federation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse/storage/event_federation.py') diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7ea0ee2323..a1982dfbb5 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -336,20 +336,21 @@ class EventFederationStore(SQLBaseStore): " SELECT 1 FROM event_backward_extremities" " WHERE event_id = ? AND room_id = ?" " )" + " AND NOT EXISTS (" + " SELECT 1 FROM events WHERE event_id = ? AND room_id = ?" + " )" ) txn.executemany(query, [ - (e_id, room_id, e_id, room_id, ) + (e_id, room_id, e_id, room_id, e_id, room_id, ) for e_id, _ in prev_events ]) - # Also delete from the backwards extremities table all ones that - # reference events that we have already seen query = ( "DELETE FROM event_backward_extremities" " WHERE event_id = ? AND room_id = ?" ) - txn.executemany(query, [(e_id, room_id) for e_id, _ in prev_events]) + txn.execute(query, (event_id, room_id)) txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id -- cgit 1.5.1