From d79c9994f416ee5dab27a277fa729ffa5ee74ccc Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 28 May 2019 18:52:41 +0100
Subject: Add DB bg update to cleanup extremities.

Due to #5269 we may have extremities in our DB that we shouldn't have,
so lets add a cleanup task such to remove those.
---
 .../schema/delta/54/delete_forward_extremities.sql    | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 synapse/storage/schema/delta/54/delete_forward_extremities.sql

(limited to 'synapse/storage/schema/delta')

diff --git a/synapse/storage/schema/delta/54/delete_forward_extremities.sql b/synapse/storage/schema/delta/54/delete_forward_extremities.sql
new file mode 100644
index 0000000000..7056bd1d00
--- /dev/null
+++ b/synapse/storage/schema/delta/54/delete_forward_extremities.sql
@@ -0,0 +1,19 @@
+/* Copyright 2019 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+INSERT INTO background_updates (update_name, progress_json) VALUES
+  ('delete_soft_failed_extremities', '{}');
+
+CREATE TABLE _extremities_to_check AS SELECT event_id FROM event_forward_extremities;
-- 
cgit 1.4.1


From 640fcbb07f8dc7d89465734f009d8e0a458c2b17 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 30 May 2019 10:55:55 +0100
Subject: Fixup comments and logging

---
 synapse/storage/events.py                           | 21 ++++++++++++---------
 .../schema/delta/54/delete_forward_extremities.sql  |  3 +++
 2 files changed, 15 insertions(+), 9 deletions(-)

(limited to 'synapse/storage/schema/delta')

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index a9664928ca..418d88b8dc 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -2387,7 +2387,8 @@ class EventsStore(
             soft_failed_events_to_lookup = set()
 
             # First, we get `batch_size` events from the table, pulling out
-            # their prev events, if any, and their prev events rejection status.
+            # their successor events, if any, and their successor events
+            # rejection status.
             txn.execute(
                 """SELECT prev_event_id, event_id, internal_metadata,
                     rejections.event_id IS NOT NULL, events.outlier
@@ -2450,11 +2451,10 @@ class EventsStore(
                     if event_id in graph:
                         # Already handled this event previously, but we still
                         # want to record the edge.
-                        graph.setdefault(event_id, set()).add(prev_event_id)
-                        logger.info("Already handled")
+                        graph[event_id].add(prev_event_id)
                         continue
 
-                    graph.setdefault(event_id, set()).add(prev_event_id)
+                    graph[event_id] = {prev_event_id}
 
                     soft_failed = json.loads(metadata).get("soft_failed")
                     if soft_failed or rejected:
@@ -2474,8 +2474,6 @@ class EventsStore(
 
             to_delete.intersection_update(original_set)
 
-            logger.info("Deleting up to %d forward extremities", len(to_delete))
-
             deleted = self._simple_delete_many_txn(
                 txn=txn,
                 table="event_forward_extremities",
@@ -2484,7 +2482,11 @@ class EventsStore(
                 keyvalues={},
             )
 
-            logger.info("Deleted %d forward extremities", deleted)
+            logger.info(
+                "Deleted %d forward extremities of %d checked, to clean up #5269",
+                deleted,
+                len(original_set),
+            )
 
             if deleted:
                 # We now need to invalidate the caches of these rooms
@@ -2496,10 +2498,11 @@ class EventsStore(
                     keyvalues={},
                     retcols=("room_id",)
                 )
-                for row in rows:
+                room_ids = set(row["room_id"] for row in rows)
+                for room_id in room_ids:
                     txn.call_after(
                         self.get_latest_event_ids_in_room.invalidate,
-                        (row["room_id"],)
+                        (room_id,)
                     )
 
             self._simple_delete_many_txn(
diff --git a/synapse/storage/schema/delta/54/delete_forward_extremities.sql b/synapse/storage/schema/delta/54/delete_forward_extremities.sql
index 7056bd1d00..aa40f13da7 100644
--- a/synapse/storage/schema/delta/54/delete_forward_extremities.sql
+++ b/synapse/storage/schema/delta/54/delete_forward_extremities.sql
@@ -13,7 +13,10 @@
  * limitations under the License.
  */
 
+-- Start a background job to cleanup extremities that were incorrectly added
+-- by bug #5269.
 INSERT INTO background_updates (update_name, progress_json) VALUES
   ('delete_soft_failed_extremities', '{}');
 
+DROP TABLE IF EXISTS _extremities_to_check;  -- To make this delta schema file idempotent.
 CREATE TABLE _extremities_to_check AS SELECT event_id FROM event_forward_extremities;
-- 
cgit 1.4.1