summary refs log tree commit diff
diff options
context:
space:
mode:
authorErik Johnston <erikj@matrix.org>2023-08-23 13:45:25 +0100
committerGitHub <noreply@github.com>2023-08-23 13:45:25 +0100
commit4adaba9acf224e14171a8a4b9c98ef0791c4a1e3 (patch)
tree91c244c5aa94ff423c3675b6910534b8213bbb4d
parentReduce DB contention on worker locks (#16160) (diff)
downloadsynapse-4adaba9acf224e14171a8a4b9c98ef0791c4a1e3.tar.xz
Fix rare deadlock when using read/write locks (#16133)
-rw-r--r--changelog.d/16133.bugfix1
-rw-r--r--synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres37
2 files changed, 38 insertions, 0 deletions
diff --git a/changelog.d/16133.bugfix b/changelog.d/16133.bugfix
new file mode 100644
index 0000000000..ed8830692f
--- /dev/null
+++ b/changelog.d/16133.bugfix
@@ -0,0 +1 @@
+Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0.
diff --git a/synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres b/synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres
new file mode 100644
index 0000000000..401c42e18a
--- /dev/null
+++ b/synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres
@@ -0,0 +1,37 @@
+/* Copyright 2023 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- To avoid the possibility of a deadlock, lock the
+-- `worker_read_write_locks_mode` table so that we serialize inserts/deletes
+-- for a specific lock name/key.
+
+CREATE OR REPLACE FUNCTION delete_read_write_lock_parent_before() RETURNS trigger AS $$
+BEGIN
+    -- `PERFORM` is a `SELECT` which discards the rows.
+    PERFORM * FROM worker_read_write_locks_mode
+        WHERE
+            lock_name = OLD.lock_name
+            AND lock_key = OLD.lock_key
+        FOR UPDATE;
+
+    RETURN OLD;
+END
+$$
+LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS delete_read_write_lock_parent_before_trigger ON worker_read_write_locks;
+CREATE TRIGGER delete_read_write_lock_parent_before_trigger BEFORE DELETE ON worker_read_write_locks
+    FOR EACH ROW
+    EXECUTE PROCEDURE delete_read_write_lock_parent_before();