From 4adaba9acf224e14171a8a4b9c98ef0791c4a1e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 Aug 2023 13:45:25 +0100 Subject: Fix rare deadlock when using read/write locks (#16133) --- changelog.d/16133.bugfix | 1 + .../80/02_read_write_locks_deadlock.sql.postgres | 37 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 changelog.d/16133.bugfix create mode 100644 synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres diff --git a/changelog.d/16133.bugfix b/changelog.d/16133.bugfix new file mode 100644 index 0000000000..ed8830692f --- /dev/null +++ b/changelog.d/16133.bugfix @@ -0,0 +1 @@ +Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0. diff --git a/synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres b/synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres new file mode 100644 index 0000000000..401c42e18a --- /dev/null +++ b/synapse/storage/schema/main/delta/80/02_read_write_locks_deadlock.sql.postgres @@ -0,0 +1,37 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- To avoid the possibility of a deadlock, lock the +-- `worker_read_write_locks_mode` table so that we serialize inserts/deletes +-- for a specific lock name/key. + +CREATE OR REPLACE FUNCTION delete_read_write_lock_parent_before() RETURNS trigger AS $$ +BEGIN + -- `PERFORM` is a `SELECT` which discards the rows. + PERFORM * FROM worker_read_write_locks_mode + WHERE + lock_name = OLD.lock_name + AND lock_key = OLD.lock_key + FOR UPDATE; + + RETURN OLD; +END +$$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS delete_read_write_lock_parent_before_trigger ON worker_read_write_locks; +CREATE TRIGGER delete_read_write_lock_parent_before_trigger BEFORE DELETE ON worker_read_write_locks + FOR EACH ROW + EXECUTE PROCEDURE delete_read_write_lock_parent_before(); -- cgit 1.4.1