Handle the fact that postgres databases can be restarted from under us

1 files changed, 27 insertions, 1 deletions

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index f5952d1fc0..874d41447a 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -300,6 +300,9 @@ class SQLBaseStore(object):
 
         def inner_func(conn, *args, **kwargs):
             with LoggingContext("runInteraction") as context:
+                if self.database_engine.is_connection_closed(conn):
+                    conn.reconnect()
+
                 current_context.copy_to(context)
                 start = time.time() * 1000
                 txn_id = self._TXN_ID
@@ -322,12 +325,35 @@ class SQLBaseStore(object):
                                 LoggingTransaction(txn, name, self.database_engine),
                                 *args, **kwargs
                             )
+                        except self.database_engine.module.OperationalError as e:
+                            # This can happen if the database disappears mid
+                            # transaction.
+                            logger.warn(
+                                "[TXN OPERROR] {%s} %s %d/%d",
+                                name, e, i, N
+                            )
+                            if i < N:
+                                i += 1
+                                try:
+                                    conn.rollback()
+                                except self.database_engine.module.Error as e1:
+                                    logger.warn(
+                                        "[TXN EROLL] {%s} %s",
+                                        name, e1,
+                                    )
+                                continue
                         except self.database_engine.module.DatabaseError as e:
                             if self.database_engine.is_deadlock(e):
                                 logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
                                 if i < N:
                                     i += 1
-                                    conn.rollback()
+                                    try:
+                                        conn.rollback()
+                                    except self.database_engine.module.Error as e1:
+                                        logger.warn(
+                                            "[TXN EROLL] {%s} %s",
+                                            name, e1,
+                                        )
                                     continue
                             raise
                 except Exception as e: