Handle the fact that postgres databases can be restarted from under us

3 files changed, 33 insertions, 1 deletions

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index f5952d1fc0..874d41447a 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -300,6 +300,9 @@ class SQLBaseStore(object):
 
         def inner_func(conn, *args, **kwargs):
             with LoggingContext("runInteraction") as context:
+                if self.database_engine.is_connection_closed(conn):
+                    conn.reconnect()
+
                 current_context.copy_to(context)
                 start = time.time() * 1000
                 txn_id = self._TXN_ID
@@ -322,12 +325,35 @@ class SQLBaseStore(object):
                                 LoggingTransaction(txn, name, self.database_engine),
                                 *args, **kwargs
                             )
+                        except self.database_engine.module.OperationalError as e:
+                            # This can happen if the database disappears mid
+                            # transaction.
+                            logger.warn(
+                                "[TXN OPERROR] {%s} %s %d/%d",
+                                name, e, i, N
+                            )
+                            if i < N:
+                                i += 1
+                                try:
+                                    conn.rollback()
+                                except self.database_engine.module.Error as e1:
+                                    logger.warn(
+                                        "[TXN EROLL] {%s} %s",
+                                        name, e1,
+                                    )
+                                continue
                         except self.database_engine.module.DatabaseError as e:
                             if self.database_engine.is_deadlock(e):
                                 logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
                                 if i < N:
                                     i += 1
-                                    conn.rollback()
+                                    try:
+                                        conn.rollback()
+                                    except self.database_engine.module.Error as e1:
+                                        logger.warn(
+                                            "[TXN EROLL] {%s} %s",
+                                            name, e1,
+                                        )
                                     continue
                             raise
                 except Exception as e:
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 6f75245fa7..00dbae7b60 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -39,3 +39,6 @@ class PostgresEngine(object):
         if isinstance(error, self.module.DatabaseError):
             return error.pgcode in ["40001", "40P01"]
         return False
+
+    def is_connection_closed(self, conn):
+        return bool(conn)
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 72c11df461..39828a597c 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -35,3 +35,6 @@ class Sqlite3Engine(object):
 
     def is_deadlock(self, error):
         return False
+
+    def is_connection_closed(self, conn):
+        return False