diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index f5952d1fc0..874d41447a 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -300,6 +300,9 @@ class SQLBaseStore(object):
def inner_func(conn, *args, **kwargs):
with LoggingContext("runInteraction") as context:
+ if self.database_engine.is_connection_closed(conn):
+ conn.reconnect()
+
current_context.copy_to(context)
start = time.time() * 1000
txn_id = self._TXN_ID
@@ -322,12 +325,35 @@ class SQLBaseStore(object):
LoggingTransaction(txn, name, self.database_engine),
*args, **kwargs
)
+ except self.database_engine.module.OperationalError as e:
+ # This can happen if the database disappears mid
+ # transaction.
+ logger.warn(
+ "[TXN OPERROR] {%s} %s %d/%d",
+ name, e, i, N
+ )
+ if i < N:
+ i += 1
+ try:
+ conn.rollback()
+ except self.database_engine.module.Error as e1:
+ logger.warn(
+ "[TXN EROLL] {%s} %s",
+ name, e1,
+ )
+ continue
except self.database_engine.module.DatabaseError as e:
if self.database_engine.is_deadlock(e):
logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N)
if i < N:
i += 1
- conn.rollback()
+ try:
+ conn.rollback()
+ except self.database_engine.module.Error as e1:
+ logger.warn(
+ "[TXN EROLL] {%s} %s",
+ name, e1,
+ )
continue
raise
except Exception as e:
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 6f75245fa7..00dbae7b60 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -39,3 +39,6 @@ class PostgresEngine(object):
if isinstance(error, self.module.DatabaseError):
return error.pgcode in ["40001", "40P01"]
return False
+
+ def is_connection_closed(self, conn):
+ return bool(conn)
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 72c11df461..39828a597c 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -35,3 +35,6 @@ class Sqlite3Engine(object):
def is_deadlock(self, error):
return False
+
+ def is_connection_closed(self, conn):
+ return False
|
