2 files changed, 9 insertions, 0 deletions
diff --git a/changelog.d/12781.misc b/changelog.d/12781.misc
new file mode 100644
index 0000000000..8a04571617
--- /dev/null
+++ b/changelog.d/12781.misc
@@ -0,0 +1 @@
+Do not keep going if there are 5 back-to-back background update failures.
\ No newline at end of file
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index c2bbbb574e..37f2d6c644 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -282,12 +282,20 @@ class BackgroundUpdater:
self._running = True
+ back_to_back_failures = 0
+
try:
logger.info("Starting background schema updates")
while self.enabled:
try:
result = await self.do_next_background_update(sleep)
+ back_to_back_failures = 0
except Exception:
+ back_to_back_failures += 1
+ if back_to_back_failures >= 5:
+ raise RuntimeError(
+ "5 back-to-back background update failures; aborting."
+ )
logger.exception("Error doing update")
else:
if result:
|