diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index b50a0c445c..6e53538a52 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -35,6 +35,7 @@ from signedjson.sign import sign_json
import simplejson as json
import logging
+import random
import sys
import urllib
import urlparse
@@ -55,6 +56,9 @@ incoming_responses_counter = metrics.register_counter(
)
+MAX_RETRIES = 4
+
+
class MatrixFederationEndpointFactory(object):
def __init__(self, hs):
self.tls_server_context_factory = hs.tls_server_context_factory
@@ -119,7 +123,7 @@ class MatrixFederationHttpClient(object):
# XXX: Would be much nicer to retry only at the transaction-layer
# (once we have reliable transactions in place)
- retries_left = 5
+ retries_left = MAX_RETRIES
http_url_bytes = urlparse.urlunparse(
("", "", path_bytes, param_bytes, query_bytes, "")
@@ -180,7 +184,9 @@ class MatrixFederationHttpClient(object):
)
if retries_left and not timeout:
- yield sleep(2 ** (5 - retries_left))
+ delay = 5 ** (MAX_RETRIES + 1 - retries_left)
+ delay *= random.uniform(0.8, 1.4)
+ yield sleep(delay)
retries_left -= 1
else:
raise
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index a42138f556..2fe6814807 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -18,6 +18,7 @@ from twisted.internet import defer
from synapse.api.errors import CodeMessageException
import logging
+import random
logger = logging.getLogger(__name__)
@@ -85,8 +86,9 @@ def get_retry_limiter(destination, clock, store, **kwargs):
class RetryDestinationLimiter(object):
def __init__(self, destination, clock, store, retry_interval,
- min_retry_interval=5000, max_retry_interval=60 * 60 * 1000,
- multiplier_retry_interval=2,):
+ min_retry_interval=10 * 60 * 1000,
+ max_retry_interval=24 * 60 * 60 * 1000,
+ multiplier_retry_interval=5,):
"""Marks the destination as "down" if an exception is thrown in the
context, except for CodeMessageException with code < 500.
@@ -140,6 +142,7 @@ class RetryDestinationLimiter(object):
# We couldn't connect.
if self.retry_interval:
self.retry_interval *= self.multiplier_retry_interval
+ self.retry_interval *= int(random.uniform(0.8, 1.4))
if self.retry_interval >= self.max_retry_interval:
self.retry_interval = self.max_retry_interval
|
