From bceec6591311eeefe94efcd4e40a116ee73aad8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 15:10:05 +0000 Subject: Slightly more aggressive retry timers at HTTP level --- synapse/http/matrixfederationclient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 6e53538a52..ca9591556d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -56,7 +56,7 @@ incoming_responses_counter = metrics.register_counter( ) -MAX_RETRIES = 4 +MAX_RETRIES = 10 class MatrixFederationEndpointFactory(object): @@ -184,7 +184,8 @@ class MatrixFederationHttpClient(object): ) if retries_left and not timeout: - delay = 5 ** (MAX_RETRIES + 1 - retries_left) + delay = 4 ** (MAX_RETRIES + 1 - retries_left) + delay = max(delay, 60) delay *= random.uniform(0.8, 1.4) yield sleep(delay) retries_left -= 1 -- cgit 1.5.1 From afdfd12bdff919ab230b394ba4e2e486f1652e37 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 16:57:06 +0000 Subject: Implement required method 'resumeProducing' --- synapse/http/matrixfederationclient.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index ca9591556d..5d80e0c4da 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -491,6 +491,9 @@ class _JsonProducer(object): def stopProducing(self): pass + def resumeProducing(self): + pass + def _flatten_response_never_received(e): if hasattr(e, "reasons"): -- cgit 1.5.1 From cf4ef5f3c7ae268a2a28c0e59eb738d6695ca865 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 18:26:50 +0000 Subject: Only retry federation requests for a long time for background requests --- synapse/federation/transport/client.py | 1 + synapse/http/matrixfederationclient.py | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 3d59e1c650..0e0cb7ebc6 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -136,6 +136,7 @@ class TransportLayerClient(object): path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, json_data_callback=json_data_callback, + long_retries=True, ) logger.debug( diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5d80e0c4da..e09a0bbe18 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -56,7 +56,8 @@ incoming_responses_counter = metrics.register_counter( ) -MAX_RETRIES = 10 +MAX_LONG_RETRIES = 10 +MAX_SHORT_RETRIES = 3 class MatrixFederationEndpointFactory(object): @@ -103,7 +104,7 @@ class MatrixFederationHttpClient(object): def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, - timeout=None): + timeout=None, long_retries=False): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [self.version_string] @@ -123,7 +124,10 @@ class MatrixFederationHttpClient(object): # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) - retries_left = MAX_RETRIES + if long_retries: + retries_left = MAX_LONG_RETRIES + else: + retries_left = MAX_SHORT_RETRIES http_url_bytes = urlparse.urlunparse( ("", "", path_bytes, param_bytes, query_bytes, "") @@ -184,9 +188,15 @@ class MatrixFederationHttpClient(object): ) if retries_left and not timeout: - delay = 4 ** (MAX_RETRIES + 1 - retries_left) - delay = max(delay, 60) - delay *= random.uniform(0.8, 1.4) + if long_retries: + delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left) + delay = max(delay, 60) + delay *= random.uniform(0.8, 1.4) + else: + delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left) + delay = max(delay, 2) + delay *= random.uniform(0.8, 1.4) + yield sleep(delay) retries_left -= 1 else: @@ -237,7 +247,8 @@ class MatrixFederationHttpClient(object): headers_dict[b"Authorization"] = auth_headers @defer.inlineCallbacks - def put_json(self, destination, path, data={}, json_data_callback=None): + def put_json(self, destination, path, data={}, json_data_callback=None, + long_retries=False): """ Sends the specifed json data using PUT Args: @@ -273,6 +284,7 @@ class MatrixFederationHttpClient(object): path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, + long_retries=long_retries, ) if 200 <= response.code < 300: -- cgit 1.5.1 From cbf3cd61515d101d57124388d629162f4983783a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 18:29:29 +0000 Subject: Add comment --- synapse/http/matrixfederationclient.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index e09a0bbe18..614c06a6d7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -259,6 +259,8 @@ class MatrixFederationHttpClient(object): the request body. This will be encoded as JSON. json_data_callback (callable): A callable returning the dict to use as the request body. + long_retries (bool): A boolean that indicates whether we should + retry for a short or long time. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result -- cgit 1.5.1 From 2f2bbb4d063501d9e07e824fb4533b62c55b93ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Nov 2015 09:34:58 +0000 Subject: Use min and not max to set an upper bound on retry interval --- synapse/http/matrixfederationclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 614c06a6d7..4e6572df72 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -190,11 +190,11 @@ class MatrixFederationHttpClient(object): if retries_left and not timeout: if long_retries: delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left) - delay = max(delay, 60) + delay = min(delay, 60) delay *= random.uniform(0.8, 1.4) else: delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left) - delay = max(delay, 2) + delay = min(delay, 2) delay *= random.uniform(0.8, 1.4) yield sleep(delay) -- cgit 1.5.1 From 6408541075078023b20d4b1c46c2c9163bda3b52 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Nov 2015 17:15:44 +0000 Subject: Don't limit connections to perspective servers --- synapse/crypto/keyring.py | 38 +++++++++++++++------------------- synapse/http/matrixfederationclient.py | 5 ++++- 2 files changed, 21 insertions(+), 22 deletions(-) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 8b6a59866f..ee7a5e3106 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -381,28 +381,24 @@ class Keyring(object): def get_server_verify_key_v2_indirect(self, server_names_and_key_ids, perspective_name, perspective_keys): - limiter = yield get_retry_limiter( - perspective_name, self.clock, self.store - ) - - with limiter: - # TODO(mark): Set the minimum_valid_until_ts to that needed by - # the events being validated or the current time if validating - # an incoming request. - query_response = yield self.client.post_json( - destination=perspective_name, - path=b"/_matrix/key/v2/query", - data={ - u"server_keys": { - server_name: { - key_id: { - u"minimum_valid_until_ts": 0 - } for key_id in key_ids - } - for server_name, key_ids in server_names_and_key_ids + # TODO(mark): Set the minimum_valid_until_ts to that needed by + # the events being validated or the current time if validating + # an incoming request. + query_response = yield self.client.post_json( + destination=perspective_name, + path=b"/_matrix/key/v2/query", + data={ + u"server_keys": { + server_name: { + key_id: { + u"minimum_valid_until_ts": 0 + } for key_id in key_ids } - }, - ) + for server_name, key_ids in server_names_and_key_ids + } + }, + long_requests=True, + ) keys = {} diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 4e6572df72..042793e13d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -302,7 +302,7 @@ class MatrixFederationHttpClient(object): defer.returnValue(json.loads(body)) @defer.inlineCallbacks - def post_json(self, destination, path, data={}): + def post_json(self, destination, path, data={}, long_requests=True): """ Sends the specifed json data using POST Args: @@ -311,6 +311,8 @@ class MatrixFederationHttpClient(object): path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. + long_retries (bool): A boolean that indicates whether we should + retry for a short or long time. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result @@ -330,6 +332,7 @@ class MatrixFederationHttpClient(object): path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, + long_requests=True, ) if 200 <= response.code < 300: -- cgit 1.5.1 From 0eabfa55f6739871ebf609a4f2aa128e72ae4bf0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Nov 2015 17:17:58 +0000 Subject: Fix typo --- synapse/crypto/keyring.py | 2 +- synapse/http/matrixfederationclient.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/http/matrixfederationclient.py') diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index ee7a5e3106..bc5bb5cdb1 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -397,7 +397,7 @@ class Keyring(object): for server_name, key_ids in server_names_and_key_ids } }, - long_requests=True, + long_retries=True, ) keys = {} diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 042793e13d..b7b7c2cce8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -302,7 +302,7 @@ class MatrixFederationHttpClient(object): defer.returnValue(json.loads(body)) @defer.inlineCallbacks - def post_json(self, destination, path, data={}, long_requests=True): + def post_json(self, destination, path, data={}, long_retries=True): """ Sends the specifed json data using POST Args: @@ -332,7 +332,7 @@ class MatrixFederationHttpClient(object): path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, - long_requests=True, + long_retries=True, ) if 200 <= response.code < 300: -- cgit 1.5.1