Merge pull request #384 from matrix-org/erikj/shorter_retries

Only retry federation requests for a long time for background requests
This commit is contained in:
Erik Johnston 2015-11-17 18:46:04 +00:00
commit b697a842a5
5 changed files with 39 additions and 7 deletions

View file

@ -136,6 +136,7 @@ class TransportLayerClient(object):
path=PREFIX + "/send/%s/" % transaction.transaction_id, path=PREFIX + "/send/%s/" % transaction.transaction_id,
data=json_data, data=json_data,
json_data_callback=json_data_callback, json_data_callback=json_data_callback,
long_retries=True,
) )
logger.debug( logger.debug(

View file

@ -56,7 +56,8 @@ incoming_responses_counter = metrics.register_counter(
) )
MAX_RETRIES = 10 MAX_LONG_RETRIES = 10
MAX_SHORT_RETRIES = 3
class MatrixFederationEndpointFactory(object): class MatrixFederationEndpointFactory(object):
@ -103,7 +104,7 @@ class MatrixFederationHttpClient(object):
def _create_request(self, destination, method, path_bytes, def _create_request(self, destination, method, path_bytes,
body_callback, headers_dict={}, param_bytes=b"", body_callback, headers_dict={}, param_bytes=b"",
query_bytes=b"", retry_on_dns_fail=True, query_bytes=b"", retry_on_dns_fail=True,
timeout=None): timeout=None, long_retries=False):
""" Creates and sends a request to the given url """ Creates and sends a request to the given url
""" """
headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"User-Agent"] = [self.version_string]
@ -123,7 +124,10 @@ class MatrixFederationHttpClient(object):
# XXX: Would be much nicer to retry only at the transaction-layer # XXX: Would be much nicer to retry only at the transaction-layer
# (once we have reliable transactions in place) # (once we have reliable transactions in place)
retries_left = MAX_RETRIES if long_retries:
retries_left = MAX_LONG_RETRIES
else:
retries_left = MAX_SHORT_RETRIES
http_url_bytes = urlparse.urlunparse( http_url_bytes = urlparse.urlunparse(
("", "", path_bytes, param_bytes, query_bytes, "") ("", "", path_bytes, param_bytes, query_bytes, "")
@ -184,9 +188,15 @@ class MatrixFederationHttpClient(object):
) )
if retries_left and not timeout: if retries_left and not timeout:
delay = 4 ** (MAX_RETRIES + 1 - retries_left) if long_retries:
delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left)
delay = max(delay, 60) delay = max(delay, 60)
delay *= random.uniform(0.8, 1.4) delay *= random.uniform(0.8, 1.4)
else:
delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left)
delay = max(delay, 2)
delay *= random.uniform(0.8, 1.4)
yield sleep(delay) yield sleep(delay)
retries_left -= 1 retries_left -= 1
else: else:
@ -237,7 +247,8 @@ class MatrixFederationHttpClient(object):
headers_dict[b"Authorization"] = auth_headers headers_dict[b"Authorization"] = auth_headers
@defer.inlineCallbacks @defer.inlineCallbacks
def put_json(self, destination, path, data={}, json_data_callback=None): def put_json(self, destination, path, data={}, json_data_callback=None,
long_retries=False):
""" Sends the specifed json data using PUT """ Sends the specifed json data using PUT
Args: Args:
@ -248,6 +259,8 @@ class MatrixFederationHttpClient(object):
the request body. This will be encoded as JSON. the request body. This will be encoded as JSON.
json_data_callback (callable): A callable returning the dict to json_data_callback (callable): A callable returning the dict to
use as the request body. use as the request body.
long_retries (bool): A boolean that indicates whether we should
retry for a short or long time.
Returns: Returns:
Deferred: Succeeds when we get a 2xx HTTP response. The result Deferred: Succeeds when we get a 2xx HTTP response. The result
@ -273,6 +286,7 @@ class MatrixFederationHttpClient(object):
path.encode("ascii"), path.encode("ascii"),
body_callback=body_callback, body_callback=body_callback,
headers_dict={"Content-Type": ["application/json"]}, headers_dict={"Content-Type": ["application/json"]},
long_retries=long_retries,
) )
if 200 <= response.code < 300: if 200 <= response.code < 300:

View file

@ -197,6 +197,7 @@ class FederationTestCase(unittest.TestCase):
'pdu_failures': [], 'pdu_failures': [],
}, },
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
) )
@defer.inlineCallbacks @defer.inlineCallbacks
@ -228,6 +229,7 @@ class FederationTestCase(unittest.TestCase):
'pdu_failures': [], 'pdu_failures': [],
}, },
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
) )
@defer.inlineCallbacks @defer.inlineCallbacks

View file

@ -409,6 +409,7 @@ class PresenceInvitesTestCase(PresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -443,6 +444,7 @@ class PresenceInvitesTestCase(PresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -483,6 +485,7 @@ class PresenceInvitesTestCase(PresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -827,6 +830,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -843,6 +847,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1033,6 +1038,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1048,6 +1054,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1078,6 +1085,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1184,6 +1192,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase):
}, },
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1200,6 +1209,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase):
}, },
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1232,6 +1242,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase):
}, },
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1265,6 +1276,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase):
}, },
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -1297,6 +1309,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase):
}, },
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )

View file

@ -218,6 +218,7 @@ class TypingNotificationsTestCase(unittest.TestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )
@ -284,6 +285,7 @@ class TypingNotificationsTestCase(unittest.TestCase):
} }
), ),
json_data_callback=ANY, json_data_callback=ANY,
long_retries=True,
), ),
defer.succeed((200, "OK")) defer.succeed((200, "OK"))
) )