From 4648dc9702f82c52d49b2c29e83a37a1a6d87a71 Mon Sep 17 00:00:00 2001 From: mzizzi Date: Tue, 8 Aug 2017 08:56:46 -0400 Subject: [PATCH] [cloud] Add more configurable backoff implementations to CloudRetry/AWSRetry (#27251) --- lib/ansible/module_utils/cloud.py | 146 ++++++++++++++++-- test/units/module_utils/cloud/__init__.py | 0 test/units/module_utils/cloud/test_backoff.py | 47 ++++++ 3 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 test/units/module_utils/cloud/__init__.py create mode 100644 test/units/module_utils/cloud/test_backoff.py diff --git a/lib/ansible/module_utils/cloud.py b/lib/ansible/module_utils/cloud.py index c0bb7b6cd07..52bb18e53ed 100644 --- a/lib/ansible/module_utils/cloud.py +++ b/lib/ansible/module_utils/cloud.py @@ -31,10 +31,15 @@ The 'cloud' module provides the following common classes: provide a backoff/retry decorator based on status codes. - Example using the AWSRetry class which inherits from CloudRetry. - @AWSRetry.retry(tries=20, delay=2, backoff=2) + + @AWSRetry.exponential_backoff(retries=10, delay=3) + get_ec2_security_group_ids_from_names() + + @AWSRetry.jittered_backoff() get_ec2_security_group_ids_from_names() """ +import random from functools import wraps import syslog import time @@ -42,6 +47,60 @@ import time from ansible.module_utils.pycompat24 import get_exception +def _exponential_backoff(retries=10, delay=2, backoff=2, max_delay=60): + """ Customizable exponential backoff strategy. + Args: + retries (int): Maximum number of times to retry a request. + delay (float): Initial (base) delay. + backoff (float): base of the exponent to use for exponential + backoff. + max_delay (int): Optional. If provided each delay generated is capped + at this amount. Defaults to 60 seconds. + Returns: + Callable that returns a generator. This generator yields durations in + seconds to be used as delays for an exponential backoff strategy. + Usage: + >>> backoff = _exponential_backoff() + >>> backoff + + >>> list(backoff()) + [2, 4, 8, 16, 32, 60, 60, 60, 60, 60] + """ + def backoff_gen(): + for retry in range(0, retries): + sleep = delay * backoff ** retry + yield sleep if max_delay is None else min(sleep, max_delay) + return backoff_gen + + +def _full_jitter_backoff(retries=10, delay=3, max_delay=60, _random=random): + """ Implements the "Full Jitter" backoff strategy described here + https://www.awsarchitectureblog.com/2015/03/backoff.html + Args: + retries (int): Maximum number of times to retry a request. + delay (float): Approximate number of seconds to sleep for the first + retry. + max_delay (int): The maximum number of seconds to sleep for any retry. + _random (random.Random or None): Makes this generator testable by + allowing developers to explicitly pass in the a seeded Random. + Returns: + Callable that returns a generator. This generator yields durations in + seconds to be used as delays for a full jitter backoff strategy. + Usage: + >>> backoff = _full_jitter_backoff(retries=5) + >>> backoff + + >>> list(backoff()) + [3, 6, 5, 23, 38] + >>> list(backoff()) + [2, 1, 6, 6, 31] + """ + def backoff_gen(): + for retry in range(0, retries): + yield _random.randint(0, min(max_delay, delay * 2 ** retry)) + return backoff_gen + + class CloudRetry(object): """ CloudRetry can be used by any cloud provider, in order to implement a backoff algorithm/retry effect based on Status Code from Exceptions. @@ -67,22 +126,18 @@ class CloudRetry(object): pass @classmethod - def backoff(cls, tries=10, delay=3, backoff=1.1): - """ Retry calling the Cloud decorated function using an exponential backoff. - Kwargs: - tries (int): Number of times to try (not retry) before giving up - default=10 - delay (int): Initial delay between retries in seconds - default=3 - backoff (int): backoff multiplier e.g. value of 2 will double the delay each retry - default=2 - + def _backoff(cls, backoff_strategy): + """ Retry calling the Cloud decorated function using the provided + backoff strategy. + Args: + backoff_strategy (callable): Callable that returns a generator. The + generator should yield sleep times for each retry of the decorated + function. """ def deco(f): @wraps(f) def retry_func(*args, **kwargs): - max_tries, max_delay = tries, delay - while max_tries > 1: + for delay in backoff_strategy(): try: return f(*args, **kwargs) except Exception: @@ -90,11 +145,9 @@ class CloudRetry(object): if isinstance(e, cls.base_class): response_code = cls.status_code_from_exception(e) if cls.found(response_code): - msg = "{0}: Retrying in {1} seconds...".format(str(e), max_delay) + msg = "{0}: Retrying in {1} seconds...".format(str(e), delay) syslog.syslog(syslog.LOG_INFO, msg) - time.sleep(max_delay) - max_tries -= 1 - max_delay *= backoff + time.sleep(delay) else: # Return original exception if exception is not a ClientError raise e @@ -106,3 +159,62 @@ class CloudRetry(object): return retry_func # true decorator return deco + + @classmethod + def exponential_backoff(cls, retries=10, delay=3, backoff=2, max_delay=60): + """ + Retry calling the Cloud decorated function using an exponential backoff. + + Kwargs: + retries (int): Number of times to retry a failed request before giving up + default=10 + delay (int or float): Initial delay between retries in seconds + default=3 + backoff (int or float): backoff multiplier e.g. value of 2 will + double the delay each retry + default=1.1 + max_delay (int or None): maximum amount of time to wait between retries. + default=60 + """ + return cls._backoff(_exponential_backoff( + retries=retries, delay=delay, backoff=backoff, max_delay=max_delay)) + + @classmethod + def jittered_backoff(cls, retries=10, delay=3, max_delay=60): + """ + Retry calling the Cloud decorated function using a jittered backoff + strategy. More on this strategy here: + + https://www.awsarchitectureblog.com/2015/03/backoff.html + + Kwargs: + retries (int): Number of times to retry a failed request before giving up + default=10 + delay (int): Initial delay between retries in seconds + default=3 + max_delay (int): maximum amount of time to wait between retries. + default=60 + """ + return cls._backoff(_full_jitter_backoff( + retries=retries, delay=delay, max_delay=max_delay)) + + @classmethod + def backoff(cls, tries=10, delay=3, backoff=1.1): + """ + Retry calling the Cloud decorated function using an exponential backoff. + + Compatibility for the original implementation of CloudRetry.backoff that + did not provide configurable backoff strategies. Developers should use + CloudRetry.exponential_backoff instead. + + Kwargs: + tries (int): Number of times to try (not retry) before giving up + default=10 + delay (int or float): Initial delay between retries in seconds + default=3 + backoff (int or float): backoff multiplier e.g. value of 2 will + double the delay each retry + default=1.1 + """ + return cls.exponential_backoff( + retries=tries - 1, delay=delay, backoff=backoff, max_delay=None) diff --git a/test/units/module_utils/cloud/__init__.py b/test/units/module_utils/cloud/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/units/module_utils/cloud/test_backoff.py b/test/units/module_utils/cloud/test_backoff.py new file mode 100644 index 00000000000..7a128f980e8 --- /dev/null +++ b/test/units/module_utils/cloud/test_backoff.py @@ -0,0 +1,47 @@ +import random + +from ansible.compat.tests import unittest +from ansible.module_utils.cloud import _exponential_backoff, \ + _full_jitter_backoff + + +class ExponentialBackoffStrategyTestCase(unittest.TestCase): + def test_no_retries(self): + strategy = _exponential_backoff(retries=0) + result = list(strategy()) + self.assertEquals(result, [], 'list should be empty') + + def test_exponential_backoff(self): + strategy = _exponential_backoff(retries=5, delay=1, backoff=2) + result = list(strategy()) + self.assertEquals(result, [1, 2, 4, 8, 16]) + + def test_max_delay(self): + strategy = _exponential_backoff(retries=7, delay=1, backoff=2, max_delay=60) + result = list(strategy()) + self.assertEquals(result, [1, 2, 4, 8, 16, 32, 60]) + + def test_max_delay_none(self): + strategy = _exponential_backoff(retries=7, delay=1, backoff=2, max_delay=None) + result = list(strategy()) + self.assertEquals(result, [1, 2, 4, 8, 16, 32, 64]) + + +class FullJitterBackoffStrategyTestCase(unittest.TestCase): + def test_no_retries(self): + strategy = _full_jitter_backoff(retries=0) + result = list(strategy()) + self.assertEquals(result, [], 'list should be empty') + + def test_full_jitter(self): + retries = 5 + seed = 1 + + r = random.Random(seed) + expected = [r.randint(0, 2**i) for i in range(0, retries)] + + strategy = _full_jitter_backoff( + retries=retries, delay=1, _random=random.Random(seed)) + result = list(strategy()) + + self.assertEquals(result, expected)