[cloud] Add more configurable backoff implementations to CloudRetry/AWSRetry (#27251)

lib/ansible/module_utils/cloud.py

@@ -31,10 +31,15 @@ The 'cloud' module provides the following common classes:
           provide a backoff/retry decorator based on status codes.
 
         - Example using the AWSRetry class which inherits from CloudRetry.
-          @AWSRetry.retry(tries=20, delay=2, backoff=2)
+
+          @AWSRetry.exponential_backoff(retries=10, delay=3)
+          get_ec2_security_group_ids_from_names()
+
+          @AWSRetry.jittered_backoff()
           get_ec2_security_group_ids_from_names()
 
 """
+import random
 from functools import wraps
 import syslog
 import time
@@ -42,6 +47,60 @@ import time
 from ansible.module_utils.pycompat24 import get_exception
 
 
+def _exponential_backoff(retries=10, delay=2, backoff=2, max_delay=60):
+    """ Customizable exponential backoff strategy.
+    Args:
+        retries (int): Maximum number of times to retry a request.
+        delay (float): Initial (base) delay.
+        backoff (float): base of the exponent to use for exponential
+            backoff.
+        max_delay (int): Optional. If provided each delay generated is capped
+            at this amount. Defaults to 60 seconds.
+    Returns:
+        Callable that returns a generator. This generator yields durations in
+        seconds to be used as delays for an exponential backoff strategy.
+    Usage:
+        >>> backoff = _exponential_backoff()
+        >>> backoff
+        <function backoff_backoff at 0x7f0d939facf8>
+        >>> list(backoff())
+        [2, 4, 8, 16, 32, 60, 60, 60, 60, 60]
+    """
+    def backoff_gen():
+        for retry in range(0, retries):
+            sleep = delay * backoff ** retry
+            yield sleep if max_delay is None else min(sleep, max_delay)
+    return backoff_gen
+
+
+def _full_jitter_backoff(retries=10, delay=3, max_delay=60, _random=random):
+    """ Implements the "Full Jitter" backoff strategy described here
+    https://www.awsarchitectureblog.com/2015/03/backoff.html
+    Args:
+        retries (int): Maximum number of times to retry a request.
+        delay (float): Approximate number of seconds to sleep for the first
+            retry.
+        max_delay (int): The maximum number of seconds to sleep for any retry.
+            _random (random.Random or None): Makes this generator testable by
+            allowing developers to explicitly pass in the a seeded Random.
+    Returns:
+        Callable that returns a generator. This generator yields durations in
+        seconds to be used as delays for a full jitter backoff strategy.
+    Usage:
+        >>> backoff = _full_jitter_backoff(retries=5)
+        >>> backoff
+        <function backoff_backoff at 0x7f0d939facf8>
+        >>> list(backoff())
+        [3, 6, 5, 23, 38]
+        >>> list(backoff())
+        [2, 1, 6, 6, 31]
+    """
+    def backoff_gen():
+        for retry in range(0, retries):
+            yield _random.randint(0, min(max_delay, delay * 2 ** retry))
+    return backoff_gen
+
+
 class CloudRetry(object):
     """ CloudRetry can be used by any cloud provider, in order to implement a
         backoff algorithm/retry effect based on Status Code from Exceptions.
@@ -67,22 +126,18 @@ class CloudRetry(object):
         pass
 
     @classmethod
-    def backoff(cls, tries=10, delay=3, backoff=1.1):
+    def _backoff(cls, backoff_strategy):
-        """ Retry calling the Cloud decorated function using an exponential backoff.
+        """ Retry calling the Cloud decorated function using the provided
-        Kwargs:
+        backoff strategy.
-            tries (int): Number of times to try (not retry) before giving up
+        Args:
-                default=10
+            backoff_strategy (callable): Callable that returns a generator. The
-            delay (int): Initial delay between retries in seconds
+            generator should yield sleep times for each retry of the decorated
-                default=3
+            function.
-            backoff (int): backoff multiplier e.g. value of 2 will double the delay each retry
-                default=2
-
         """
         def deco(f):
             @wraps(f)
             def retry_func(*args, **kwargs):
-                max_tries, max_delay = tries, delay
+                for delay in backoff_strategy():
-                while max_tries > 1:
                     try:
                         return f(*args, **kwargs)
                     except Exception:
@@ -90,11 +145,9 @@ class CloudRetry(object):
                         if isinstance(e, cls.base_class):
                             response_code = cls.status_code_from_exception(e)
                             if cls.found(response_code):
-                                msg = "{0}: Retrying in {1} seconds...".format(str(e), max_delay)
+                                msg = "{0}: Retrying in {1} seconds...".format(str(e), delay)
                                 syslog.syslog(syslog.LOG_INFO, msg)
-                                time.sleep(max_delay)
+                                time.sleep(delay)
-                                max_tries -= 1
-                                max_delay *= backoff
                             else:
                                 # Return original exception if exception is not a ClientError
                                 raise e
@@ -106,3 +159,62 @@ class CloudRetry(object):
             return retry_func  # true decorator
 
         return deco
+
+    @classmethod
+    def exponential_backoff(cls, retries=10, delay=3, backoff=2, max_delay=60):
+        """
+        Retry calling the Cloud decorated function using an exponential backoff.
+
+        Kwargs:
+            retries (int): Number of times to retry a failed request before giving up
+                default=10
+            delay (int or float): Initial delay between retries in seconds
+                default=3
+            backoff (int or float): backoff multiplier e.g. value of 2 will
+                double the delay each retry
+                default=1.1
+            max_delay (int or None): maximum amount of time to wait between retries.
+                default=60
+        """
+        return cls._backoff(_exponential_backoff(
+            retries=retries, delay=delay, backoff=backoff, max_delay=max_delay))
+
+    @classmethod
+    def jittered_backoff(cls, retries=10, delay=3, max_delay=60):
+        """
+        Retry calling the Cloud decorated function using a jittered backoff
+        strategy. More on this strategy here:
+
+        https://www.awsarchitectureblog.com/2015/03/backoff.html
+
+        Kwargs:
+            retries (int): Number of times to retry a failed request before giving up
+                default=10
+            delay (int): Initial delay between retries in seconds
+                default=3
+            max_delay (int): maximum amount of time to wait between retries.
+                default=60
+        """
+        return cls._backoff(_full_jitter_backoff(
+            retries=retries, delay=delay, max_delay=max_delay))
+
+    @classmethod
+    def backoff(cls, tries=10, delay=3, backoff=1.1):
+        """
+        Retry calling the Cloud decorated function using an exponential backoff.
+
+        Compatibility for the original implementation of CloudRetry.backoff that
+        did not provide configurable backoff strategies. Developers should use
+        CloudRetry.exponential_backoff instead.
+
+        Kwargs:
+            tries (int): Number of times to try (not retry) before giving up
+                default=10
+            delay (int or float): Initial delay between retries in seconds
+                default=3
+            backoff (int or float): backoff multiplier e.g. value of 2 will
+                double the delay each retry
+                default=1.1
+        """
+        return cls.exponential_backoff(
+            retries=tries - 1, delay=delay, backoff=backoff, max_delay=None)

test/units/module_utils/cloud/test_backoff.py

@@ -0,0 +1,47 @@
+import random
+
+from ansible.compat.tests import unittest
+from ansible.module_utils.cloud import _exponential_backoff, \
+    _full_jitter_backoff
+
+
+class ExponentialBackoffStrategyTestCase(unittest.TestCase):
+    def test_no_retries(self):
+        strategy = _exponential_backoff(retries=0)
+        result = list(strategy())
+        self.assertEquals(result, [], 'list should be empty')
+
+    def test_exponential_backoff(self):
+        strategy = _exponential_backoff(retries=5, delay=1, backoff=2)
+        result = list(strategy())
+        self.assertEquals(result, [1, 2, 4, 8, 16])
+
+    def test_max_delay(self):
+        strategy = _exponential_backoff(retries=7, delay=1, backoff=2, max_delay=60)
+        result = list(strategy())
+        self.assertEquals(result, [1, 2, 4, 8, 16, 32, 60])
+
+    def test_max_delay_none(self):
+        strategy = _exponential_backoff(retries=7, delay=1, backoff=2, max_delay=None)
+        result = list(strategy())
+        self.assertEquals(result, [1, 2, 4, 8, 16, 32, 64])
+
+
+class FullJitterBackoffStrategyTestCase(unittest.TestCase):
+    def test_no_retries(self):
+        strategy = _full_jitter_backoff(retries=0)
+        result = list(strategy())
+        self.assertEquals(result, [], 'list should be empty')
+
+    def test_full_jitter(self):
+        retries = 5
+        seed = 1
+
+        r = random.Random(seed)
+        expected = [r.randint(0, 2**i) for i in range(0, retries)]
+
+        strategy = _full_jitter_backoff(
+            retries=retries, delay=1, _random=random.Random(seed))
+        result = list(strategy())
+
+        self.assertEquals(result, expected)