From 1db444cdc2790064e14432557b3b1295cc32f23d Mon Sep 17 00:00:00 2001
From: Joel Thompson <jathomps08@gmail.com>
Date: Sat, 16 Apr 2016 15:43:03 -0400
Subject: [PATCH] Add exponential backoff retries to ec2_elb_lb (#3379)

ec2_elb_lb doesn't react well to AWS API throttling errors. This
implements an exponential backoff operation around some of the AWS API
calls (with random jitter, in line with AWS recommendations) to make
this more resilient.
---
 .../modules/cloud/amazon/ec2_elb_lb.py        | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/lib/ansible/modules/cloud/amazon/ec2_elb_lb.py b/lib/ansible/modules/cloud/amazon/ec2_elb_lb.py
index f7c9bd424a1..e5eb2824977 100644
--- a/lib/ansible/modules/cloud/amazon/ec2_elb_lb.py
+++ b/lib/ansible/modules/cloud/amazon/ec2_elb_lb.py
@@ -358,6 +358,29 @@ try:
 except ImportError:
     HAS_BOTO = False
 
+import time
+import random
+
+def _throttleable_operation(max_retries):
+    def _operation_wrapper(op):
+        def _do_op(*args, **kwargs):
+            retry = 0
+            while True:
+                try:
+                    return op(*args, **kwargs)
+                except boto.exception.BotoServerError, e:
+                    if retry < max_retries and e.code in \
+                            ("Throttling", "RequestLimitExceeded"):
+                        retry = retry + 1
+                        time.sleep(min(random.random() * (2 ** retry), 300))
+                        continue
+                    else:
+                        raise
+        return _do_op
+    return _operation_wrapper
+
+
+_THROTTLING_RETRIES = 5
 
 class ElbManager(object):
     """Handles ELB creation and destruction"""
@@ -401,6 +424,7 @@ class ElbManager(object):
         self.elb = self._get_elb()
         self.ec2_conn = self._get_ec2_connection()
 
+    @_throttleable_operation(_THROTTLING_RETRIES)
     def ensure_ok(self):
         """Create the ELB"""
         if not self.elb:
@@ -544,6 +568,7 @@ class ElbManager(object):
 
         return info
 
+    @_throttleable_operation(_THROTTLING_RETRIES)
     def _wait_for_elb_removed(self):
         polling_increment_secs = 15
         max_retries = (self.wait_timeout / polling_increment_secs)
@@ -561,6 +586,7 @@ class ElbManager(object):
 
         return status_achieved
 
+    @_throttleable_operation(_THROTTLING_RETRIES)
     def _wait_for_elb_interface_removed(self):
         polling_increment_secs = 15
         max_retries = (self.wait_timeout / polling_increment_secs)
@@ -588,6 +614,7 @@ class ElbManager(object):
 
         return status_achieved
 
+    @_throttleable_operation(_THROTTLING_RETRIES)
     def _get_elb(self):
         elbs = self.elb_conn.get_all_load_balancers()
         for elb in elbs:
@@ -609,6 +636,7 @@ class ElbManager(object):
         except (boto.exception.NoAuthHandlerFound, StandardError), e:
             self.module.fail_json(msg=str(e))
 
+    @_throttleable_operation(_THROTTLING_RETRIES)
     def _delete_elb(self):
         # True if succeeds, exception raised if not
         result = self.elb_conn.delete_load_balancer(name=self.name)
@@ -625,6 +653,16 @@ class ElbManager(object):
                                                       subnets=self.subnets,
                                                       scheme=self.scheme)
         if self.elb:
+            # HACK: Work around a boto bug in which the listeners attribute is
+            # always set to the listeners argument to create_load_balancer, and
+            # not the complex_listeners
+            # We're not doing a self.elb = self._get_elb here because there
+            # might be eventual consistency issues and it doesn't necessarily
+            # make sense to wait until the ELB gets returned from the EC2 API.
+            # This is necessary in the event we hit the throttling errors and
+            # need to retry ensure_ok
+            # See https://github.com/boto/boto/issues/3526
+            self.elb.listeners = self.listeners
             self.changed = True
             self.status = 'created'