From e9a0fad36bc3d6b2c446db25b31412461b24c9c0 Mon Sep 17 00:00:00 2001 From: Mick Bass Date: Sat, 25 Oct 2014 00:16:10 -0700 Subject: [PATCH] add retry with exponential backoff when we receive throttling error code from cloudformation --- .../modules/cloud/amazon/cloudformation.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/ansible/modules/cloud/amazon/cloudformation.py b/lib/ansible/modules/cloud/amazon/cloudformation.py index b382e3f05ff..6963fb6f1d4 100644 --- a/lib/ansible/modules/cloud/amazon/cloudformation.py +++ b/lib/ansible/modules/cloud/amazon/cloudformation.py @@ -160,7 +160,7 @@ def stack_operation(cfn, stack_name, operation): operation_complete = False while operation_complete == False: try: - stack = cfn.describe_stacks(stack_name)[0] + stack = invoke_with_throttling_retries(cfn.describe_stacks, stack_name)[0] existed.append('yes') except: if 'yes' in existed: @@ -189,6 +189,19 @@ def stack_operation(cfn, stack_name, operation): time.sleep(5) return result +IGNORE_CODE = 'Throttling' +MAX_RETRIES=3 +def invoke_with_throttling_retries(function_ref, *argv): + retries=0 + while True: + try: + retval=function_ref(*argv) + return retval + except boto.exception.BotoServerError, e: + if e.code != IGNORE_CODE or retries==MAX_RETRIES: + raise e + time.sleep(5 * (2**retries)) + retries += 1 def main(): argument_spec = ec2_argument_spec() @@ -288,7 +301,7 @@ def main(): # and get the outputs of the stack if state == 'present' or update: - stack = cfn.describe_stacks(stack_name)[0] + stack = invoke_with_throttling_retries(cfn.describe_stacks,stack_name)[0] for output in stack.outputs: stack_outputs[output.key] = output.value result['stack_outputs'] = stack_outputs @@ -299,7 +312,7 @@ def main(): if state == 'absent': try: - cfn.describe_stacks(stack_name) + invoke_with_throttling_retries(cfn.describe_stacks,stack_name) operation = 'DELETE' except Exception, err: error_msg = boto_exception(err)