cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

add retry with exponential backoff when we receive throttling error code from cloudformation

Browse source
This commit is contained in:
Mick Bass 2014-10-25 00:16:10 -07:00 committed by Matt Clay
parent cffb2bdee9
commit e9a0fad36b
1 changed files with 16 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
lib/ansible/modules/cloud/amazon/cloudformation.py
View file

@ -160,7 +160,7 @@ def stack_operation(cfn, stack_name, operation):
operation_complete = False operation_complete = False
while operation_complete == False: while operation_complete == False:
try: try:
stack = cfn.describe_stacks(stack_name)[0] stack = invoke_with_throttling_retries(cfn.describe_stacks, stack_name)[0]
existed.append('yes') existed.append('yes')
except: except:
if 'yes' in existed: if 'yes' in existed:
@ -189,6 +189,19 @@ def stack_operation(cfn, stack_name, operation):
time.sleep(5) time.sleep(5)
return result return result
IGNORE_CODE = 'Throttling'
MAX_RETRIES=3
def invoke_with_throttling_retries(function_ref, *argv):
retries=0
while True:
try:
retval=function_ref(*argv)
return retval
except boto.exception.BotoServerError, e:
if e.code != IGNORE_CODE or retries==MAX_RETRIES:
raise e
time.sleep(5 * (2**retries))
retries += 1
def main(): def main():
argument_spec = ec2_argument_spec() argument_spec = ec2_argument_spec()
@ -288,7 +301,7 @@ def main():
# and get the outputs of the stack # and get the outputs of the stack
if state == 'present' or update: if state == 'present' or update:
stack = cfn.describe_stacks(stack_name)[0] stack = invoke_with_throttling_retries(cfn.describe_stacks,stack_name)[0]
for output in stack.outputs: for output in stack.outputs:
stack_outputs[output.key] = output.value stack_outputs[output.key] = output.value
result['stack_outputs'] = stack_outputs result['stack_outputs'] = stack_outputs
@ -299,7 +312,7 @@ def main():
if state == 'absent': if state == 'absent':
try: try:
cfn.describe_stacks(stack_name) invoke_with_throttling_retries(cfn.describe_stacks,stack_name)
operation = 'DELETE' operation = 'DELETE'
except Exception, err: except Exception, err:
error_msg = boto_exception(err) error_msg = boto_exception(err)