From 28d0a173dbb5a912e5c3f7f17ac0677677958d5b Mon Sep 17 00:00:00 2001 From: Dennis Benkert Date: Mon, 25 Jun 2018 18:39:32 +0200 Subject: [PATCH] [aws] add limit on number of CloudFormation stack events fetched by cloudformation module (#41840) * Add a module parameter to configure the max fetched AWS CFN stack events * Add version documentation for new configuration option * Increase default in order to make sure that enough are fetched by default. This align roughly with the limit of manageable resources in CloudFormation. --- .../modules/cloud/amazon/cloudformation.py | 42 +++++++++++-------- .../cloud/amazon/test_cloudformation.py | 13 +++--- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/lib/ansible/modules/cloud/amazon/cloudformation.py b/lib/ansible/modules/cloud/amazon/cloudformation.py index ca0d736e413..889b49da286 100644 --- a/lib/ansible/modules/cloud/amazon/cloudformation.py +++ b/lib/ansible/modules/cloud/amazon/cloudformation.py @@ -113,6 +113,11 @@ options: must be specified (but only one of them). If 'state' ispresent, the stack does exist, and neither 'template', 'template_body' nor 'template_url' are specified, the previous template will be reused. version_added: "2.5" + events_limit: + description: + - Maximum number of CloudFormation events to fetch from a stack when creating or updating it. + default: 200 + version_added: "2.7" author: "James S. Martin (@jsmartin)" extends_documentation_fragment: @@ -276,7 +281,7 @@ from ansible.module_utils.basic import AnsibleModule from ansible.module_utils._text import to_bytes, to_native -def get_stack_events(cfn, stack_name, token_filter=None): +def get_stack_events(cfn, stack_name, events_limit, token_filter=None): '''This event data was never correct, it worked as a side effect. So the v2.3 format is different.''' ret = {'events': [], 'log': []} @@ -284,7 +289,8 @@ def get_stack_events(cfn, stack_name, token_filter=None): pg = cfn.get_paginator( 'describe_stack_events' ).paginate( - StackName=stack_name + StackName=stack_name, + PaginationConfig={'MaxItems': events_limit} ) if token_filter is not None: events = list(pg.search( @@ -312,7 +318,7 @@ def get_stack_events(cfn, stack_name, token_filter=None): return ret -def create_stack(module, stack_params, cfn): +def create_stack(module, stack_params, cfn, events_limit): if 'TemplateBody' not in stack_params and 'TemplateURL' not in stack_params: module.fail_json(msg="Either 'template', 'template_body' or 'template_url' is required when the stack does not exist.") @@ -329,7 +335,7 @@ def create_stack(module, stack_params, cfn): try: cfn.create_stack(**stack_params) - result = stack_operation(cfn, stack_params['StackName'], 'CREATE', stack_params.get('ClientRequestToken', None)) + result = stack_operation(cfn, stack_params['StackName'], 'CREATE', events_limit, stack_params.get('ClientRequestToken', None)) except Exception as err: error_msg = boto_exception(err) module.fail_json(msg="Failed to create stack {0}: {1}.".format(stack_params.get('StackName'), error_msg), exception=traceback.format_exc()) @@ -343,7 +349,7 @@ def list_changesets(cfn, stack_name): return [cs['ChangeSetName'] for cs in res['Summaries']] -def create_changeset(module, stack_params, cfn): +def create_changeset(module, stack_params, cfn, events_limit): if 'TemplateBody' not in stack_params and 'TemplateURL' not in stack_params: module.fail_json(msg="Either 'template' or 'template_url' is required.") if module.params['changeset_name'] is not None: @@ -382,7 +388,7 @@ def create_changeset(module, stack_params, cfn): break # Lets not hog the cpu/spam the AWS API time.sleep(1) - result = stack_operation(cfn, stack_params['StackName'], 'CREATE_CHANGESET') + result = stack_operation(cfn, stack_params['StackName'], 'CREATE_CHANGESET', events_limit) result['warnings'] = ['Created changeset named %s for stack %s' % (changeset_name, stack_params['StackName']), 'You can execute it using: aws cloudformation execute-change-set --change-set-name %s' % cs['Id'], 'NOTE that dependencies on this stack might fail due to pending changes!'] @@ -398,7 +404,7 @@ def create_changeset(module, stack_params, cfn): return result -def update_stack(module, stack_params, cfn): +def update_stack(module, stack_params, cfn, events_limit): if 'TemplateBody' not in stack_params and 'TemplateURL' not in stack_params: stack_params['UsePreviousTemplate'] = True @@ -407,7 +413,7 @@ def update_stack(module, stack_params, cfn): # don't need to be updated. try: cfn.update_stack(**stack_params) - result = stack_operation(cfn, stack_params['StackName'], 'UPDATE', stack_params.get('ClientRequestToken', None)) + result = stack_operation(cfn, stack_params['StackName'], 'UPDATE', events_limit, stack_params.get('ClientRequestToken', None)) except Exception as err: error_msg = boto_exception(err) if 'No updates are to be performed.' in error_msg: @@ -439,7 +445,7 @@ def boto_supports_termination_protection(cfn): return hasattr(cfn, "update_termination_protection") -def stack_operation(cfn, stack_name, operation, op_token=None): +def stack_operation(cfn, stack_name, operation, events_limit, op_token=None): '''gets the status of a stack while it is created/updated/deleted''' existed = [] while True: @@ -450,15 +456,15 @@ def stack_operation(cfn, stack_name, operation, op_token=None): # If the stack previously existed, and now can't be found then it's # been deleted successfully. if 'yes' in existed or operation == 'DELETE': # stacks may delete fast, look in a few ways. - ret = get_stack_events(cfn, stack_name, op_token) + ret = get_stack_events(cfn, stack_name, events_limit, op_token) ret.update({'changed': True, 'output': 'Stack Deleted'}) return ret else: return {'changed': True, 'failed': True, 'output': 'Stack Not Found', 'exception': traceback.format_exc()} - ret = get_stack_events(cfn, stack_name, op_token) + ret = get_stack_events(cfn, stack_name, events_limit, op_token) if not stack: if 'yes' in existed or operation == 'DELETE': # stacks may delete fast, look in a few ways. - ret = get_stack_events(cfn, stack_name, op_token) + ret = get_stack_events(cfn, stack_name, events_limit, op_token) ret.update({'changed': True, 'output': 'Stack Deleted'}) return ret else: @@ -567,7 +573,8 @@ def main(): changeset_name=dict(default=None, required=False), role_arn=dict(default=None, required=False), tags=dict(default=None, type='dict'), - termination_protection=dict(default=None, type='bool') + termination_protection=dict(default=None, type='bool'), + events_limit=dict(default=200, type='int'), ) ) @@ -665,14 +672,14 @@ def main(): if state == 'present': if not stack_info: - result = create_stack(module, stack_params, cfn) + result = create_stack(module, stack_params, cfn, module.params.get('events_limit')) elif module.params.get('create_changeset'): - result = create_changeset(module, stack_params, cfn) + result = create_changeset(module, stack_params, cfn, module.params.get('events_limit')) else: if module.params.get('termination_protection') is not None: update_termination_protection(module, cfn, stack_params['StackName'], bool(module.params.get('termination_protection'))) - result = update_stack(module, stack_params, cfn) + result = update_stack(module, stack_params, cfn, module.params.get('events_limit')) # format the stack output @@ -709,7 +716,8 @@ def main(): cfn.delete_stack(StackName=stack_params['StackName']) else: cfn.delete_stack(StackName=stack_params['StackName'], RoleARN=stack_params['RoleARN']) - result = stack_operation(cfn, stack_params['StackName'], 'DELETE', stack_params.get('ClientRequestToken', None)) + result = stack_operation(cfn, stack_params['StackName'], 'DELETE', module.params.get('events_limit'), + stack_params.get('ClientRequestToken', None)) except Exception as err: module.fail_json(msg=boto_exception(err), exception=traceback.format_exc()) diff --git a/test/units/modules/cloud/amazon/test_cloudformation.py b/test/units/modules/cloud/amazon/test_cloudformation.py index 9ba0ca9aeb4..511c136ef38 100644 --- a/test/units/modules/cloud/amazon/test_cloudformation.py +++ b/test/units/modules/cloud/amazon/test_cloudformation.py @@ -44,6 +44,8 @@ bad_json_tpl = """{ } }""" +default_events_limit = 10 + class FakeModule(object): def __init__(self, **kwargs): @@ -68,7 +70,7 @@ def test_invalid_template_json(placeboify): } m = FakeModule(disable_rollback=False) with pytest.raises(Exception, message='Malformed JSON should cause the test to fail') as exc_info: - cfn_module.create_stack(m, params, connection) + cfn_module.create_stack(m, params, connection, default_events_limit) assert exc_info.match('FAIL') assert "ValidationError" in m.exit_kwargs['msg'] @@ -81,7 +83,7 @@ def test_client_request_token_s3_stack(maybe_sleep, placeboify): 'ClientRequestToken': '3faf3fb5-b289-41fc-b940-44151828f6cf', } m = FakeModule(disable_rollback=False) - result = cfn_module.create_stack(m, params, connection) + result = cfn_module.create_stack(m, params, connection, default_events_limit) assert result['changed'] assert len(result['events']) > 1 # require that the final recorded stack state was CREATE_COMPLETE @@ -97,7 +99,7 @@ def test_basic_s3_stack(maybe_sleep, placeboify): 'TemplateBody': basic_yaml_tpl } m = FakeModule(disable_rollback=False) - result = cfn_module.create_stack(m, params, connection) + result = cfn_module.create_stack(m, params, connection, default_events_limit) assert result['changed'] assert len(result['events']) > 1 # require that the final recorded stack state was CREATE_COMPLETE @@ -108,7 +110,7 @@ def test_basic_s3_stack(maybe_sleep, placeboify): def test_delete_nonexistent_stack(maybe_sleep, placeboify): connection = placeboify.client('cloudformation') - result = cfn_module.stack_operation(connection, 'ansible-test-nonexist', 'DELETE') + result = cfn_module.stack_operation(connection, 'ansible-test-nonexist', 'DELETE', default_events_limit) assert result['changed'] assert 'Stack does not exist.' in result['log'] @@ -124,7 +126,8 @@ def test_missing_template_body(placeboify): cfn_module.create_stack( module=m, stack_params={}, - cfn=None + cfn=None, + events_limit=default_events_limit ) assert exc_info.match('FAIL') assert not m.exit_args