From 6f74fca238239fb163f6c4ef8249d7bc4a822cbe Mon Sep 17 00:00:00 2001 From: Tom De Keyser Date: Mon, 9 Sep 2019 19:08:21 +0200 Subject: [PATCH] New module for AWS Step Functions state machines (#59116) * add new module: aws_stepfunctions_state_machine * add integration tests for new module: aws_stepfunctions_state_machine * fix sanity checks * use files/ folder instead for integration test * rename role name in integration test * attempt further permissions * iam states prefix * iam integration test prefix * add iam policy for running step functions state machine actions * slightly increase iam permission scope * rename integration test folder to proper name * move main() method to end of file * move contents of integration-policy.json for state machines to compute-policy.json * make check_mode return proper changed value + add check_mode integration tests * rename module to aws_step_functions_state_machine * fix missed rename in integration test variable * add purge_tags option * bump to version 2.10 --- .../testing_policies/compute-policy.json | 17 ++ .../aws_step_functions_state_machine.py | 232 ++++++++++++++++++ .../aws_step_functions_state_machine/aliases | 2 + .../defaults/main.yml | 2 + .../files/alternative_state_machine.json | 10 + .../files/state_machine.json | 10 + .../state_machines_iam_trust_policy.json | 12 + .../tasks/main.yml | 185 ++++++++++++++ 8 files changed, 470 insertions(+) create mode 100644 lib/ansible/modules/cloud/amazon/aws_step_functions_state_machine.py create mode 100644 test/integration/targets/aws_step_functions_state_machine/aliases create mode 100644 test/integration/targets/aws_step_functions_state_machine/defaults/main.yml create mode 100644 test/integration/targets/aws_step_functions_state_machine/files/alternative_state_machine.json create mode 100644 test/integration/targets/aws_step_functions_state_machine/files/state_machine.json create mode 100644 test/integration/targets/aws_step_functions_state_machine/files/state_machines_iam_trust_policy.json create mode 100644 test/integration/targets/aws_step_functions_state_machine/tasks/main.yml diff --git a/hacking/aws_config/testing_policies/compute-policy.json b/hacking/aws_config/testing_policies/compute-policy.json index 6e339326f18..866a82b64d9 100644 --- a/hacking/aws_config/testing_policies/compute-policy.json +++ b/hacking/aws_config/testing_policies/compute-policy.json @@ -268,6 +268,23 @@ "Resource": [ "*" ] + }, + { + "Sid": "AllowStepFunctionsStateMachine", + "Effect": "Allow", + "Action": [ + "states:CreateStateMachine", + "states:DeleteStateMachine", + "states:DescribeStateMachine", + "states:ListStateMachines", + "states:ListTagsForResource", + "states:TagResource", + "states:UntagResource", + "states:UpdateStateMachine" + ], + "Resource": [ + "arn:aws:states:*" + ] } ] } diff --git a/lib/ansible/modules/cloud/amazon/aws_step_functions_state_machine.py b/lib/ansible/modules/cloud/amazon/aws_step_functions_state_machine.py new file mode 100644 index 00000000000..329ee4283d4 --- /dev/null +++ b/lib/ansible/modules/cloud/amazon/aws_step_functions_state_machine.py @@ -0,0 +1,232 @@ +#!/usr/bin/python +# Copyright (c) 2019, Tom De Keyser (@tdekeyser) +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = ''' +--- +module: aws_step_functions_state_machine + +short_description: Manage AWS Step Functions state machines + +version_added: "2.10" + +description: + - Create, update and delete state machines in AWS Step Functions. + - Calling the module in C(state=present) for an existing AWS Step Functions state machine + will attempt to update the state machine definition, IAM Role, or tags with the provided data. + +options: + name: + description: + - Name of the state machine + required: true + type: str + definition: + description: + - The Amazon States Language definition of the state machine. See + U(https://docs.aws.amazon.com/step-functions/latest/dg/concepts-amazon-states-language.html) for more + information on the Amazon States Language. + - "This parameter is required when C(state=present)." + type: json + role_arn: + description: + - The ARN of the IAM Role that will be used by the state machine for its executions. + - "This parameter is required when C(state=present)." + type: str + state: + description: + - Desired state for the state machine + default: present + choices: [ present, absent ] + type: str + tags: + description: + - A hash/dictionary of tags to add to the new state machine or to add/remove from an existing one. + type: dict + purge_tags: + description: + - If yes, existing tags will be purged from the resource to match exactly what is defined by I(tags) parameter. + If the I(tags) parameter is not set then tags will not be modified. + default: yes + type: bool + +extends_documentation_fragment: + - aws + - ec2 + +author: + - Tom De Keyser (@tdekeyser) +''' + +EXAMPLES = ''' +# Create a new AWS Step Functions state machine +- name: Setup HelloWorld state machine + aws_step_functions_state_machine: + name: "HelloWorldStateMachine" + definition: "{{ lookup('file','state_machine.json') }}" + role_arn: arn:aws:iam::987654321012:role/service-role/invokeLambdaStepFunctionsRole + tags: + project: helloWorld + +# Update an existing state machine +- name: Change IAM Role and tags of HelloWorld state machine + aws_step_functions_state_machine: + name: HelloWorldStateMachine + definition: "{{ lookup('file','state_machine.json') }}" + role_arn: arn:aws:iam::987654321012:role/service-role/anotherStepFunctionsRole + tags: + otherTag: aDifferentTag + +# Remove the AWS Step Functions state machine +- name: Delete HelloWorld state machine + aws_step_functions_state_machine: + name: HelloWorldStateMachine + state: absent +''' + +RETURN = ''' +state_machine_arn: + description: ARN of the AWS Step Functions state machine + type: str + returned: always +''' + +from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.ec2 import ansible_dict_to_boto3_tag_list, AWSRetry, compare_aws_tags, boto3_tag_list_to_ansible_dict + +try: + from botocore.exceptions import ClientError, BotoCoreError +except ImportError: + pass # caught by AnsibleAWSModule + + +def manage_state_machine(state, sfn_client, module): + state_machine_arn = get_state_machine_arn(sfn_client, module) + + if state == 'present': + if state_machine_arn is None: + create(sfn_client, module) + else: + update(state_machine_arn, sfn_client, module) + elif state == 'absent': + if state_machine_arn is not None: + remove(state_machine_arn, sfn_client, module) + + check_mode(module, msg='State is up-to-date.') + module.exit_json(changed=False) + + +def create(sfn_client, module): + check_mode(module, msg='State machine would be created.', changed=True) + + tags = module.params.get('tags') + sfn_tags = ansible_dict_to_boto3_tag_list(tags, tag_name_key_name='key', tag_value_key_name='value') if tags else [] + + state_machine = sfn_client.create_state_machine( + name=module.params.get('name'), + definition=module.params.get('definition'), + roleArn=module.params.get('role_arn'), + tags=sfn_tags + ) + module.exit_json(changed=True, state_machine_arn=state_machine.get('stateMachineArn')) + + +def remove(state_machine_arn, sfn_client, module): + check_mode(module, msg='State machine would be deleted: {0}'.format(state_machine_arn), changed=True) + + sfn_client.delete_state_machine(stateMachineArn=state_machine_arn) + module.exit_json(changed=True, state_machine_arn=state_machine_arn) + + +def update(state_machine_arn, sfn_client, module): + tags_to_add, tags_to_remove = compare_tags(state_machine_arn, sfn_client, module) + + if params_changed(state_machine_arn, sfn_client, module) or tags_to_add or tags_to_remove: + check_mode(module, msg='State machine would be updated: {0}'.format(state_machine_arn), changed=True) + + sfn_client.update_state_machine( + stateMachineArn=state_machine_arn, + definition=module.params.get('definition'), + roleArn=module.params.get('role_arn') + ) + sfn_client.untag_resource( + resourceArn=state_machine_arn, + tagKeys=tags_to_remove + ) + sfn_client.tag_resource( + resourceArn=state_machine_arn, + tags=ansible_dict_to_boto3_tag_list(tags_to_add, tag_name_key_name='key', tag_value_key_name='value') + ) + + module.exit_json(changed=True, state_machine_arn=state_machine_arn) + + +def compare_tags(state_machine_arn, sfn_client, module): + new_tags = module.params.get('tags') + current_tags = sfn_client.list_tags_for_resource(resourceArn=state_machine_arn).get('tags') + return compare_aws_tags(boto3_tag_list_to_ansible_dict(current_tags), new_tags if new_tags else {}, module.params.get('purge_tags')) + + +def params_changed(state_machine_arn, sfn_client, module): + """ + Check whether the state machine definition or IAM Role ARN is different + from the existing state machine parameters. + """ + current = sfn_client.describe_state_machine(stateMachineArn=state_machine_arn) + return current.get('definition') != module.params.get('definition') or current.get('roleArn') != module.params.get('role_arn') + + +def get_state_machine_arn(sfn_client, module): + """ + Finds the state machine ARN based on the name parameter. Returns None if + there is no state machine with this name. + """ + target_name = module.params.get('name') + all_state_machines = sfn_client.list_state_machines(aws_retry=True).get('stateMachines') + + for state_machine in all_state_machines: + if state_machine.get('name') == target_name: + return state_machine.get('stateMachineArn') + + +def check_mode(module, msg='', changed=False): + if module.check_mode: + module.exit_json(changed=changed, output=msg) + + +def main(): + module_args = dict( + name=dict(type='str', required=True), + definition=dict(type='json'), + role_arn=dict(type='str'), + state=dict(choices=['present', 'absent'], default='present'), + tags=dict(default=None, type='dict'), + purge_tags=dict(default=True, type='bool'), + ) + module = AnsibleAWSModule( + argument_spec=module_args, + required_if=[('state', 'present', ['role_arn']), ('state', 'present', ['definition'])], + supports_check_mode=True + ) + + sfn_client = module.client('stepfunctions', retry_decorator=AWSRetry.jittered_backoff(retries=5)) + state = module.params.get('state') + + try: + manage_state_machine(state, sfn_client, module) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg='Failed to manage state machine') + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/aws_step_functions_state_machine/aliases b/test/integration/targets/aws_step_functions_state_machine/aliases new file mode 100644 index 00000000000..6e3860bee23 --- /dev/null +++ b/test/integration/targets/aws_step_functions_state_machine/aliases @@ -0,0 +1,2 @@ +cloud/aws +shippable/aws/group2 diff --git a/test/integration/targets/aws_step_functions_state_machine/defaults/main.yml b/test/integration/targets/aws_step_functions_state_machine/defaults/main.yml new file mode 100644 index 00000000000..45e7f6e3aad --- /dev/null +++ b/test/integration/targets/aws_step_functions_state_machine/defaults/main.yml @@ -0,0 +1,2 @@ +state_machine_name: "{{ resource_prefix }}_step_functions_state_machine_ansible_test" +step_functions_role_name: "ansible-test-sts-{{ resource_prefix }}-step_functions-role" diff --git a/test/integration/targets/aws_step_functions_state_machine/files/alternative_state_machine.json b/test/integration/targets/aws_step_functions_state_machine/files/alternative_state_machine.json new file mode 100644 index 00000000000..1c1275a6561 --- /dev/null +++ b/test/integration/targets/aws_step_functions_state_machine/files/alternative_state_machine.json @@ -0,0 +1,10 @@ +{ + "StartAt": "HelloWorld", + "States": { + "HelloWorld": { + "Type": "Pass", + "Result": "Some other result", + "End": true + } + } +} \ No newline at end of file diff --git a/test/integration/targets/aws_step_functions_state_machine/files/state_machine.json b/test/integration/targets/aws_step_functions_state_machine/files/state_machine.json new file mode 100644 index 00000000000..c07d5cebad4 --- /dev/null +++ b/test/integration/targets/aws_step_functions_state_machine/files/state_machine.json @@ -0,0 +1,10 @@ +{ + "StartAt": "HelloWorld", + "States": { + "HelloWorld": { + "Type": "Pass", + "Result": "Hello World!", + "End": true + } + } +} \ No newline at end of file diff --git a/test/integration/targets/aws_step_functions_state_machine/files/state_machines_iam_trust_policy.json b/test/integration/targets/aws_step_functions_state_machine/files/state_machines_iam_trust_policy.json new file mode 100644 index 00000000000..48d627220fc --- /dev/null +++ b/test/integration/targets/aws_step_functions_state_machine/files/state_machines_iam_trust_policy.json @@ -0,0 +1,12 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "states.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} \ No newline at end of file diff --git a/test/integration/targets/aws_step_functions_state_machine/tasks/main.yml b/test/integration/targets/aws_step_functions_state_machine/tasks/main.yml new file mode 100644 index 00000000000..dffc5b81501 --- /dev/null +++ b/test/integration/targets/aws_step_functions_state_machine/tasks/main.yml @@ -0,0 +1,185 @@ +--- + +- name: Integration test for AWS Step Function state machine module + block: + + # ==== Setup ================================================== + + - name: Set connection information for all tasks + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token }}" + region: "{{ aws_region }}" + no_log: yes + + - name: Create IAM service role needed for Step Functions + iam_role: + name: "{{ step_functions_role_name }}" + description: Role with permissions for AWS Step Functions actions. + assume_role_policy_document: "{{ lookup('file', 'state_machines_iam_trust_policy.json') }}" + state: present + <<: *aws_connection_info + register: step_functions_role + + - name: Pause a few seconds to ensure IAM role is available to next task + pause: + seconds: 10 + + # ==== Tests =================================================== + + - name: Create a new state machine -- check_mode + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + definition: "{{ lookup('file','state_machine.json') }}" + role_arn: "{{ step_functions_role.iam_role.arn }}" + tags: + project: helloWorld + state: present + <<: *aws_connection_info + register: creation_check + check_mode: yes + + - assert: + that: + - creation_check.changed == True + - creation_check.output == 'State machine would be created.' + + - name: Create a new state machine + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + definition: "{{ lookup('file','state_machine.json') }}" + role_arn: "{{ step_functions_role.iam_role.arn }}" + tags: + project: helloWorld + state: present + <<: *aws_connection_info + register: creation_output + + - assert: + that: + - creation_output.changed == True + + - name: Pause a few seconds to ensure state machine role is available + pause: + seconds: 5 + + - name: Idempotent rerun of same state function -- check_mode + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + definition: "{{ lookup('file','state_machine.json') }}" + role_arn: "{{ step_functions_role.iam_role.arn }}" + tags: + project: helloWorld + state: present + <<: *aws_connection_info + register: result + check_mode: yes + + - assert: + that: + - result.changed == False + - result.output == 'State is up-to-date.' + + - name: Idempotent rerun of same state function + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + definition: "{{ lookup('file','state_machine.json') }}" + role_arn: "{{ step_functions_role.iam_role.arn }}" + tags: + project: helloWorld + state: present + <<: *aws_connection_info + register: result + + - assert: + that: + - result.changed == False + + - name: Update an existing state machine -- check_mode + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + definition: "{{ lookup('file','alternative_state_machine.json') }}" + role_arn: "{{ step_functions_role.iam_role.arn }}" + tags: + differentTag: different_tag + state: present + <<: *aws_connection_info + register: update_check + check_mode: yes + + - assert: + that: + - update_check.changed == True + - "update_check.output == 'State machine would be updated: {{ creation_output.state_machine_arn }}'" + + - name: Update an existing state machine + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + definition: "{{ lookup('file','alternative_state_machine.json') }}" + role_arn: "{{ step_functions_role.iam_role.arn }}" + tags: + differentTag: different_tag + state: present + <<: *aws_connection_info + register: update_output + + - assert: + that: + - update_output.changed == True + - update_output.state_machine_arn == creation_output.state_machine_arn + + - name: Remove state machine -- check_mode + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + state: absent + <<: *aws_connection_info + register: deletion_check + check_mode: yes + + - assert: + that: + - deletion_check.changed == True + - "deletion_check.output == 'State machine would be deleted: {{ creation_output.state_machine_arn }}'" + + - name: Remove state machine + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + state: absent + <<: *aws_connection_info + register: deletion_output + + - assert: + that: + - deletion_output.changed == True + - deletion_output.state_machine_arn == creation_output.state_machine_arn + + - name: Non-existent state machine is absent + aws_step_functions_state_machine: + name: "non_existing_state_machine" + state: absent + <<: *aws_connection_info + register: result + + - assert: + that: + - result.changed == False + + # ==== Cleanup ==================================================== + + always: + + - name: Cleanup - delete state machine + aws_step_functions_state_machine: + name: "{{ state_machine_name }}" + state: absent + <<: *aws_connection_info + ignore_errors: true + + - name: Cleanup - delete IAM role needed for Step Functions test + iam_role: + name: "{{ step_functions_role_name }}" + state: absent + <<: *aws_connection_info + ignore_errors: true