ecs_cluster test suite refactor (#57716)

* Combine testing policies

Because of the maximum of 10 policies per group, need to
consolidate testing policies as best we can.

* Tidy put-account-setting tasks and add permission

Using `environment` and `command` rather than `shell` avoids the
need for `no_log` and means that people can fix the problem

* refactor ecs_cluster test suite

move from runme.sh technique to virtualenv

use ec2_instance rather than ec2 module to
avoid need for boto

hacking/aws_config/testing_policies/container-policy.json

@@ -39,6 +39,7 @@
             "ecs:DeregisterTaskDefinition",
             "ecs:Describe*",
             "ecs:List*",
+            "ecs:PutAccountSetting",
             "ecs:RegisterTaskDefinition",
             "ecs:RunTask",
             "ecs:StartTask",

hacking/aws_config/testing_policies/database-policy.json

@@ -63,6 +63,21 @@
             "Resource": [
                 "arn:aws:rds:{{aws_region}}:{{aws_account}}:pg:*"
             ]
+        },
+        {
+          "Sid": "AllowRedshiftManagment",
+          "Action": [
+              "redshift:CreateCluster",
+              "redshift:CreateTags",
+              "redshift:DeleteCluster",
+              "redshift:DeleteTags",
+              "redshift:DescribeClusters",
+              "redshift:DescribeTags",
+              "redshift:ModifyCluster",
+              "redshift:RebootCluster"
+          ],
+          "Effect": "Allow",
+          "Resource": "*"
         }
     ]
 }

hacking/aws_config/testing_policies/efs-policy.json

@@ -1,36 +0,0 @@
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Sid": "ManageNetwork",
-            "Effect": "Allow",
-            "Action": [
-                "ec2:CreateNetworkInterface",
-                "ec2:CreateSubnet",
-                "ec2:CreateTags",
-                "ec2:CreateVpc",
-                "ec2:DeleteNetworkInterface",
-                "ec2:DeleteSubnet",
-                "ec2:DeleteVpc",
-                "ec2:DescribeNetworkInterfaceAttribute",
-                "ec2:DescribeNetworkInterfaces",
-                "ec2:DescribeSecurityGroups",
-                "ec2:DescribeSubnets",
-                "ec2:DescribeTags",
-                "ec2:DescribeVpcAttribute",
-                "ec2:DescribeVpcClassicLink",
-                "ec2:DescribeVpcs",
-                "ec2:ModifyVpcAttribute"
-            ],
-            "Resource": "*"
-        },
-        {
-            "Sid": "ManageEFS",
-            "Effect": "Allow",
-            "Action": [
-                "elasticfilesystem:*"
-            ],
-            "Resource": "*"
-        }
-    ]
-}

hacking/aws_config/testing_policies/kms-policy.json

@@ -1,54 +0,0 @@
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Sid": "AllowAccessToUnspecifiedKMSResources",
-            "Effect": "Allow",
-            "Action": [
-                "iam:ListRoles",
-                "kms:CancelKeyDeletion",
-                "kms:CreateAlias",
-                "kms:CreateGrant",
-                "kms:CreateKey",
-                "kms:DeleteAlias",
-                "kms:Describe*",
-                "kms:DisableKey",
-                "kms:EnableKey",
-                "kms:GenerateRandom",
-                "kms:Get*",
-                "kms:List*",
-                "kms:RetireGrant",
-                "kms:ScheduleKeyDeletion",
-                "kms:TagResource",
-                "kms:UntagResource",
-                "kms:UpdateGrant",
-                "kms:UpdateKeyDescription"
-            ],
-            "Resource": "*"
-        },
-        {
-            "Sid": "AllowAccessToSpecifiedIAMResources",
-            "Effect": "Allow",
-            "Action": [
-                "iam:CreateRole",
-                "iam:DeleteRole",
-                "iam:GetRole",
-                "iam:ListAttachedRolePolicies",
-                "iam:ListInstanceProfilesForRole",
-                "iam:PassRole",
-                "iam:UpdateAssumeRolePolicy"
-            ],
-            "Resource": "arn:aws:iam::{{aws_account}}:role/ansible-test-*"
-        },
-        {
-            "Sid": "AllowInstanceProfileCreation",
-            "Effect": "Allow",
-            "Action": [
-                "iam:AddRoleToInstanceProfile",
-                "iam:CreateInstanceProfile",
-                "iam:RemoveRoleFromInstanceProfile"
-            ],
-            "Resource": "arn:aws:iam::{{aws_account}}:instance-profile/ansible-test-*"
-        }
-    ]
-}

hacking/aws_config/testing_policies/redshift-policy.json

@@ -1,20 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Sid": "AllowRedshiftManagment",
-      "Action": [
-        "redshift:CreateCluster",
-        "redshift:CreateTags",
-        "redshift:DeleteCluster",
-        "redshift:DeleteTags",
-        "redshift:DescribeClusters",
-        "redshift:DescribeTags",
-        "redshift:ModifyCluster",
-        "redshift:RebootCluster"
-      ],
-      "Effect": "Allow",
-      "Resource": "*"
-    }
-  ]
-}

hacking/aws_config/testing_policies/security-policy.json

@@ -63,7 +63,6 @@
             "Effect": "Allow",
             "Sid": "AllowReplacementOfSpecificInstanceProfiles"
         },
-
         {
             "Sid": "AllowWAFusage",
             "Action": "waf:*",
@@ -93,10 +92,66 @@
             ]
         },
         {
-            "Sid": "AllowWAFRegionalusage",
-            "Action": "waf-regional:*",
+            "Sid": "AllowSTSAnsibleTests",
+            "Action": [
+                "iam:CreateRole",
+                "iam:DeleteRole",
+                "iam:DetachRolePolicy",
+                "sts:AssumeRole",
+                "iam:AttachRolePolicy",
+                "iam:CreateInstanceProfile"
+            ],
             "Effect": "Allow",
+            "Resource": [
+                "arn:aws:iam::{{aws_account}}:role/ansible-test-sts-*",
+                "arn:aws:iam::{{aws_account}}:instance-profile/ansible-test-sts-*"
+            ]
+        },
+        {
+            "Sid": "AllowAccessToUnspecifiedKMSResources",
+            "Effect": "Allow",
+            "Action": [
+                "kms:CancelKeyDeletion",
+                "kms:CreateAlias",
+                "kms:CreateGrant",
+                "kms:CreateKey",
+                "kms:DeleteAlias",
+                "kms:Describe*",
+                "kms:DisableKey",
+                "kms:EnableKey",
+                "kms:GenerateRandom",
+                "kms:Get*",
+                "kms:List*",
+                "kms:RetireGrant",
+                "kms:ScheduleKeyDeletion",
+                "kms:TagResource",
+                "kms:UntagResource",
+                "kms:UpdateGrant",
+                "kms:UpdateKeyDescription"
+            ],
             "Resource": "*"
+        },
+        {
+            "Sid": "AllowAccessToSpecifiedIAMResources",
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateRole",
+                "iam:DeleteRole",
+                "iam:GetRole",
+                "iam:PassRole",
+                "iam:UpdateAssumeRolePolicy"
+            ],
+            "Resource": "arn:aws:iam::{{aws_account}}:role/ansible-test-*"
+        },
+        {
+            "Sid": "AllowInstanceProfileCreation",
+            "Effect": "Allow",
+            "Action": [
+                "iam:AddRoleToInstanceProfile",
+                "iam:CreateInstanceProfile",
+                "iam:RemoveRoleFromInstanceProfile"
+            ],
+            "Resource": "arn:aws:iam::{{aws_account}}:instance-profile/ansible-test-*"
         }
     ]
 }

hacking/aws_config/testing_policies/storage-policy.json

@@ -37,6 +37,14 @@
             ],
             "Effect": "Allow",
             "Resource": "*"
+        },
+        {
+            "Sid": "ManageEFS",
+            "Effect": "Allow",
+            "Action": [
+                "elasticfilesystem:*"
+            ],
+            "Resource": "*"
         }
     ]
 }

hacking/aws_config/testing_policies/sts-policy.json

@@ -1,23 +0,0 @@
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Sid": "AllowSTSAnsibleTests",
-            "Action": [
-                "iam:Get*",
-                "iam:List*",
-                "iam:CreateRole",
-                "iam:DeleteRole",
-                "iam:DetachRolePolicy",
-                "sts:AssumeRole",
-                "iam:AttachRolePolicy",
-                "iam:CreateInstanceProfile"
-            ],
-            "Effect": "Allow",
-            "Resource": [
-                "arn:aws:iam::{{aws_account}}:role/ansible-test-sts-*",
-                "arn:aws:iam::{{aws_account}}:instance-profile/ansible-test-sts-*"
-            ]
-        }
-    ]
-}

test/integration/targets/ecs_cluster/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - setup_remote_tmp_dir

test/integration/targets/ecs_cluster/playbooks/full_test.yml

@@ -1,6 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-
-  roles:
-    - ecs_cluster

test/integration/targets/ecs_cluster/playbooks/network_assign_public_ip_fail.yml

@@ -1,130 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-  vars:
-     resource_prefix: 'ansible-testing'
-
-  tasks:
-    - block:
-        - name: set up aws connection info
-          set_fact:
-            aws_connection_info: &aws_connection_info
-              aws_access_key: "{{ aws_access_key }}"
-              aws_secret_key: "{{ aws_secret_key }}"
-              security_token: "{{ security_token }}"
-              region: "{{ aws_region }}"
-          no_log: True
-
-        - name: create ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: present
-            <<: *aws_connection_info
-
-        - name: create ecs_taskdefinition with bridged network
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            state: present
-            network_mode: bridge
-            <<: *aws_connection_info
-          register: ecs_taskdefinition_creation
-
-        - name: create ecs_taskdefinition with awsvpc network
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}-vpc"
-            state: present
-            network_mode: awsvpc
-            <<: *aws_connection_info
-          register: ecs_taskdefinition_creation_vpc
-
-        - name: ecs_taskdefinition works fine even when older botocore is used
-          assert:
-            that:
-              - ecs_taskdefinition_creation_vpc.changed
-
-        - name: create ecs_service using awsvpc network_configuration
-          ecs_service:
-            name: "{{ resource_prefix }}-vpc"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}-vpc"
-            desired_count: 1
-            network_configuration:
-              subnets:
-                - subnet-abcd1234
-              security_groups:
-                - sg-abcd1234
-              assign_public_ip: true
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation_vpc
-          ignore_errors: yes
-
-        - name: check that graceful failure message is returned from ecs_service
-          assert:
-            that:
-              - ecs_service_creation_vpc.failed
-              - 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.8.4 or higher to use assign_public_ip in network_configuration"'
-
-      always:
-        - name: scale down ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 0
-            state: present
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: pause to wait for scale down
-          pause:
-            seconds: 30
-
-        - name: remove ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs task definition
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs task definition vpc
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}-vpc"
-            revision: "{{ ecs_taskdefinition_creation_vpc.taskdefinition.revision }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes

test/integration/targets/ecs_cluster/playbooks/network_fail.yml

@@ -1,223 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-  vars:
-     resource_prefix: 'ansible-testing'
-
-  tasks:
-    - block:
-        - name: set up aws connection info
-          set_fact:
-            aws_connection_info: &aws_connection_info
-              aws_access_key: "{{ aws_access_key }}"
-              aws_secret_key: "{{ aws_secret_key }}"
-              security_token: "{{ security_token }}"
-              region: "{{ aws_region }}"
-          no_log: True
-
-        - name: create ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: present
-            <<: *aws_connection_info
-
-        - name: create ecs_taskdefinition with bridged network
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            state: present
-            network_mode: bridge
-            <<: *aws_connection_info
-          register: ecs_taskdefinition_creation
-
-        - name: create ecs_taskdefinition with awsvpc network
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}-vpc"
-            state: present
-            network_mode: awsvpc
-            <<: *aws_connection_info
-          register: ecs_taskdefinition_creation_vpc
-
-        - name: create ecs_taskdefinition and execution_role_arn (expected to fail)
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}-vpc"
-            execution_role_arn: not_a_real_arn
-            state: present
-            network_mode: awsvpc
-            <<: *aws_connection_info
-          ignore_errors: yes
-          register: ecs_taskdefinition_arn
-
-        - name: check that graceful failure message is returned from ecs_taskdefinition_arn
-          assert:
-            that:
-              - ecs_taskdefinition_arn.failed
-              - 'ecs_taskdefinition_arn.msg == "botocore needs to be version 1.10.44 or higher to use execution_role_arn"'
-
-        - name: ecs_taskdefinition works fine even when older botocore is used
-          assert:
-            that:
-              - ecs_taskdefinition_creation_vpc.changed
-
-        - name: create ecs_service using bridged network
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation
-
-        - name: create ecs_service using awsvpc network_configuration
-          ecs_service:
-            name: "{{ resource_prefix }}-vpc"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}-vpc"
-            desired_count: 1
-            network_configuration:
-              subnets:
-                - subnet-abcd1234
-              security_groups:
-                - sg-abcd1234
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation_vpc
-          ignore_errors: yes
-
-        - name: check that graceful failure message is returned from ecs_service
-          assert:
-            that:
-              - ecs_service_creation_vpc.failed
-              - 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
-
-        - name: create ecs_service using awsvpc network_configuration and launch_type
-          ecs_service:
-            name: "{{ resource_prefix }}-vpc"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}-vpc"
-            desired_count: 1
-            network_configuration:
-              subnets:
-                - subnet-abcd1234
-              security_groups:
-                - sg-abcd1234
-            launch_type: FARGATE
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation_vpc_launchtype
-          ignore_errors: yes
-
-        - name: check that graceful failure message is returned from ecs_service
-          assert:
-            that:
-              - ecs_service_creation_vpc_launchtype.failed
-              - 'ecs_service_creation_vpc_launchtype.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
-
-        - name: create ecs_service with launchtype and missing network_configuration
-          ecs_service:
-            name: "{{ resource_prefix }}-vpc"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}-vpc"
-            desired_count: 1
-            launch_type: FARGATE
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation_vpc_launchtype_nonet
-          ignore_errors: yes
-
-        - name: check that graceful failure message is returned from ecs_service
-          assert:
-            that:
-              - ecs_service_creation_vpc_launchtype_nonet.failed
-              - 'ecs_service_creation_vpc_launchtype_nonet.msg == "launch_type is FARGATE but all of the following are missing: network_configuration"'
-
-        - name: create ecs_task using awsvpc network_configuration
-          ecs_task:
-            cluster: "{{ resource_prefix }}-vpc"
-            task_definition: "{{ resource_prefix }}"
-            operation: run
-            count: 1
-            started_by: me
-            network_configuration:
-              subnets:
-                - subnet-abcd1234
-              security_groups:
-                - sg-abcd1234
-            <<: *aws_connection_info
-          register: ecs_task_creation_vpc
-          ignore_errors: yes
-
-        - name: check that graceful failure message is returned from ecs_task
-          assert:
-            that:
-              - ecs_task_creation_vpc.failed
-              - 'ecs_task_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
-
-
-      always:
-        - name: scale down ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 0
-            state: present
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: pause to wait for scale down
-          pause:
-            seconds: 30
-
-        - name: remove ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs task definition
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs task definition vpc
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}-vpc"
-            revision: "{{ ecs_taskdefinition_creation_vpc.taskdefinition.revision }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes

test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment.yml

@@ -1,109 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-  vars:
-     resource_prefix: 'ansible-testing-fnd'
-
-  tasks:
-    - block:
-        - name: set up aws connection info
-          set_fact:
-            aws_connection_info: &aws_connection_info
-              aws_access_key: "{{ aws_access_key }}"
-              aws_secret_key: "{{ aws_secret_key }}"
-              security_token: "{{ security_token }}"
-              region: "{{ aws_region }}"
-          no_log: True
-
-        - name: create ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: present
-            <<: *aws_connection_info
-
-        - name: create ecs_taskdefinition
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            state: present
-            <<: *aws_connection_info
-          register: ecs_taskdefinition_creation
-
-        - name: create ecs_service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation
-
-        - name: ecs_service works fine even when older botocore is used
-          assert:
-            that:
-              - ecs_service_creation.changed
-
-        - name: create ecs_service using force_new_deployment
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            force_new_deployment: true
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation_force_new_deploy
-          ignore_errors: yes
-
-        - name: check that module returns success
-          assert:
-            that:
-              - ecs_service_creation_force_new_deploy.changed
-
-      always:
-        - name: scale down ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 0
-            state: present
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: pause to wait for scale down
-          pause:
-            seconds: 30
-
-        - name: remove ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs task definition
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes

test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment_fail.yml

@@ -1,110 +0,0 @@
-- hosts: localhost
-  connection: local
-  environment: "{{ ansible_test.environment }}"
-  vars:
-     resource_prefix: 'ansible-testing-fndf'
-
-  tasks:
-    - block:
-        - name: set up aws connection info
-          set_fact:
-            aws_connection_info: &aws_connection_info
-              aws_access_key: "{{ aws_access_key }}"
-              aws_secret_key: "{{ aws_secret_key }}"
-              security_token: "{{ security_token }}"
-              region: "{{ aws_region }}"
-          no_log: True
-
-        - name: create ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: present
-            <<: *aws_connection_info
-
-        - name: create ecs_taskdefinition
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            state: present
-            <<: *aws_connection_info
-          register: ecs_taskdefinition_creation
-
-        - name: create ecs_service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation
-
-        - name: ecs_service works fine even when older botocore is used
-          assert:
-            that:
-              - ecs_service_creation.changed
-
-        - name: create ecs_service using force_new_deployment
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            force_new_deployment: true
-            state: present
-            <<: *aws_connection_info
-          register: ecs_service_creation_force_new_deploy
-          ignore_errors: yes
-
-        - name: check that graceful failure message is returned from ecs_service
-          assert:
-            that:
-              - ecs_service_creation_force_new_deploy.failed
-              - 'ecs_service_creation_force_new_deploy.msg == "botocore needs to be version 1.8.4 or higher to use force_new_deployment"'
-
-      always:
-        - name: scale down ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 0
-            state: present
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: pause to wait for scale down
-          pause:
-            seconds: 30
-
-        - name: remove ecs service
-          ecs_service:
-            name: "{{ resource_prefix }}"
-            cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
-            desired_count: 1
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs task definition
-          ecs_taskdefinition:
-            containers:
-              - name: my_container
-                image: ubuntu
-                memory: 128
-            family: "{{ resource_prefix }}"
-            revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes
-
-        - name: remove ecs cluster
-          ecs_cluster:
-            name: "{{ resource_prefix }}"
-            state: absent
-            <<: *aws_connection_info
-          ignore_errors: yes

test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/meta/main.yml

@@ -1 +0,0 @@
-dependencies: []

test/integration/targets/ecs_cluster/runme.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux
-
-# Test graceful failure for older versions of botocore
-source virtualenv.sh
-pip install 'botocore<=1.7.40' boto3
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/network_fail.yml "$@"
-
-# Test graceful failure for assign public ip
-# applies for botocore >= 1.7.44 and < 1.8.4
-source virtualenv.sh
-pip install 'botocore>=1.7.44,<1.8.4' boto3
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/network_assign_public_ip_fail.yml "$@"
-
-# Test graceful failure for force new deployment #42518
-# applies for botocore < 1.8.4
-source virtualenv.sh
-pip install 'botocore>=1.7.44,<1.8.4' boto3
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/network_force_new_deployment_fail.yml "$@"
-
-# Test force new deployment #42518
-# applies for botocore < 1.8.4
-source virtualenv.sh
-pip install 'botocore>1.8.4' boto3
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/network_force_new_deployment.yml "$@"
-
-# Run full test suite
-source virtualenv.sh
-pip install 'botocore>=1.10.37' boto3 # version 1.10.37 for scheduling strategy
-ansible-playbook -i ../../inventory -e @../../integration_config.yml -v playbooks/full_test.yml "$@"

test/integration/targets/ecs_cluster/tasks/full_test.yml

@@ -136,17 +136,17 @@
         ecs_image_id: "{{ (ec2_ami_info.images|first).image_id }}"
 
     - name: provision ec2 instance to create an image
-      ec2:
+      ec2_instance:
         key_name: '{{ ec2_keypair|default(setup_key.key.name) }}'
         instance_type: t2.micro
         state: present
-        image: '{{ ecs_image_id }}'
+        image_id: '{{ ecs_image_id }}'
         wait: yes
         user_data: "{{ user_data }}"
-        instance_profile_name: ecsInstanceRole
-        instance_tags:
+        instance_role: ecsInstanceRole
+        tags:
           Name: '{{ resource_prefix }}_ecs_agent'
-        group_id: '{{ setup_sg.group_id }}'
+        security_group: '{{ setup_sg.group_id }}'
         vpc_subnet_id: '{{ setup_subnet.results[0].subnet.id }}'
         <<: *aws_connection_info
       register: setup_instance
@@ -1036,7 +1036,7 @@
       ignore_errors: yes
 
     - name: remove setup ec2 instance
-      ec2:
+      ec2_instance:
         instance_ids: '{{ setup_instance.instance_ids }}'
         state: absent
         wait: yes

test/integration/targets/ecs_cluster/tasks/main.yml

@@ -0,0 +1,53 @@
+- set_fact:
+    virtualenv: "{{ remote_tmp_dir }}/virtualenv"
+    virtualenv_command: "{{ ansible_python_interpreter }} -m virtualenv"
+
+- set_fact:
+    virtualenv_interpreter: "{{ virtualenv }}/bin/python"
+
+- pip:
+    name: virtualenv
+
+- pip:
+    name:
+      - 'botocore<1.8.4'
+      - boto3
+      - coverage
+    virtualenv: "{{ virtualenv }}"
+    virtualenv_command: "{{ virtualenv_command }}"
+    virtualenv_site_packages: no
+
+- include_tasks: network_assign_public_ip_fail.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+
+- include_tasks: network_force_new_deployment_fail.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+
+- file:
+    path: "{{ virtualenv }}"
+    state: absent
+
+# Test graceful failures when botocore<1.12.38
+
+- pip:
+    name:
+      - 'botocore>=1.12.60'
+      - boto3
+      - coverage
+    virtualenv: "{{ virtualenv }}"
+    virtualenv_command: "{{ virtualenv_command }}"
+    virtualenv_site_packages: no
+
+- include_tasks: network_force_new_deployment.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+
+- include_tasks: full_test.yml
+  vars:
+    ansible_python_interpreter: "{{ virtualenv_interpreter }}"
+
+- file:
+    path: "{{ virtualenv }}"
+    state: absent

test/integration/targets/ecs_cluster/tasks/network_assign_public_ip_fail.yml

@@ -0,0 +1,123 @@
+- block:
+    - name: set up aws connection info
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: True
+
+    - name: create ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: present
+        <<: *aws_connection_info
+
+    - name: create ecs_taskdefinition with bridged network
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        state: present
+        network_mode: bridge
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_creation
+
+    - name: create ecs_taskdefinition with awsvpc network
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}-vpc"
+        state: present
+        network_mode: awsvpc
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_creation_vpc
+
+    - name: ecs_taskdefinition works fine even when older botocore is used
+      assert:
+        that:
+          - ecs_taskdefinition_creation_vpc.changed
+
+    - name: create ecs_service using awsvpc network_configuration
+      ecs_service:
+        name: "{{ resource_prefix }}-vpc"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}-vpc"
+        desired_count: 1
+        network_configuration:
+          subnets:
+            - subnet-abcd1234
+          security_groups:
+            - sg-abcd1234
+          assign_public_ip: true
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_vpc
+      ignore_errors: yes
+
+    - name: check that graceful failure message is returned from ecs_service
+      assert:
+        that:
+          - ecs_service_creation_vpc.failed
+          - 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.8.4 or higher to use assign_public_ip in network_configuration"'
+
+  always:
+    - name: scale down ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 0
+        state: present
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: pause to wait for scale down
+      pause:
+        seconds: 30
+
+    - name: remove ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs task definition
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs task definition vpc
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}-vpc"
+        revision: "{{ ecs_taskdefinition_creation_vpc.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes

test/integration/targets/ecs_cluster/tasks/network_fail.yml

@@ -0,0 +1,216 @@
+- block:
+    - name: set up aws connection info
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: True
+
+    - name: create ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: present
+        <<: *aws_connection_info
+
+    - name: create ecs_taskdefinition with bridged network
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        state: present
+        network_mode: bridge
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_creation
+
+    - name: create ecs_taskdefinition with awsvpc network
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}-vpc"
+        state: present
+        network_mode: awsvpc
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_creation_vpc
+
+    - name: create ecs_taskdefinition and execution_role_arn (expected to fail)
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}-vpc"
+        execution_role_arn: not_a_real_arn
+        state: present
+        network_mode: awsvpc
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_taskdefinition_arn
+
+    - name: check that graceful failure message is returned from ecs_taskdefinition_arn
+      assert:
+        that:
+          - ecs_taskdefinition_arn.failed
+          - 'ecs_taskdefinition_arn.msg == "botocore needs to be version 1.10.44 or higher to use execution_role_arn"'
+
+    - name: ecs_taskdefinition works fine even when older botocore is used
+      assert:
+        that:
+          - ecs_taskdefinition_creation_vpc.changed
+
+    - name: create ecs_service using bridged network
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation
+
+    - name: create ecs_service using awsvpc network_configuration
+      ecs_service:
+        name: "{{ resource_prefix }}-vpc"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}-vpc"
+        desired_count: 1
+        network_configuration:
+          subnets:
+            - subnet-abcd1234
+          security_groups:
+            - sg-abcd1234
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_vpc
+      ignore_errors: yes
+
+    - name: check that graceful failure message is returned from ecs_service
+      assert:
+        that:
+          - ecs_service_creation_vpc.failed
+          - 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
+
+    - name: create ecs_service using awsvpc network_configuration and launch_type
+      ecs_service:
+        name: "{{ resource_prefix }}-vpc"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}-vpc"
+        desired_count: 1
+        network_configuration:
+          subnets:
+            - subnet-abcd1234
+          security_groups:
+            - sg-abcd1234
+        launch_type: FARGATE
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_vpc_launchtype
+      ignore_errors: yes
+
+    - name: check that graceful failure message is returned from ecs_service
+      assert:
+        that:
+          - ecs_service_creation_vpc_launchtype.failed
+          - 'ecs_service_creation_vpc_launchtype.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
+
+    - name: create ecs_service with launchtype and missing network_configuration
+      ecs_service:
+        name: "{{ resource_prefix }}-vpc"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}-vpc"
+        desired_count: 1
+        launch_type: FARGATE
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_vpc_launchtype_nonet
+      ignore_errors: yes
+
+    - name: check that graceful failure message is returned from ecs_service
+      assert:
+        that:
+          - ecs_service_creation_vpc_launchtype_nonet.failed
+          - 'ecs_service_creation_vpc_launchtype_nonet.msg == "launch_type is FARGATE but all of the following are missing: network_configuration"'
+
+    - name: create ecs_task using awsvpc network_configuration
+      ecs_task:
+        cluster: "{{ resource_prefix }}-vpc"
+        task_definition: "{{ resource_prefix }}"
+        operation: run
+        count: 1
+        started_by: me
+        network_configuration:
+          subnets:
+            - subnet-abcd1234
+          security_groups:
+            - sg-abcd1234
+        <<: *aws_connection_info
+      register: ecs_task_creation_vpc
+      ignore_errors: yes
+
+    - name: check that graceful failure message is returned from ecs_task
+      assert:
+        that:
+          - ecs_task_creation_vpc.failed
+          - 'ecs_task_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
+
+
+  always:
+    - name: scale down ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 0
+        state: present
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: pause to wait for scale down
+      pause:
+        seconds: 30
+
+    - name: remove ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs task definition
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs task definition vpc
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}-vpc"
+        revision: "{{ ecs_taskdefinition_creation_vpc.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes

test/integration/targets/ecs_cluster/tasks/network_force_new_deployment.yml

@@ -0,0 +1,102 @@
+- block:
+    - name: set up aws connection info
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: True
+
+    - name: create ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: present
+        <<: *aws_connection_info
+
+    - name: create ecs_taskdefinition
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        state: present
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_creation
+
+    - name: create ecs_service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation
+
+    - name: ecs_service works fine even when older botocore is used
+      assert:
+        that:
+          - ecs_service_creation.changed
+
+    - name: create ecs_service using force_new_deployment
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        force_new_deployment: true
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_force_new_deploy
+      ignore_errors: yes
+
+    - name: check that module returns success
+      assert:
+        that:
+          - ecs_service_creation_force_new_deploy.changed
+
+  always:
+    - name: scale down ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 0
+        state: present
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: pause to wait for scale down
+      pause:
+        seconds: 30
+
+    - name: remove ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs task definition
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes

test/integration/targets/ecs_cluster/tasks/network_force_new_deployment_fail.yml

@@ -0,0 +1,103 @@
+- block:
+    - name: set up aws connection info
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: True
+
+    - name: create ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: present
+        <<: *aws_connection_info
+
+    - name: create ecs_taskdefinition
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        state: present
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_creation
+
+    - name: create ecs_service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation
+
+    - name: ecs_service works fine even when older botocore is used
+      assert:
+        that:
+          - ecs_service_creation.changed
+
+    - name: create ecs_service using force_new_deployment
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        force_new_deployment: true
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_force_new_deploy
+      ignore_errors: yes
+
+    - name: check that graceful failure message is returned from ecs_service
+      assert:
+        that:
+          - ecs_service_creation_force_new_deploy.failed
+          - 'ecs_service_creation_force_new_deploy.msg == "botocore needs to be version 1.8.4 or higher to use force_new_deployment"'
+
+  always:
+    - name: scale down ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 0
+        state: present
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: pause to wait for scale down
+      pause:
+        seconds: 30
+
+    - name: remove ecs service
+      ecs_service:
+        name: "{{ resource_prefix }}"
+        cluster: "{{ resource_prefix }}"
+        task_definition: "{{ resource_prefix }}"
+        desired_count: 1
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs task definition
+      ecs_taskdefinition:
+        containers:
+          - name: my_container
+            image: ubuntu
+            memory: 128
+        family: "{{ resource_prefix }}"
+        revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove ecs cluster
+      ecs_cluster:
+        name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes