Add Fargate support for ECS modules

Fargate instances do not require memory and cpu descriptors. EC2 instances
 do require descriptions. https://botocore.readthedocs.io/en/latest/reference/services/ecs.html#ECS.Client.describe_task_definition

Fargate requires that cpu and memory be defined at task definition level.
EC2 launch requires them to be defined at the container level.

Fargate requires the use of awsvpc for the networking_mode. Also updated,
the documentation regarding where and when memory/cpu needs to the assigned.

The task_definition variable for the awspvc configuration colided with
the ecs_service for the bridge network. This would cause the test to fail.

Add testing for fargate

Add examples for fargate and ec2

lib/ansible/modules/cloud/amazon/ecs_service.py

@@ -107,9 +107,8 @@ options:
         description:
           - The launch type on which to run your service
         required: false
-        version_added: 2.5
+        version_added: 2.7
         choices: ["EC2", "FARGATE"]
-        default: "EC2"
 extends_documentation_fragment:
     - aws
     - ec2
@@ -376,10 +375,12 @@ class EcsServiceManager:
             role=role,
             deploymentConfiguration=deployment_configuration,
             placementConstraints=placement_constraints,
-            placementStrategy=placement_strategy,
+            placementStrategy=placement_strategy
-            launchType=launch_type)
+        )
         if network_configuration:
             params['networkConfiguration'] = network_configuration
+        if launch_type:
+            params['launchType'] = launch_type
         response = self.ecs.create_service(**params)
         return self.jsonize(response['service'])
 
@@ -441,12 +442,13 @@ def main():
         placement_constraints=dict(required=False, default=[], type='list'),
         placement_strategy=dict(required=False, default=[], type='list'),
         network_configuration=dict(required=False, type='dict'),
-        launch_type=dict(required=False, choices=['EC2', 'FARGATE'], default='EC2')
+        launch_type=dict(required=False, choices=['EC2', 'FARGATE'])
     ))
 
     module = AnsibleAWSModule(argument_spec=argument_spec,
                               supports_check_mode=True,
-                              required_if=[('state', 'present', ['task_definition', 'desired_count'])],
+                              required_if=[('state', 'present', ['task_definition', 'desired_count']),
+                                           ('launch_type', 'FARGATE', ['network_configuration'])],
                               required_together=[['load_balancers', 'role']])
 
     service_mgr = EcsServiceManager(module)
@@ -468,10 +470,16 @@ def main():
         module.fail_json(msg="Exception describing service '" + module.params['name'] + "' in cluster '" + module.params['cluster'] + "': " + str(e))
 
     results = dict(changed=False)
+
+    if module.params['launch_type']:
+        if not module.botocore_at_least('1.8.4'):
+            module.fail_json(msg='botocore needs to be version 1.8.4 or higher to use launch_type')
+
     if module.params['state'] == 'present':
 
         matching = False
         update = False
+
         if existing and 'status' in existing and existing['status'] == "ACTIVE":
             if service_mgr.is_matching_service(module.params, existing):
                 matching = True
@@ -501,18 +509,21 @@ def main():
                         if 'containerPort' in loadBalancer:
                             loadBalancer['containerPort'] = int(loadBalancer['containerPort'])
                     # doesn't exist. create it.
-                    response = service_mgr.create_service(module.params['name'],
+                    try:
-                                                          module.params['cluster'],
+                        response = service_mgr.create_service(module.params['name'],
-                                                          module.params['task_definition'],
+                                                              module.params['cluster'],
-                                                          loadBalancers,
+                                                              module.params['task_definition'],
-                                                          module.params['desired_count'],
+                                                              loadBalancers,
-                                                          clientToken,
+                                                              module.params['desired_count'],
-                                                          role,
+                                                              clientToken,
-                                                          deploymentConfiguration,
+                                                              role,
-                                                          module.params['placement_constraints'],
+                                                              deploymentConfiguration,
-                                                          module.params['placement_strategy'],
+                                                              module.params['placement_constraints'],
-                                                          network_configuration,
+                                                              module.params['placement_strategy'],
-                                                          module.params['launch_type'])
+                                                              network_configuration,
+                                                              module.params['launch_type'])
+                    except botocore.exceptions.ClientError as e:
+                        module.fail_json(msg=e.message)
 
                 results['service'] = response
 

lib/ansible/modules/cloud/amazon/ecs_taskdefinition.py

@@ -77,21 +77,20 @@ options:
         description:
             - The launch type on which to run your task
         required: false
-        version_added: 2.5
+        version_added: 2.7
         choices: ["EC2", "FARGATE"]
-        default: "EC2"
     cpu:
         description:
-            - The number of cpu units used by the task. If using the EC2 launch type, this field is optional and any value can be used. 
+            - The number of cpu units used by the task. If using the EC2 launch type, this field is optional and any value can be used.
               If using the Fargate launch type, this field is required and you must use one of [256, 512, 1024, 2048, 4096]
         required: false
-        version_added: 2.5
+        version_added: 2.7
     memory:
         description:
-            - The amount (in MiB) of memory used by the task. If using the EC2 launch type, this field is optional and any value can be used. 
+            - The amount (in MiB) of memory used by the task. If using the EC2 launch type, this field is optional and any value can be used.
               If using the Fargate launch type, this field is required and is limited by the cpu
         required: false
-        version_added: 2.5
+        version_added: 2.7
 extends_documentation_fragment:
     - aws
     - ec2
@@ -137,6 +136,36 @@ EXAMPLES = '''
     family: test-cluster-taskdef
     state: present
   register: task_output
+
+- name: Create task definition
+  ecs_taskdefinition:
+    family: nginx
+    containers:
+    - name: nginx
+      essential: true
+      image: "nginx"
+      portMappings:
+      - containerPort: 8080
+        hostPort:      8080
+      cpu: 512
+      memory: 1GB
+    state: present
+
+- name: Create task definition
+  ecs_taskdefinition:
+    family: nginx
+    containers:
+    - name: nginx
+      essential: true
+      image: "nginx"
+      portMappings:
+      - containerPort: 8080
+        hostPort:      8080
+    launch_type: FARGATE
+    cpu: 512
+    memory: 1GB
+    state: present
+    network_mode: awsvpc
 '''
 RETURN = '''
 taskdefinition:
@@ -198,13 +227,14 @@ class EcsTaskManager:
             taskRoleArn=task_role_arn,
             networkMode=network_mode,
             containerDefinitions=container_definitions,
-            volumes=volumes,
+            volumes=volumes
-            requiresCompatibilities=launch_type
         )
         if cpu:
             params['cpu'] = cpu
         if memory:
             params['memory'] = memory
+        if launch_type:
+            params['requiresCompatibilities'] = [launch_type]
 
         try:
             response = self.ecs.register_task_definition(**params)
@@ -249,9 +279,14 @@ class EcsTaskManager:
         response = self.ecs.deregister_task_definition(taskDefinition=taskArn)
         return response['taskDefinition']
 
+    def ecs_api_supports_requirescompatibilities(self):
+        from distutils.version import LooseVersion
+        # Checking to make sure botocore is greater than a specific version.
+        # Support for requiresCompatibilities is only available in versions beyond 1.8.4
+        return LooseVersion(botocore.__version__) >= LooseVersion('1.8.4')
+
 
 def main():
-
     argument_spec = ec2_argument_spec()
     argument_spec.update(dict(
         state=dict(required=True, choices=['present', 'absent']),
@@ -263,12 +298,15 @@ def main():
         network_mode=dict(required=False, default='bridge', choices=['bridge', 'host', 'none', 'awsvpc'], type='str'),
         task_role_arn=dict(required=False, default='', type='str'),
         volumes=dict(required=False, type='list'),
-        launch_type=dict(required=False, choices=['EC2', 'FARGATE'], default='EC2'),
+        launch_type=dict(required=False, choices=['EC2', 'FARGATE']),
-        cpu=dict(required=False, type='str'),
+        cpu=dict(),
         memory=dict(required=False, type='str')
     ))
 
-    module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True)
+    module = AnsibleModule(argument_spec=argument_spec,
+                           supports_check_mode=True,
+                           required_if=[('launch_type', 'FARGATE', ['cpu', 'memory'])]
+                           )
 
     if not HAS_BOTO3:
         module.fail_json(msg='boto3 is required.')
@@ -277,6 +315,10 @@ def main():
     task_mgr = EcsTaskManager(module)
     results = dict(changed=False)
 
+    if module.params['launch_type']:
+        if not task_mgr.ecs_api_supports_requirescompatibilities():
+            module.fail_json(msg='botocore needs to be version 1.8.4 or higher to use launch_type')
+
     for container in module.params.get('containers', []):
         for environment in container.get('environment', []):
             environment['value'] = to_text(environment['value'])
@@ -288,12 +330,10 @@ def main():
         if 'family' not in module.params or not module.params['family']:
             module.fail_json(msg="To use task definitions, a family must be specified")
 
+        network_mode = module.params['network_mode']
         launch_type = module.params['launch_type']
-        if launch_type == 'EC2' and ('cpu' not in module.params or  not module.params['cpu']):
+        if launch_type == 'FARGATE' and network_mode != 'awsvpc':
-            module.fail_json(msg="To use FARGATE launch type, cpu must be specified")
+            module.fail_json(msg="To use FARGATE launch type, network_mode must be awsvpc")
-
-        if launch_type == 'EC2' and ('memory' not in module.params or  not module.params['memory']):
-            module.fail_json(msg="To use FARGATE launch type, memory must be specified")
 
         family = module.params['family']
         existing_definitions_in_family = task_mgr.describe_task_definitions(module.params['family'])

test/integration/targets/ecs_cluster/playbooks/network_fail.yml

@@ -1,5 +1,7 @@
 - hosts: localhost
   connection: local
+  vars:
+     resource_prefix: 'ansible-testing'
 
   tasks:
     - block:
@@ -61,7 +63,7 @@
           ecs_service:
             name: "{{ resource_prefix }}-vpc"
             cluster: "{{ resource_prefix }}"
-            task_definition: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}-vpc"
             desired_count: 1
             network_configuration:
               subnets:
@@ -79,6 +81,47 @@
               - ecs_service_creation_vpc.failed
               - 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
 
+        - name: create ecs_service using awsvpc network_configuration and launch_type
+          ecs_service:
+            name: "{{ resource_prefix }}-vpc"
+            cluster: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}-vpc"
+            desired_count: 1
+            network_configuration:
+              subnets:
+                - subnet-abcd1234
+              groups:
+                - sg-abcd1234
+            launch_type: FARGATE
+            state: present
+            <<: *aws_connection_info
+          register: ecs_service_creation_vpc_launchtype
+          ignore_errors: yes
+
+        - name: check that graceful failure message is returned from ecs_service
+          assert:
+            that:
+              - ecs_service_creation_vpc_launchtype.failed
+              - 'ecs_service_creation_vpc_launchtype.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
+
+        - name: create ecs_service with launchtype and missing network_configuration
+          ecs_service:
+            name: "{{ resource_prefix }}-vpc"
+            cluster: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}-vpc"
+            desired_count: 1
+            launch_type: FARGATE
+            state: present
+            <<: *aws_connection_info
+          register: ecs_service_creation_vpc_launchtype_nonet
+          ignore_errors: yes
+
+        - name: check that graceful failure message is returned from ecs_service
+          assert:
+            that:
+              - ecs_service_creation_vpc_launchtype_nonet.failed
+              - 'ecs_service_creation_vpc_launchtype_nonet.msg == "launch_type is FARGATE but all of the following are missing: network_configuration"'
+
         - name: create ecs_task using awsvpc network_configuration
           ecs_task:
             cluster: "{{ resource_prefix }}-vpc"
@@ -101,6 +144,7 @@
               - ecs_task_creation_vpc.failed
               - 'ecs_task_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
 
+
       always:
         - name: scale down ecs service
           ecs_service:
@@ -138,6 +182,18 @@
             <<: *aws_connection_info
           ignore_errors: yes
 
+        - name: remove ecs task definition vpc
+          ecs_taskdefinition:
+            containers:
+              - name: my_container
+                image: ubuntu
+                memory: 128
+            family: "{{ resource_prefix }}-vpc"
+            revision: "{{ ecs_taskdefinition_creation_vpc.taskdefinition.revision }}"
+            state: absent
+            <<: *aws_connection_info
+          ignore_errors: yes
+
         - name: remove ecs cluster
           ecs_cluster:
             name: "{{ resource_prefix }}"

test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/defaults/main.yml

@@ -3,6 +3,8 @@
 ecs_agent_images:
   us-east-1: ami-71ef560b
   us-east-2: ami-1b8ca37e
+  us-west-2: ami-d2f489aa
+  us-west-1: ami-6b81980b
 
 ecs_cluster_name: "{{ resource_prefix }}"
 user_data: |
@@ -33,3 +35,11 @@ ecs_service_placement_strategy:
 ecs_task_container_port: 8080
 ecs_target_group_name: "{{ resource_prefix[:28] }}-tg"
 ecs_load_balancer_name: "{{ resource_prefix[:29] }}-lb"
+ecs_fargate_task_containers:
+- name: "{{ ecs_task_name }}"
+  image: "{{ ecs_task_image_path }}"
+  essential: true
+  portMappings:
+  - containerPort: "{{ ecs_task_container_port }}"
+    hostPort: "{{ ecs_task_host_port|default(0) }}"
+  #mountPoints: "{{ ecs_task_mount_points|default([]) }}"

test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml

@@ -344,6 +344,11 @@
         that:
           - delete_ecs_service.changed
 
+    - name: assert that deleting ECS service worked
+      assert:
+        that:
+          - delete_ecs_service.changed
+
     - name: create VPC-networked task definition with host port set to 0 (expected to fail)
       ecs_taskdefinition:
         containers: "{{ ecs_task_containers }}"
@@ -382,6 +387,17 @@
         that:
           - "ecs_taskdefinition_facts.network_mode == 'awsvpc'"
 
+    - name: pause to allow service to scale down
+      pause:
+        seconds: 60
+
+    - name: delete ECS service definition
+      ecs_service:
+        state: absent
+        name: "{{ ecs_service_name }}4"
+        cluster: "{{ ecs_cluster_name }}"
+        <<: *aws_connection_info
+      register: delete_ecs_service
 
     - name: create ECS service definition with network configuration
       ecs_service:
@@ -448,7 +464,6 @@
           - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.subnets|length == 2"
           - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.securityGroups|length == 1"
 
-
     - name: obtain facts for all ECS services in the cluster
       ecs_service_facts:
         cluster: "{{ ecs_cluster_name }}"
@@ -511,6 +526,103 @@
         task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition.taskdefinition.revision + 1}}"
         <<: *aws_connection_info
 
+    # ============================================================
+    # Begin tests for Fargate
+
+    - name: create Fargate VPC-networked task definition with host port set to 8080 and unsupported network mode (expected to fail)
+      ecs_taskdefinition:
+        containers: "{{ ecs_fargate_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        network_mode: bridge
+        launch_type: FARGATE
+        cpu: 512
+        memory: 1024
+        state: present
+        <<: *aws_connection_info
+      vars:
+        ecs_task_host_port: 8080
+      ignore_errors: yes
+      register: ecs_fargate_task_definition_bridged_with_host_port
+
+    - name: check that fargate task definition with bridged networking fails gracefully
+      assert:
+        that:
+          - ecs_fargate_task_definition_bridged_with_host_port is failed
+          - 'ecs_fargate_task_definition_bridged_with_host_port.msg == "To use FARGATE launch type, network_mode must be awsvpc"' 
+
+    - name: create Fargate VPC-networked task definition without CPU or Memory (expected to Fail)
+      ecs_taskdefinition:
+        containers: "{{ ecs_fargate_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        network_mode: awsvpc
+        launch_type: FARGATE
+        state: present
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_fargate_task_definition_vpc_no_mem
+
+    - name: check that fargate task definition without memory or cpu fails gracefully
+      assert:
+        that:
+          - ecs_fargate_task_definition_vpc_no_mem is failed
+          - 'ecs_fargate_task_definition_vpc_no_mem.msg == "launch_type is FARGATE but all of the following are missing: cpu, memory"' 
+
+    - name: create Fargate VPC-networked task definition with CPU or Memory 
+      ecs_taskdefinition:
+        containers: "{{ ecs_fargate_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        network_mode: awsvpc
+        launch_type: FARGATE
+        cpu: 512
+        memory: 1024
+        state: present
+        <<: *aws_connection_info
+      vars:
+        ecs_task_host_port: 8080
+      register: ecs_fargate_task_definition
+
+    - name: obtain ECS task definition facts
+      ecs_taskdefinition_facts:
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
+        <<: *aws_connection_info
+
+    - name: create fargate ECS service without network config (expected to fail)
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}4"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
+        desired_count: 1
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        launch_type: FARGATE
+        <<: *aws_connection_info
+      register: ecs_fargate_service_network_without_awsvpc
+      ignore_errors: yes
+     
+    - name: assert that using Fargate ECS service fails
+      assert:
+        that:
+          - ecs_fargate_service_network_without_awsvpc is failed
+
+    - name: create fargate ECS service with network config 
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}4"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
+        desired_count: 1
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        launch_type: FARGATE
+        network_configuration:
+          subnets: "{{ setup_subnet.results | json_query('[].subnet.id') }}"
+          security_groups:
+            - '{{ setup_sg.group_id }}'
+        <<: *aws_connection_info
+      register: ecs_fargate_service_network_with_awsvpc
+
+    # ============================================================
+    # End tests for Fargate
+
   always:
     # TEAR DOWN: snapshot, ec2 instance, ec2 key pair, security group, vpc
     - name: Announce teardown start
@@ -568,6 +680,18 @@
       ignore_errors: yes
       register: ecs_service_scale_down
 
+    - name: scale down Fargate ECS service
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}4"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
+        desired_count: 0
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_service_scale_down
+
     - name: pause to allow services to scale down
       pause:
         seconds: 60
@@ -589,6 +713,15 @@
         <<: *aws_connection_info
       ignore_errors: yes
 
+    - name: remove fargate ECS service 
+      ecs_service:
+        state: absent
+        name: "{{ ecs_service_name }}4"
+        cluster: "{{ ecs_cluster_name }}"
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_fargate_service_network_with_awsvpc
+ 
     - name: remove ecs task definition
       ecs_taskdefinition:
         containers: "{{ ecs_task_containers }}"
@@ -600,6 +733,17 @@
         ecs_task_host_port: 8080
       ignore_errors: yes
 
+    - name: remove ecs task definition again
+      ecs_taskdefinition:
+        containers: "{{ ecs_task_containers }}"
+        family: "{{ ecs_task_name }}"
+        revision: "{{ ecs_task_definition_again.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      vars:
+        ecs_task_host_port: 8080
+      ignore_errors: yes
+
     - name: remove second ecs task definition
       ecs_taskdefinition:
         containers: "{{ ecs_task_containers }}"
@@ -611,6 +755,15 @@
         ecs_task_host_port: 8080
       ignore_errors: yes
 
+    - name: remove fargate ecs task definition
+      ecs_taskdefinition:
+        containers: "{{ ecs_fargate_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        revision: "{{ ecs_fargate_task_definition.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      ignore_errors: yes
+
     - name: remove load balancer
       elb_application_lb:
         name: "{{ ecs_load_balancer_name }}"