cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add Fargate support for ECS modules

Browse source 
Fargate instances do not require memory and cpu descriptors. EC2 instances
 do require descriptions. https://botocore.readthedocs.io/en/latest/reference/services/ecs.html#ECS.Client.describe_task_definition

Fargate requires that cpu and memory be defined at task definition level.
EC2 launch requires them to be defined at the container level.

Fargate requires the use of awsvpc for the networking_mode. Also updated,
the documentation regarding where and when memory/cpu needs to the assigned.

The task_definition variable for the awspvc configuration colided with
the ecs_service for the bridge network. This would cause the test to fail.

Add testing for fargate

Add examples for fargate and ec2
This commit is contained in:
Michael Mayer 2018-05-24 11:29:20 -07:00 committed by Will Thames
parent 8eb9cc3217
commit fbcd6f8a65
5 changed files with 307 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
lib/ansible/modules/cloud/amazon/ecs_service.py
View file

@ -107,9 +107,8 @@ options:
description:
- The launch type on which to run your service
required: false
version_added: 2.5
version_added: 2.7
choices: ["EC2", "FARGATE"]
default: "EC2"
extends_documentation_fragment:
- aws
- ec2
@ -376,10 +375,12 @@ class EcsServiceManager:
role=role,
deploymentConfiguration=deployment_configuration,
placementConstraints=placement_constraints,
placementStrategy=placement_strategy,
launchType=launch_type)
placementStrategy=placement_strategy
)
if network_configuration:
params['networkConfiguration'] = network_configuration
if launch_type:
params['launchType'] = launch_type
response = self.ecs.create_service(**params)
return self.jsonize(response['service'])
@ -441,12 +442,13 @@ def main():
placement_constraints=dict(required=False, default=[], type='list'),
placement_strategy=dict(required=False, default=[], type='list'),
network_configuration=dict(required=False, type='dict'),
launch_type=dict(required=False, choices=['EC2', 'FARGATE'], default='EC2')
launch_type=dict(required=False, choices=['EC2', 'FARGATE'])
))
module = AnsibleAWSModule(argument_spec=argument_spec,
supports_check_mode=True,
required_if=[('state', 'present', ['task_definition', 'desired_count'])],
required_if=[('state', 'present', ['task_definition', 'desired_count']),
('launch_type', 'FARGATE', ['network_configuration'])],
required_together=[['load_balancers', 'role']])
service_mgr = EcsServiceManager(module)
@ -468,10 +470,16 @@ def main():
module.fail_json(msg="Exception describing service '" + module.params['name'] + "' in cluster '" + module.params['cluster'] + "': " + str(e))
results = dict(changed=False)
if module.params['launch_type']:
if not module.botocore_at_least('1.8.4'):
module.fail_json(msg='botocore needs to be version 1.8.4 or higher to use launch_type')
if module.params['state'] == 'present':
matching = False
update = False
if existing and 'status' in existing and existing['status'] == "ACTIVE":
if service_mgr.is_matching_service(module.params, existing):
matching = True
@ -501,6 +509,7 @@ def main():
if 'containerPort' in loadBalancer:
loadBalancer['containerPort'] = int(loadBalancer['containerPort'])
# doesn't exist. create it.
try:
response = service_mgr.create_service(module.params['name'],
module.params['cluster'],
module.params['task_definition'],
@ -513,6 +522,8 @@ def main():
module.params['placement_strategy'],
network_configuration,
module.params['launch_type'])
except botocore.exceptions.ClientError as e:
module.fail_json(msg=e.message)
results['service'] = response

70
lib/ansible/modules/cloud/amazon/ecs_taskdefinition.py
View file

@ -77,21 +77,20 @@ options:
description:
- The launch type on which to run your task
required: false
version_added: 2.5
version_added: 2.7
choices: ["EC2", "FARGATE"]
default: "EC2"
cpu:
description:
- The number of cpu units used by the task. If using the EC2 launch type, this field is optional and any value can be used.
If using the Fargate launch type, this field is required and you must use one of [256, 512, 1024, 2048, 4096]
required: false
version_added: 2.5
version_added: 2.7
memory:
description:
- The amount (in MiB) of memory used by the task. If using the EC2 launch type, this field is optional and any value can be used.
If using the Fargate launch type, this field is required and is limited by the cpu
required: false
version_added: 2.5
version_added: 2.7
extends_documentation_fragment:
- aws
- ec2
@ -137,6 +136,36 @@ EXAMPLES = '''
family: test-cluster-taskdef
state: present
register: task_output
- name: Create task definition
ecs_taskdefinition:
family: nginx
containers:
- name: nginx
essential: true
image: "nginx"
portMappings:
- containerPort: 8080
hostPort: 8080
cpu: 512
memory: 1GB
state: present
- name: Create task definition
ecs_taskdefinition:
family: nginx
containers:
- name: nginx
essential: true
image: "nginx"
portMappings:
- containerPort: 8080
hostPort: 8080
launch_type: FARGATE
cpu: 512
memory: 1GB
state: present
network_mode: awsvpc
'''
RETURN = '''
taskdefinition:
@ -198,13 +227,14 @@ class EcsTaskManager:
taskRoleArn=task_role_arn,
networkMode=network_mode,
containerDefinitions=container_definitions,
volumes=volumes,
requiresCompatibilities=launch_type
volumes=volumes
)
if cpu:
params['cpu'] = cpu
if memory:
params['memory'] = memory
if launch_type:
params['requiresCompatibilities'] = [launch_type]
try:
response = self.ecs.register_task_definition(**params)
@ -249,9 +279,14 @@ class EcsTaskManager:
response = self.ecs.deregister_task_definition(taskDefinition=taskArn)
return response['taskDefinition']
def ecs_api_supports_requirescompatibilities(self):
from distutils.version import LooseVersion
# Checking to make sure botocore is greater than a specific version.
# Support for requiresCompatibilities is only available in versions beyond 1.8.4
return LooseVersion(botocore.__version__) >= LooseVersion('1.8.4')
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(dict(
state=dict(required=True, choices=['present', 'absent']),
@ -263,12 +298,15 @@ def main():
network_mode=dict(required=False, default='bridge', choices=['bridge', 'host', 'none', 'awsvpc'], type='str'),
task_role_arn=dict(required=False, default='', type='str'),
volumes=dict(required=False, type='list'),
launch_type=dict(required=False, choices=['EC2', 'FARGATE'], default='EC2'),
cpu=dict(required=False, type='str'),
launch_type=dict(required=False, choices=['EC2', 'FARGATE']),
cpu=dict(),
memory=dict(required=False, type='str')
))
module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True)
module = AnsibleModule(argument_spec=argument_spec,
supports_check_mode=True,
required_if=[('launch_type', 'FARGATE', ['cpu', 'memory'])]
)
if not HAS_BOTO3:
module.fail_json(msg='boto3 is required.')
@ -277,6 +315,10 @@ def main():
task_mgr = EcsTaskManager(module)
results = dict(changed=False)
if module.params['launch_type']:
if not task_mgr.ecs_api_supports_requirescompatibilities():
module.fail_json(msg='botocore needs to be version 1.8.4 or higher to use launch_type')
for container in module.params.get('containers', []):
for environment in container.get('environment', []):
environment['value'] = to_text(environment['value'])
@ -288,12 +330,10 @@ def main():
if 'family' not in module.params or not module.params['family']:
module.fail_json(msg="To use task definitions, a family must be specified")
network_mode = module.params['network_mode']
launch_type = module.params['launch_type']
if launch_type == 'EC2' and ('cpu' not in module.params or not module.params['cpu']):
module.fail_json(msg="To use FARGATE launch type, cpu must be specified")
if launch_type == 'EC2' and ('memory' not in module.params or not module.params['memory']):
module.fail_json(msg="To use FARGATE launch type, memory must be specified")
if launch_type == 'FARGATE' and network_mode != 'awsvpc':
module.fail_json(msg="To use FARGATE launch type, network_mode must be awsvpc")
family = module.params['family']
existing_definitions_in_family = task_mgr.describe_task_definitions(module.params['family'])

58
test/integration/targets/ecs_cluster/playbooks/network_fail.yml
View file

@ -1,5 +1,7 @@
- hosts: localhost
connection: local
vars:
resource_prefix: 'ansible-testing'
tasks:
- block:
@ -61,7 +63,7 @@
ecs_service:
name: "{{ resource_prefix }}-vpc"
cluster: "{{ resource_prefix }}"
task_definition: "{{ resource_prefix }}"
task_definition: "{{ resource_prefix }}-vpc"
desired_count: 1
network_configuration:
subnets:
@ -79,6 +81,47 @@
- ecs_service_creation_vpc.failed
- 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
- name: create ecs_service using awsvpc network_configuration and launch_type
ecs_service:
name: "{{ resource_prefix }}-vpc"
cluster: "{{ resource_prefix }}"
task_definition: "{{ resource_prefix }}-vpc"
desired_count: 1
network_configuration:
subnets:
- subnet-abcd1234
groups:
- sg-abcd1234
launch_type: FARGATE
state: present
<<: *aws_connection_info
register: ecs_service_creation_vpc_launchtype
ignore_errors: yes
- name: check that graceful failure message is returned from ecs_service
assert:
that:
- ecs_service_creation_vpc_launchtype.failed
- 'ecs_service_creation_vpc_launchtype.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
- name: create ecs_service with launchtype and missing network_configuration
ecs_service:
name: "{{ resource_prefix }}-vpc"
cluster: "{{ resource_prefix }}"
task_definition: "{{ resource_prefix }}-vpc"
desired_count: 1
launch_type: FARGATE
state: present
<<: *aws_connection_info
register: ecs_service_creation_vpc_launchtype_nonet
ignore_errors: yes
- name: check that graceful failure message is returned from ecs_service
assert:
that:
- ecs_service_creation_vpc_launchtype_nonet.failed
- 'ecs_service_creation_vpc_launchtype_nonet.msg == "launch_type is FARGATE but all of the following are missing: network_configuration"'
- name: create ecs_task using awsvpc network_configuration
ecs_task:
cluster: "{{ resource_prefix }}-vpc"
@ -101,6 +144,7 @@
- ecs_task_creation_vpc.failed
- 'ecs_task_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
always:
- name: scale down ecs service
ecs_service:
@ -138,6 +182,18 @@
<<: *aws_connection_info
ignore_errors: yes
- name: remove ecs task definition vpc
ecs_taskdefinition:
containers:
- name: my_container
image: ubuntu
memory: 128
family: "{{ resource_prefix }}-vpc"
revision: "{{ ecs_taskdefinition_creation_vpc.taskdefinition.revision }}"
state: absent
<<: *aws_connection_info
ignore_errors: yes
- name: remove ecs cluster
ecs_cluster:
name: "{{ resource_prefix }}"

10
test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/defaults/main.yml
View file

@ -3,6 +3,8 @@
ecs_agent_images:
us-east-1: ami-71ef560b
us-east-2: ami-1b8ca37e
us-west-2: ami-d2f489aa
us-west-1: ami-6b81980b
ecs_cluster_name: "{{ resource_prefix }}"
user_data: |
@ -33,3 +35,11 @@ ecs_service_placement_strategy:
ecs_task_container_port: 8080
ecs_target_group_name: "{{ resource_prefix[:28] }}-tg"
ecs_load_balancer_name: "{{ resource_prefix[:29] }}-lb"
ecs_fargate_task_containers:
- name: "{{ ecs_task_name }}"
image: "{{ ecs_task_image_path }}"
essential: true
portMappings:
- containerPort: "{{ ecs_task_container_port }}"
hostPort: "{{ ecs_task_host_port|default(0) }}"
#mountPoints: "{{ ecs_task_mount_points|default([]) }}"

155
test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml
View file

@ -344,6 +344,11 @@
that:
- delete_ecs_service.changed
- name: assert that deleting ECS service worked
assert:
that:
- delete_ecs_service.changed
- name: create VPC-networked task definition with host port set to 0 (expected to fail)
ecs_taskdefinition:
containers: "{{ ecs_task_containers }}"
@ -382,6 +387,17 @@
that:
- "ecs_taskdefinition_facts.network_mode == 'awsvpc'"
- name: pause to allow service to scale down
pause:
seconds: 60
- name: delete ECS service definition
ecs_service:
state: absent
name: "{{ ecs_service_name }}4"
cluster: "{{ ecs_cluster_name }}"
<<: *aws_connection_info
register: delete_ecs_service
- name: create ECS service definition with network configuration
ecs_service:
@ -448,7 +464,6 @@
- "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.subnets|length == 2"
- "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.securityGroups|length == 1"
- name: obtain facts for all ECS services in the cluster
ecs_service_facts:
cluster: "{{ ecs_cluster_name }}"
@ -511,6 +526,103 @@
task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition.taskdefinition.revision + 1}}"
<<: *aws_connection_info
# ============================================================
# Begin tests for Fargate
- name: create Fargate VPC-networked task definition with host port set to 8080 and unsupported network mode (expected to fail)
ecs_taskdefinition:
containers: "{{ ecs_fargate_task_containers }}"
family: "{{ ecs_task_name }}-vpc"
network_mode: bridge
launch_type: FARGATE
cpu: 512
memory: 1024
state: present
<<: *aws_connection_info
vars:
ecs_task_host_port: 8080
ignore_errors: yes
register: ecs_fargate_task_definition_bridged_with_host_port
- name: check that fargate task definition with bridged networking fails gracefully
assert:
that:
- ecs_fargate_task_definition_bridged_with_host_port is failed
- 'ecs_fargate_task_definition_bridged_with_host_port.msg == "To use FARGATE launch type, network_mode must be awsvpc"'
- name: create Fargate VPC-networked task definition without CPU or Memory (expected to Fail)
ecs_taskdefinition:
containers: "{{ ecs_fargate_task_containers }}"
family: "{{ ecs_task_name }}-vpc"
network_mode: awsvpc
launch_type: FARGATE
state: present
<<: *aws_connection_info
ignore_errors: yes
register: ecs_fargate_task_definition_vpc_no_mem
- name: check that fargate task definition without memory or cpu fails gracefully
assert:
that:
- ecs_fargate_task_definition_vpc_no_mem is failed
- 'ecs_fargate_task_definition_vpc_no_mem.msg == "launch_type is FARGATE but all of the following are missing: cpu, memory"'
- name: create Fargate VPC-networked task definition with CPU or Memory
ecs_taskdefinition:
containers: "{{ ecs_fargate_task_containers }}"
family: "{{ ecs_task_name }}-vpc"
network_mode: awsvpc
launch_type: FARGATE
cpu: 512
memory: 1024
state: present
<<: *aws_connection_info
vars:
ecs_task_host_port: 8080
register: ecs_fargate_task_definition
- name: obtain ECS task definition facts
ecs_taskdefinition_facts:
task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
<<: *aws_connection_info
- name: create fargate ECS service without network config (expected to fail)
ecs_service:
state: present
name: "{{ ecs_service_name }}4"
cluster: "{{ ecs_cluster_name }}"
task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
desired_count: 1
deployment_configuration: "{{ ecs_service_deployment_configuration }}"
launch_type: FARGATE
<<: *aws_connection_info
register: ecs_fargate_service_network_without_awsvpc
ignore_errors: yes
- name: assert that using Fargate ECS service fails
assert:
that:
- ecs_fargate_service_network_without_awsvpc is failed
- name: create fargate ECS service with network config
ecs_service:
state: present
name: "{{ ecs_service_name }}4"
cluster: "{{ ecs_cluster_name }}"
task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
desired_count: 1
deployment_configuration: "{{ ecs_service_deployment_configuration }}"
launch_type: FARGATE
network_configuration:
subnets: "{{ setup_subnet.results | json_query('[].subnet.id') }}"
security_groups:
- '{{ setup_sg.group_id }}'
<<: *aws_connection_info
register: ecs_fargate_service_network_with_awsvpc
# ============================================================
# End tests for Fargate
always:
# TEAR DOWN: snapshot, ec2 instance, ec2 key pair, security group, vpc
- name: Announce teardown start
@ -568,6 +680,18 @@
ignore_errors: yes
register: ecs_service_scale_down
- name: scale down Fargate ECS service
ecs_service:
state: present
name: "{{ ecs_service_name }}4"
cluster: "{{ ecs_cluster_name }}"
task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_fargate_task_definition.taskdefinition.revision }}"
desired_count: 0
deployment_configuration: "{{ ecs_service_deployment_configuration }}"
<<: *aws_connection_info
ignore_errors: yes
register: ecs_service_scale_down
- name: pause to allow services to scale down
pause:
seconds: 60
@ -589,6 +713,15 @@
<<: *aws_connection_info
ignore_errors: yes
- name: remove fargate ECS service
ecs_service:
state: absent
name: "{{ ecs_service_name }}4"
cluster: "{{ ecs_cluster_name }}"
<<: *aws_connection_info
ignore_errors: yes
register: ecs_fargate_service_network_with_awsvpc
- name: remove ecs task definition
ecs_taskdefinition:
containers: "{{ ecs_task_containers }}"
@ -600,6 +733,17 @@
ecs_task_host_port: 8080
ignore_errors: yes
- name: remove ecs task definition again
ecs_taskdefinition:
containers: "{{ ecs_task_containers }}"
family: "{{ ecs_task_name }}"
revision: "{{ ecs_task_definition_again.taskdefinition.revision }}"
state: absent
<<: *aws_connection_info
vars:
ecs_task_host_port: 8080
ignore_errors: yes
- name: remove second ecs task definition
ecs_taskdefinition:
containers: "{{ ecs_task_containers }}"
@ -611,6 +755,15 @@
ecs_task_host_port: 8080
ignore_errors: yes
- name: remove fargate ecs task definition
ecs_taskdefinition:
containers: "{{ ecs_fargate_task_containers }}"
family: "{{ ecs_task_name }}-vpc"
revision: "{{ ecs_fargate_task_definition.taskdefinition.revision }}"
state: absent
<<: *aws_connection_info
ignore_errors: yes
- name: remove load balancer
elb_application_lb:
name: "{{ ecs_load_balancer_name }}"