From cae14e16ac684ae29362bf66a05ceb768edd07bf Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Thu, 14 Dec 2017 18:41:03 -0500 Subject: [PATCH] Port ec2_vpc_net to boto3 and add support to expand existing VPCs - fixes #31216 (#33105) * Port ec2_vpc_net to boto3 and add support to expand existing VPCs * Add s-hertel as an author for ec2_vpc_net * Update ec2_vpc_net test for new error triggered by lack of credentials Fix backwards compatibility Document new return value * Fix pep8 and return documentation --- .../modules/cloud/amazon/ec2_vpc_net.py | 247 ++++++++++-------- .../targets/ec2_vpc_net/tasks/main.yml | 2 +- 2 files changed, 139 insertions(+), 110 deletions(-) diff --git a/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py b/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py index 520424bfb42..a33a98c5890 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py +++ b/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py @@ -16,61 +16,63 @@ DOCUMENTATION = ''' module: ec2_vpc_net short_description: Configure AWS virtual private clouds description: - - Create or terminate AWS virtual private clouds. This module has a dependency on python-boto. + - Create, modify, and terminate AWS virtual private clouds. version_added: "2.0" -author: Jonathan Davila (@defionscode) +author: + - Jonathan Davila (@defionscode) + - Sloane Hertel (@s-hertel) options: name: description: - - The name to give your VPC. This is used in combination with the cidr_block parameter to determine if a VPC already exists. + - The name to give your VPC. This is used in combination with C(cidr_block) to determine if a VPC already exists. required: yes cidr_block: description: - - The CIDR of the VPC + - The primary CIDR of the VPC. After 2.5 a list of CIDRs can be provided. The first in the list will be used as the primary CIDR + and is used in conjunction with the C(name) to ensure idempotence. required: yes + purge_cidrs: + description: + - Remove CIDRs that are associated with the VPC and are not specified in C(cidr_block). + default: no + choices: [ 'yes', 'no' ] + version_added: '2.5' tenancy: description: - Whether to be default or dedicated tenancy. This cannot be changed after the VPC has been created. - required: false default: default choices: [ 'default', 'dedicated' ] dns_support: description: - Whether to enable AWS DNS support. - required: false default: yes choices: [ 'yes', 'no' ] dns_hostnames: description: - Whether to enable AWS hostname support. - required: false default: yes choices: [ 'yes', 'no' ] dhcp_opts_id: description: - the id of the DHCP options to use for this vpc - default: null - required: false tags: description: - The tags you want attached to the VPC. This is independent of the name value, note if you pass a 'Name' key it would override the Name of the VPC if it's different. - default: None - required: false aliases: [ 'resource_tags' ] state: description: - The state of the VPC. Either absent or present. default: present - required: false choices: [ 'present', 'absent' ] multi_ok: description: - By default the module will not create another VPC if there is another VPC with the same name and CIDR block. Specify this as true if you want duplicate VPCs created. default: false - required: false - +requirements: + - boto3 + - botocore extends_documentation_fragment: - aws - ec2 @@ -133,17 +135,30 @@ vpc.is_default: returned: success type: boolean sample: false +vpc.cidr_block_association_set: + description: IPv4 CIDR blocks associated with the VPC + returned: success + type: list + sample: + "cidr_block_association_set": [ + { + "association_id": "vpc-cidr-assoc-97aeeefd", + "cidr_block": "20.0.0.0/24", + "cidr_block_state": { + "state": "associated" + } + } + ] ''' try: - import boto.vpc - from boto.exception import BotoServerError, NoAuthHandlerFound + import botocore except ImportError: - pass # Taken care of by ec2.HAS_BOTO + pass # Handled by AnsibleAWSModule -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.ec2 import (HAS_BOTO, AnsibleAWSError, boto_exception, connect_to_aws, - ec2_argument_spec, get_aws_connection_info) +from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec, ansible_dict_to_boto3_tag_list, camel_dict_to_snake_dict +from ansible.module_utils.six import string_types def vpc_exists(module, vpc, name, cidr_block, multi): @@ -151,95 +166,102 @@ def vpc_exists(module, vpc, name, cidr_block, multi): with a CIDR, it will check for matching tags to determine if it is a match otherwise it will assume the VPC does not exist and thus return None. """ - matched_vpc = None - try: - matching_vpcs = vpc.get_all_vpcs(filters={'tag:Name': name, 'cidr-block': cidr_block}) - except Exception as e: - e_msg = boto_exception(e) - module.fail_json(msg=e_msg) + matching_vpcs = vpc.describe_vpcs(Filters=[{'Name': 'tag:Name', 'Values': [name]}, {'Name': 'cidr-block', 'Values': cidr_block}])['Vpcs'] + # If an exact matching using a list of CIDRs isn't found, check for a match with the first CIDR as is documented for C(cidr_block) + if not matching_vpcs: + matching_vpcs = vpc.describe_vpcs(Filters=[{'Name': 'tag:Name', 'Values': [name]}, {'Name': 'cidr-block', 'Values': [cidr_block[0]]}])['Vpcs'] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to describe VPCs") if multi: return None elif len(matching_vpcs) == 1: - matched_vpc = matching_vpcs[0] + return matching_vpcs[0]['VpcId'] elif len(matching_vpcs) > 1: module.fail_json(msg='Currently there are %d VPCs that have the same name and ' 'CIDR block you specified. If you would like to create ' 'the VPC anyway please pass True to the multi_ok param.' % len(matching_vpcs)) - - return matched_vpc + return None -def update_vpc_tags(vpc, module, vpc_obj, tags, name): +def get_vpc(module, connection, vpc_id): + try: + vpc_obj = connection.describe_vpcs(VpcIds=[vpc_id])['Vpcs'][0] + classic_link = connection.describe_vpc_classic_link(VpcIds=[vpc_id])['Vpcs'][0].get('ClassicLinkEnabled') + vpc_obj['ClassicLinkEnabled'] = classic_link + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to describe VPCs") + + return vpc_obj + + +def update_vpc_tags(connection, module, vpc_id, tags, name): if tags is None: tags = dict() tags.update({'Name': name}) try: - current_tags = dict((t.name, t.value) for t in vpc.get_all_tags(filters={'resource-id': vpc_obj.id})) + current_tags = dict((t['Key'], t['Value']) for t in connection.describe_tags(Filters=[{'Name': 'resource-id', 'Values': [vpc_id]}])['Tags']) if tags != current_tags: if not module.check_mode: - vpc.create_tags(vpc_obj.id, tags) + tags = ansible_dict_to_boto3_tag_list(tags) + connection.create_tags(Resources=[vpc_id], Tags=tags) return True else: return False - except Exception as e: - e_msg = boto_exception(e) - module.fail_json(msg=e_msg) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to update tags") def update_dhcp_opts(connection, module, vpc_obj, dhcp_id): - - if vpc_obj.dhcp_options_id != dhcp_id: + if vpc_obj['DhcpOptionsId'] != dhcp_id: if not module.check_mode: - connection.associate_dhcp_options(dhcp_id, vpc_obj.id) + try: + connection.associate_dhcp_options(DhcpOptionsId=dhcp_id, VpcId=vpc_obj['VpcId']) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to associate DhcpOptionsId {0}".format(dhcp_id)) return True else: return False -def get_vpc_values(vpc_obj): - - if vpc_obj is not None: - vpc_values = vpc_obj.__dict__ - if "region" in vpc_values: - vpc_values.pop("region") - if "item" in vpc_values: - vpc_values.pop("item") - if "connection" in vpc_values: - vpc_values.pop("connection") - return vpc_values - else: - return None +def create_vpc(connection, module, cidr_block, tenancy): + try: + if not module.check_mode: + vpc_obj = connection.create_vpc(CidrBlock=cidr_block, InstanceTenancy=tenancy) + else: + module.exit_json(changed=True) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "Failed to create the VPC") + return vpc_obj['Vpc']['VpcId'] def main(): argument_spec = ec2_argument_spec() argument_spec.update(dict( - name=dict(type='str', default=None, required=True), - cidr_block=dict(type='str', default=None, required=True), + name=dict(required=True), + cidr_block=dict(type='list', required=True), tenancy=dict(choices=['default', 'dedicated'], default='default'), dns_support=dict(type='bool', default=True), dns_hostnames=dict(type='bool', default=True), - dhcp_opts_id=dict(type='str', default=None, required=False), - tags=dict(type='dict', required=False, default=None, aliases=['resource_tags']), + dhcp_opts_id=dict(), + tags=dict(type='dict', aliases=['resource_tags']), state=dict(choices=['present', 'absent'], default='present'), - multi_ok=dict(type='bool', default=False) + multi_ok=dict(type='bool', default=False), + purge_cidrs=dict(type='bool', default=False), ) ) - module = AnsibleModule( + module = AnsibleAWSModule( argument_spec=argument_spec, supports_check_mode=True ) - if not HAS_BOTO: - module.fail_json(msg='boto is required for this module') - name = module.params.get('name') cidr_block = module.params.get('cidr_block') + purge_cidrs = module.params.get('purge_cidrs') tenancy = module.params.get('tenancy') dns_support = module.params.get('dns_support') dns_hostnames = module.params.get('dns_hostnames') @@ -250,15 +272,8 @@ def main(): changed = False - region, ec2_url, aws_connect_params = get_aws_connection_info(module) - - if region: - try: - connection = connect_to_aws(boto.vpc, region, **aws_connect_params) - except (NoAuthHandlerFound, AnsibleAWSError) as e: - module.fail_json(msg=str(e)) - else: - module.fail_json(msg="region must be specified") + region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) + connection = boto3_conn(module, conn_type='client', resource='ec2', region=region, **aws_connect_params) if dns_hostnames and not dns_support: module.fail_json(msg='In order to enable DNS Hostnames you must also enable DNS support') @@ -266,70 +281,84 @@ def main(): if state == 'present': # Check if VPC exists - vpc_obj = vpc_exists(module, connection, name, cidr_block, multi) + vpc_id = vpc_exists(module, connection, name, cidr_block, multi) - if vpc_obj is None: - try: + if vpc_id is None: + vpc_id = create_vpc(connection, module, cidr_block[0], tenancy) + changed = True + + vpc_obj = get_vpc(module, connection, vpc_id) + + associated_cidrs = dict((cidr['CidrBlock'], cidr['AssociationId']) for cidr in vpc_obj.get('CidrBlockAssociationSet', []) + if cidr['CidrBlockState']['State'] != 'disassociated') + to_add = [cidr for cidr in cidr_block if cidr not in associated_cidrs] + to_remove = [associated_cidrs[cidr] for cidr in associated_cidrs if cidr not in cidr_block] + + if len(cidr_block) > 1: + for cidr in to_add: changed = True - if not module.check_mode: - vpc_obj = connection.create_vpc(cidr_block, instance_tenancy=tenancy) - else: - module.exit_json(changed=changed) - except BotoServerError as e: - module.fail_json(msg=e) + connection.associate_vpc_cidr_block(CidrBlock=cidr, VpcId=vpc_id) + + if purge_cidrs: + for association_id in to_remove: + changed = True + try: + connection.disassociate_vpc_cidr_block(AssociationId=association_id) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "Unable to disassociate {0}. You must detach or delete all gateways and resources that " + "are associated with the CIDR block before you can disassociate it.".format(association_id)) if dhcp_id is not None: try: if update_dhcp_opts(connection, module, vpc_obj, dhcp_id): changed = True - except BotoServerError as e: - module.fail_json(msg=e) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "Failed to update DHCP options") if tags is not None or name is not None: try: - if update_vpc_tags(connection, module, vpc_obj, tags, name): + if update_vpc_tags(connection, module, vpc_id, tags, name): changed = True - except BotoServerError as e: - module.fail_json(msg=e) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to update tags") - # Note: Boto currently doesn't currently provide an interface to ec2-describe-vpc-attribute - # which is needed in order to detect the current status of DNS options. For now we just update - # the attribute each time and is not used as a changed-factor. - try: + current_dns_enabled = connection.describe_vpc_attribute(Attribute='enableDnsSupport', VpcId=vpc_id)['EnableDnsSupport']['Value'] + current_dns_hostnames = connection.describe_vpc_attribute(Attribute='enableDnsHostnames', VpcId=vpc_id)['EnableDnsHostnames']['Value'] + if current_dns_enabled != dns_support: + changed = True if not module.check_mode: - connection.modify_vpc_attribute(vpc_obj.id, enable_dns_support=dns_support) - connection.modify_vpc_attribute(vpc_obj.id, enable_dns_hostnames=dns_hostnames) - except BotoServerError as e: - e_msg = boto_exception(e) - module.fail_json(msg=e_msg) + try: + connection.modify_vpc_attribute(VpcId=vpc_id, EnableDnsSupport={'Value': dns_support}) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "Failed to update enabled dns support attribute") + if current_dns_hostnames != dns_hostnames: + changed = True + if not module.check_mode: + try: + connection.modify_vpc_attribute(VpcId=vpc_id, EnableDnsHostnames={'Value': dns_hostnames}) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, "Failed to update enabled dns hostnames attribute") - if not module.check_mode: - # get the vpc obj again in case it has changed - try: - vpc_obj = connection.get_all_vpcs(vpc_obj.id)[0] - except BotoServerError as e: - e_msg = boto_exception(e) - module.fail_json(msg=e_msg) + final_state = camel_dict_to_snake_dict(get_vpc(module, connection, vpc_id)) + final_state['id'] = final_state.pop('vpc_id') - module.exit_json(changed=changed, vpc=get_vpc_values(vpc_obj)) + module.exit_json(changed=changed, vpc=final_state) elif state == 'absent': # Check if VPC exists - vpc_obj = vpc_exists(module, connection, name, cidr_block, multi) + vpc_id = vpc_exists(module, connection, name, cidr_block, multi) - if vpc_obj is not None: + if vpc_id is not None: try: if not module.check_mode: - connection.delete_vpc(vpc_obj.id) - vpc_obj = None + connection.delete_vpc(VpcId=vpc_id) changed = True - except BotoServerError as e: - e_msg = boto_exception(e) - module.fail_json(msg="%s. You may want to use the ec2_vpc_subnet, ec2_vpc_igw, " - "and/or ec2_vpc_route_table modules to ensure the other components are absent." % e_msg) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to delete VPC {0} You may want to use the ec2_vpc_subnet, ec2_vpc_igw, " + "and/or ec2_vpc_route_table modules to ensure the other components are absent.".format(vpc_id)) - module.exit_json(changed=changed, vpc=get_vpc_values(vpc_obj)) + module.exit_json(changed=changed, vpc={}) if __name__ == '__main__': diff --git a/test/integration/targets/ec2_vpc_net/tasks/main.yml b/test/integration/targets/ec2_vpc_net/tasks/main.yml index f4a22b306e1..9e8eb5d09f5 100644 --- a/test/integration/targets/ec2_vpc_net/tasks/main.yml +++ b/test/integration/targets/ec2_vpc_net/tasks/main.yml @@ -29,7 +29,7 @@ assert: that: - 'result is failed' - - 'result.msg.startswith("No handler was ready to authenticate")' + - '"Unable to locate credentials" in result.msg' # ============================================================