Feature/aws helper function for tags (#23387)

* Add new helper function for comparing AWS tag key pair dicts. Also modify boto3_tag_list_to_ansible_dict function to be more generic when looking for key names because AWS sometimes uses 'Key', sometimes 'TagKey' and who knows what the future holds! Fixed modules to work with changes.

* Review changes

* Add some more doc to GUIDELINES for tags and fix var name for snaked values in ec2_group_facts
This commit is contained in:
Rob 2017-05-11 16:39:51 +10:00 committed by John R Barker
parent b2a2f69a6e
commit fd1debb869
6 changed files with 76 additions and 26 deletions

View file

@ -415,11 +415,13 @@ def ansible_dict_to_boto3_filter_list(filters_dict):
return filters_list
def boto3_tag_list_to_ansible_dict(tags_list):
def boto3_tag_list_to_ansible_dict(tags_list, tag_name_key_name='Key', tag_value_key_name='Value'):
""" Convert a boto3 list of resource tags to a flat dict of key:value pairs
Args:
tags_list (list): List of dicts representing AWS tags.
tag_name_key_name (str): Value to use as the key for all tag keys (useful because boto3 doesn't always use "Key")
tag_value_key_name (str): Value to use as the key for all tag values (useful because boto3 doesn't always use "Value")
Basic Usage:
>>> tags_list = [{'Key': 'MyTagKey', 'Value': 'MyTagValue'}]
>>> boto3_tag_list_to_ansible_dict(tags_list)
@ -438,19 +440,19 @@ def boto3_tag_list_to_ansible_dict(tags_list):
tags_dict = {}
for tag in tags_list:
if 'key' in tag:
tags_dict[tag['key']] = tag['value']
elif 'Key' in tag:
tags_dict[tag['Key']] = tag['Value']
if tag_name_key_name in tag:
tags_dict[tag[tag_name_key_name]] = tag[tag_value_key_name]
return tags_dict
def ansible_dict_to_boto3_tag_list(tags_dict):
def ansible_dict_to_boto3_tag_list(tags_dict, tag_name_key_name='Key', tag_value_key_name='Value'):
""" Convert a flat dict of key:value pairs representing AWS resource tags to a boto3 list of dicts
Args:
tags_dict (dict): Dict representing AWS resource tags.
tag_name_key_name (str): Value to use as the key for all tag keys (useful because boto3 doesn't always use "Key")
tag_value_key_name (str): Value to use as the key for all tag values (useful because boto3 doesn't always use "Value")
Basic Usage:
>>> tags_dict = {'MyTagKey': 'MyTagValue'}
>>> ansible_dict_to_boto3_tag_list(tags_dict)
@ -469,7 +471,7 @@ def ansible_dict_to_boto3_tag_list(tags_dict):
tags_list = []
for k,v in tags_dict.items():
tags_list.append({'Key': k, 'Value': v})
tags_list.append({tag_name_key_name: k, tag_value_key_name: v})
return tags_list
@ -623,3 +625,29 @@ def map_complex_type(complex_type, type_map):
elif type_map:
return globals()['__builtins__'][type_map](complex_type)
return new_type
def compare_aws_tags(current_tags_dict, new_tags_dict, purge_tags=True):
"""
Compare two dicts of AWS tags. Dicts are expected to of been created using 'boto3_tag_list_to_ansible_dict' helper function.
Two dicts are returned - the first is tags to be set, the second is any tags to remove
:param current_tags_dict:
:param new_tags_dict:
:param purge_tags:
:return: tag_key_value_pairs_to_set: a dict of key value pairs that need to be set in AWS. If all tags are identical this dict will be empty
:return: tag_keys_to_unset: a list of key names that need to be unset in AWS. If no tags need to be unset this list will be empty
"""
tag_key_value_pairs_to_set = {}
tag_keys_to_unset = []
for key in current_tags_dict.keys():
if key not in new_tags_dict and purge_tags:
tag_keys_to_unset.append(key)
for key in set(new_tags_dict.keys()) - set(tag_keys_to_unset):
if new_tags_dict[key] != current_tags_dict.get(key):
tag_key_value_pairs_to_set[key] = new_tags_dict[key]
return tag_key_value_pairs_to_set, tag_keys_to_unset

View file

@ -255,6 +255,19 @@ else:
aws_object.set_policy(user_policy)
```
### Dealing with tags
AWS has a concept of resource tags. Usually the boto3 API has separate calls for tagging and
untagging a resource. For example, the ec2 API has a create_tags and delete_tags call.
It is common practice in Ansible AWS modules to have a 'purge_tags' parameter that defaults to true.
The purge_tags parameter means that existing tags will be deleted if they are not specified in
by the Ansible playbook.
There is a helper function 'compare_aws_tags' to ease dealing with tags. It can compare two dicts and
return the tags to set and the tags to delete. See the Helper function section below for more detail.
### Helper functions
Along with the connection functions in Ansible ec2.py module_utils, there are some other useful functions detailed below.
@ -272,12 +285,15 @@ any boto3 _facts modules.
#### boto3_tag_list_to_ansible_dict
Converts a boto3 tag list to an Ansible dict. Boto3 returns tags as a list of dicts containing keys called
'Key' and 'Value'. This function converts this list in to a single dict where the dict key is the tag
key and the dict value is the tag value.
'Key' and 'Value' by default. This key names can be overriden when calling the function. For example, if you have already
camel_cased your list of tags you may want to pass lowercase key names instead i.e. 'key' and 'value'.
This function converts the list in to a single dict where the dict key is the tag key and the dict value is the tag value.
#### ansible_dict_to_boto3_tag_list
Opposite of above. Converts an Ansible dict to a boto3 tag list of dicts.
Opposite of above. Converts an Ansible dict to a boto3 tag list of dicts. You can again override the key names used if 'Key'
and 'Value' is not suitable.
#### get_ec2_security_group_ids_from_names
@ -290,3 +306,12 @@ across VPCs.
Pass any JSON policy dict to this function in order to sort any list contained therein. This is useful
because AWS rarely return lists in the same order that they were submitted so without this function, comparison
of identical policies returns false.
### compare_aws_tags
Pass two dicts of tags and an optional purge parameter and this function will return a dict containing key pairs you need
to modify and a list of tag key names that you need to remove. Purge is True by default. If purge is False then any
existing tags will not be modified.
This function is useful when using boto3 'add_tags' and 'remove_tags' functions. Be sure to use the other helper function
'boto3_tag_list_to_ansible_dict' to get an appropriate tag dict before calling this function.

View file

@ -73,7 +73,7 @@ from ansible.module_utils.ec2 import (AnsibleAWSError,
connect_to_aws, ec2_argument_spec, get_aws_connection_info)
def list_ec2_snapshots_boto3(connection, module):
def list_ec2_eni_boto3(connection, module):
if module.params.get("filters") is None:
filters = []
@ -81,16 +81,17 @@ def list_ec2_snapshots_boto3(connection, module):
filters = ansible_dict_to_boto3_filter_list(module.params.get("filters"))
try:
network_interfaces_result = connection.describe_network_interfaces(Filters=filters)
network_interfaces_result = connection.describe_network_interfaces(Filters=filters)['NetworkInterfaces']
except (ClientError, NoCredentialsError) as e:
module.fail_json(msg=e.message)
# Turn the boto3 result in to ansible_friendly_snaked_names
snaked_network_interfaces_result = camel_dict_to_snake_dict(network_interfaces_result)
for network_interfaces in snaked_network_interfaces_result['network_interfaces']:
network_interfaces['tag_set'] = boto3_tag_list_to_ansible_dict(network_interfaces['tag_set'])
# Modify boto3 tags list to be ansible friendly dict and then camel_case
camel_network_interfaces = []
for network_interface in network_interfaces_result:
network_interface['TagSet'] = boto3_tag_list_to_ansible_dict(network_interface['TagSet'])
camel_network_interfaces.append(camel_dict_to_snake_dict(network_interface))
module.exit_json(**snaked_network_interfaces_result)
module.exit_json(network_interfaces=camel_network_interfaces)
def get_eni_info(interface):
@ -168,7 +169,7 @@ def main():
else:
module.fail_json(msg="region must be specified")
list_ec2_snapshots_boto3(connection, module)
list_ec2_eni_boto3(connection, module)
else:
region, ec2_url, aws_connect_params = get_aws_connection_info(module)

View file

@ -153,16 +153,12 @@ def main():
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc())
# Turn the boto3 result in to ansible_friendly_snaked_names
# Modify boto3 tags list to be ansible friendly dict and then camel_case
snaked_security_groups = []
for security_group in security_groups['SecurityGroups']:
security_group['Tags'] = boto3_tag_list_to_ansible_dict(security_group['Tags'])
snaked_security_groups.append(camel_dict_to_snake_dict(security_group))
# Turn the boto3 result in to ansible friendly tag dictionary
for security_group in snaked_security_groups:
if 'tags' in security_group:
security_group['tags'] = boto3_tag_list_to_ansible_dict(security_group['tags'])
module.exit_json(security_groups=snaked_security_groups)

View file

@ -210,7 +210,7 @@ def list_ec2_snapshots(connection, module):
# Turn the boto3 result in to ansible friendly tag dictionary
for snapshot in snaked_snapshots:
if 'tags' in snapshot:
snapshot['tags'] = boto3_tag_list_to_ansible_dict(snapshot['tags'])
snapshot['tags'] = boto3_tag_list_to_ansible_dict(snapshot['tags'], 'key', 'value')
module.exit_json(snapshots=snaked_snapshots)

View file

@ -132,7 +132,7 @@ def list_ec2_vpc_nacls(connection, module):
# Turn the boto3 result in to ansible friendly tag dictionary
for nacl in snaked_nacls:
if 'tags' in nacl:
nacl['tags'] = boto3_tag_list_to_ansible_dict(nacl['tags'])
nacl['tags'] = boto3_tag_list_to_ansible_dict(nacl['tags'], 'key', 'value')
if 'entries' in nacl:
nacl['egress'] = [nacl_entry_to_list(e) for e in nacl['entries']
if e['rule_number'] != 32767 and e['egress']]