Backport aws ec2 missing region discovery (#51626)

* aws_ec2 Implement the missing 'region discovery' (#51333)

* aws_ec2 Implement the missing 'region discovery'

  fixes #45288

  tries to use api as documented (which seems to fail in latest boto3 versions)
  and fallback to boto3 'hardcoded' list of regions

* fixes and cleanup, add error for worst case scenario

* fix tests, remove more unused code

* add load_name

* acually load the plugin

* set plugin as required

* reverted test changes, removed options tests

* fixes as per feedback and cleanup

* Allow default regions list to use flexible credential types
This commit is contained in:
Elijah DeLee 2019-02-11 14:13:48 -05:00 committed by Toshio Kuratomi
parent bedfa3f3ff
commit fe79534415
3 changed files with 53 additions and 92 deletions

View file

@ -0,0 +1,2 @@
bugfixes:
- Fix aws_ec2 inventory plugin code to automatically populate regions when missing as documentation states, also leverage config system vs self default/type validation

View file

@ -48,18 +48,27 @@ DOCUMENTATION = '''
- name: AWS_SESSION_TOKEN - name: AWS_SESSION_TOKEN
- name: EC2_SECURITY_TOKEN - name: EC2_SECURITY_TOKEN
regions: regions:
description: A list of regions in which to describe EC2 instances. By default this is all regions except us-gov-west-1 description:
and cn-north-1. - A list of regions in which to describe EC2 instances.
- If empty (the default) default this will include all regions, except possibly restricted ones like us-gov-west-1 and cn-north-1.
type: list
default: []
hostnames: hostnames:
description: A list in order of precedence for hostname variables. You can use the options specified in description: A list in order of precedence for hostname variables. You can use the options specified in
U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). To use tags as hostnames U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). To use tags as hostnames
use the syntax tag:Name=Value to use the hostname Name_Value, or tag:Name to use the value of the Name tag. use the syntax tag:Name=Value to use the hostname Name_Value, or tag:Name to use the value of the Name tag.
type: list
default: []
filters: filters:
description: A dictionary of filter value pairs. Available filters are listed here description: A dictionary of filter value pairs. Available filters are listed here
U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options) U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options)
type: dict
default: {}
strict_permissions: strict_permissions:
description: By default if a 403 (Forbidden) is encountered this plugin will fail. You can set strict_permissions to description: By default if a 403 (Forbidden) is encountered this plugin will fail. You can set strict_permissions to
False in the inventory config file which will allow 403 errors to be gracefully skipped. False in the inventory config file which will allow 403 errors to be gracefully skipped.
type: bool
default: True
''' '''
EXAMPLES = ''' EXAMPLES = '''
@ -127,9 +136,8 @@ compose:
ansible_host: private_ip_address ansible_host: private_ip_address
''' '''
from ansible.errors import AnsibleError, AnsibleParserError from ansible.errors import AnsibleError
from ansible.module_utils._text import to_native, to_text from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.six import string_types
from ansible.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict from ansible.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict
from ansible.module_utils.ec2 import camel_dict_to_snake_dict from ansible.module_utils.ec2 import camel_dict_to_snake_dict
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name
@ -311,6 +319,19 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
return boto_params return boto_params
def _get_connection(self, credentials, region='us-east-1'):
try:
connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region, **credentials)
except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e:
if self.boto_profile:
try:
connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region)
except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e:
raise AnsibleError("Insufficient credentials found: %s" % to_native(e))
else:
raise AnsibleError("Insufficient credentials found: %s" % to_native(e))
return connection
def _boto3_conn(self, regions): def _boto3_conn(self, regions):
''' '''
:param regions: A list of regions to create a boto3 client :param regions: A list of regions to create a boto3 client
@ -320,17 +341,27 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
credentials = self._get_credentials() credentials = self._get_credentials()
for region in regions: if not regions:
try: try:
connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region, **credentials) # as per https://boto3.amazonaws.com/v1/documentation/api/latest/guide/ec2-example-regions-avail-zones.html
except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e: client = self._get_connection(credentials)
if self.boto_profile: resp = client.describe_regions()
try: regions = [x['RegionName'] for x in resp.get('Regions', [])]
connection = boto3.session.Session(profile_name=self.boto_profile).client('ec2', region) except botocore.exceptions.NoRegionError:
except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError) as e: # above seems to fail depending on boto3 version, ignore and lets try something else
raise AnsibleError("Insufficient credentials found: %s" % to_native(e)) pass
else:
raise AnsibleError("Insufficient credentials found: %s" % to_native(e)) # fallback to local list hardcoded in boto3 if still no regions
if not regions:
session = boto3.Session()
regions = session.get_available_regions('ec2')
# I give up, now you MUST give me regions
if not regions:
raise AnsibleError('Unable to get regions list from available methods, you must specify the "regions" option to continue.')
for region in regions:
connection = self._get_connection(credentials, region)
yield connection, region yield connection, region
def _get_instances_by_region(self, regions, filters, strict_permissions): def _get_instances_by_region(self, regions, filters, strict_permissions):
@ -489,49 +520,6 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
display.debug("aws_ec2 inventory filename must end with 'aws_ec2.yml' or 'aws_ec2.yaml'") display.debug("aws_ec2 inventory filename must end with 'aws_ec2.yml' or 'aws_ec2.yaml'")
return False return False
def _get_query_options(self, config_data):
'''
:param config_data: contents of the inventory config file
:return A list of regions to query,
a list of boto3 filter dicts,
a list of possible hostnames in order of preference
a boolean to indicate whether to fail on permission errors
'''
options = {'regions': {'type_to_be': list, 'value': config_data.get('regions', [])},
'filters': {'type_to_be': dict, 'value': config_data.get('filters', {})},
'hostnames': {'type_to_be': list, 'value': config_data.get('hostnames', [])},
'strict_permissions': {'type_to_be': bool, 'value': config_data.get('strict_permissions', True)}}
# validate the options
for name in options:
options[name]['value'] = self._validate_option(name, options[name]['type_to_be'], options[name]['value'])
regions = options['regions']['value']
filters = ansible_dict_to_boto3_filter_list(options['filters']['value'])
hostnames = options['hostnames']['value']
strict_permissions = options['strict_permissions']['value']
return regions, filters, hostnames, strict_permissions
def _validate_option(self, name, desired_type, option_value):
'''
:param name: the option name
:param desired_type: the class the option needs to be
:param option: the value the user has provided
:return The option of the correct class
'''
if isinstance(option_value, string_types) and desired_type == list:
option_value = [option_value]
if option_value is None:
option_value = desired_type()
if not isinstance(option_value, desired_type):
raise AnsibleParserError("The option %s (%s) must be a %s" % (name, option_value, desired_type))
return option_value
def parse(self, inventory, loader, path, cache=True): def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path) super(InventoryModule, self).parse(inventory, loader, path)
@ -539,7 +527,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
self._set_credentials() self._set_credentials()
# get user specifications # get user specifications
regions, filters, hostnames, strict_permissions = self._get_query_options(config_data) regions = self.get_option('regions')
filters = ansible_dict_to_boto3_filter_list(self.get_option('filters'))
hostnames = self.get_option('hostnames')
strict_permissions = self.get_option('strict_permissions')
cache_key = self.get_cache_key(path) cache_key = self.get_cache_key(path)
# false when refresh_cache or --flush-cache is used # false when refresh_cache or --flush-cache is used

View file

@ -28,9 +28,8 @@ import datetime
boto3 = pytest.importorskip('boto3') boto3 = pytest.importorskip('boto3')
botocore = pytest.importorskip('botocore') botocore = pytest.importorskip('botocore')
from ansible.errors import AnsibleError, AnsibleParserError from ansible.errors import AnsibleError
from ansible.plugins.inventory.aws_ec2 import InventoryModule from ansible.plugins.inventory.aws_ec2 import InventoryModule, instance_data_filter_to_boto_attr
from ansible.plugins.inventory.aws_ec2 import instance_data_filter_to_boto_attr
instances = { instances = {
u'Instances': [ u'Instances': [
@ -176,36 +175,5 @@ def test_insufficient_credentials(inventory):
assert "Insufficient boto credentials found" in error_message assert "Insufficient boto credentials found" in error_message
def test_validate_option(inventory):
assert ['us-east-1'] == inventory._validate_option('regions', list, 'us-east-1')
assert ['us-east-1'] == inventory._validate_option('regions', list, ['us-east-1'])
def test_illegal_option(inventory):
bad_filters = [{'tag:Environment': 'dev'}]
with pytest.raises(AnsibleParserError) as error_message:
inventory._validate_option('filters', dict, bad_filters)
assert "The option filters ([{'tag:Environment': 'dev'}]) must be a <class 'dict'>" == error_message
def test_empty_config_query_options(inventory):
regions, filters, hostnames, strict_permissions = inventory._get_query_options({})
assert regions == filters == hostnames == []
assert strict_permissions is True
def test_conig_query_options(inventory):
regions, filters, hostnames, strict_permissions = inventory._get_query_options(
{'regions': ['us-east-1', 'us-east-2'],
'filters': {'tag:Environment': ['dev', 'prod']},
'hostnames': 'ip-address',
'strict_permissions': False}
)
assert regions == ['us-east-1', 'us-east-2']
assert filters == [{'Name': 'tag:Environment', 'Values': ['dev', 'prod']}]
assert hostnames == ['ip-address']
assert strict_permissions is False
def test_verify_file_bad_config(inventory): def test_verify_file_bad_config(inventory):
assert inventory.verify_file('not_aws_config.yml') is False assert inventory.verify_file('not_aws_config.yml') is False