From d804ac6f4ba78a033b8addee102222a308e7fbe9 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 21 Sep 2017 18:06:56 +0200 Subject: [PATCH] Implement AND'd filters in ec2.py/ini (#30272) * Implement AND'd filters in ec2.py/ini remove debug print * Adjusting code to changed filters' data structure --- contrib/inventory/ec2.ini | 11 ++++- contrib/inventory/ec2.py | 101 +++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/contrib/inventory/ec2.ini b/contrib/inventory/ec2.ini index ab1ad6222dc..8637d0ff59f 100644 --- a/contrib/inventory/ec2.ini +++ b/contrib/inventory/ec2.ini @@ -159,7 +159,9 @@ group_by_elasticache_replication_group = True # inventory. For the full list of possible filters, please read the EC2 API # docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters # Filters are key/value pairs separated by '=', to list multiple filters use -# a list separated by commas. See examples below. +# a list separated by commas. To "AND" criteria together, use "&". Note that +# the "AND" is not useful along with stack_filters and so such usage is not allowed. +# See examples below. # If you want to apply multiple filters simultaneously, set stack_filters to # True. Default behaviour is to combine the results of all filters. Stacking @@ -181,6 +183,13 @@ stack_filters = False # (ex. webservers15, webservers1a, webservers123 etc) # instance_filters = tag:Name=webservers1* +# Retrieve only instances of type t1.micro that also have tag env=stage +# instance_filters = instance-type=t1.micro&tag:env=stage + +# Retrieve instances of type t1.micro AND tag env=stage, as well as any instance +# that are of type m3.large, regardless of env tag +# instance_filters = instance-type=t1.micro&tag:env=stage,instance-type=m3.large + # An IAM role can be assumed, so all requests are run as that role. # This can be useful for connecting across different accounts, or to limit user # access diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index f8bccac2f6e..b0ce35c9c76 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -488,19 +488,27 @@ class Ec2Inventory(object): self.stack_filters = False # Instance filters (see boto and EC2 API docs). Ignore invalid filters. - self.ec2_instance_filters = defaultdict(list) + self.ec2_instance_filters = [] if config.has_option('ec2', 'instance_filters'): + filters = config.get('ec2', 'instance_filters') - filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f] + if self.stack_filters and '&' in filters: + self.fail_with_error("AND filters along with stack_filter enabled is not supported.\n") - for instance_filter in filters: - instance_filter = instance_filter.strip() - if not instance_filter or '=' not in instance_filter: - continue - filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] - if not filter_key: - continue - self.ec2_instance_filters[filter_key].append(filter_value) + filter_sets = [f for f in filters.split(',') if f] + + for filter_set in filter_sets: + filters = {} + filter_set = filter_set.strip() + for instance_filter in filter_set.split("&"): + instance_filter = instance_filter.strip() + if not instance_filter or '=' not in instance_filter: + continue + filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] + if not filter_key: + continue + filters[filter_key] = filter_value + self.ec2_instance_filters.append(filters.copy()) def parse_cli_args(self): ''' Command line argument processing ''' @@ -582,12 +590,12 @@ class Ec2Inventory(object): if self.ec2_instance_filters: if self.stack_filters: filters_dict = {} - for filter_key, filter_values in self.ec2_instance_filters.items(): - filters_dict[filter_key] = filter_values + for filters in self.ec2_instance_filters: + filters_dict.update(filters) reservations.extend(conn.get_all_instances(filters=filters_dict)) else: - for filter_key, filter_values in self.ec2_instance_filters.items(): - reservations.extend(conn.get_all_instances(filters={filter_key: filter_values})) + for filters in self.ec2_instance_filters: + reservations.extend(conn.get_all_instances(filters=filters)) else: reservations = conn.get_all_instances() @@ -627,31 +635,28 @@ class Ec2Inventory(object): ''' return True if given tags match configured filters ''' if not self.ec2_instance_filters: return True - match = self.stack_filters - for filter_name, filter_value in self.ec2_instance_filters.items(): - if filter_name[:4] != 'tag:': - continue - filter_name = filter_name[4:] - if filter_name not in tags: - if self.stack_filters: - match = False - break - continue - if isinstance(filter_value, list): - if self.stack_filters and tags[filter_name] not in filter_value: - match = False - break - if not self.stack_filters and tags[filter_name] in filter_value: - match = True - break - if isinstance(filter_value, six.string_types): - if self.stack_filters and tags[filter_name] != filter_value: - match = False - break - if not self.stack_filters and tags[filter_name] == filter_value: - match = True - break - return match + + for filters in self.ec2_instance_filters: + for filter_name, filter_value in filters.items(): + if filter_name[:4] != 'tag:': + continue + filter_name = filter_name[4:] + if filter_name not in tags: + if self.stack_filters: + return False + continue + if isinstance(filter_value, list): + if self.stack_filters and tags[filter_name] not in filter_value: + return False + if not self.stack_filters and tags[filter_name] in filter_value: + return True + if isinstance(filter_value, six.string_types): + if self.stack_filters and tags[filter_name] != filter_value: + return False + if not self.stack_filters and tags[filter_name] == filter_value: + return True + + return self.stack_filters def get_rds_instances_by_region(self, region): ''' Makes an AWS API call to the list of RDS instances in a particular @@ -718,7 +723,7 @@ class Ec2Inventory(object): if 'LatestRestorableTime' in c: del c['LatestRestorableTime'] - if self.ec2_instance_filters == {}: + if not self.ec2_instance_filters: matches_filter = True else: matches_filter = False @@ -730,14 +735,18 @@ class Ec2Inventory(object): c['Tags'] = tags['TagList'] if self.ec2_instance_filters: - for filter_key, filter_values in self.ec2_instance_filters.items(): - # get AWS tag key e.g. tag:env will be 'env' - tag_name = filter_key.split(":", 1)[1] - # Filter values is a list (if you put multiple values for the same tag name) - matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) + for filters in self.ec2_instance_filters: + for filter_key, filter_values in filters.items(): + # get AWS tag key e.g. tag:env will be 'env' + tag_name = filter_key.split(":", 1)[1] + # Filter values is a list (if you put multiple values for the same tag name) + matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) + + if matches_filter: + # it matches a filter, so stop looking for further matches + break if matches_filter: - # it matches a filter, so stop looking for further matches break except Exception as e: