From 91785ba045831a6aa39733f614ba59d5f37dbb26 Mon Sep 17 00:00:00 2001 From: Herby Gillot Date: Mon, 23 Sep 2013 14:00:56 -0400 Subject: [PATCH 1/3] ec2 inventory: Add the ability to group instances by Route 53 domain names. --- plugins/inventory/ec2.ini | 11 +++++-- plugins/inventory/ec2.py | 65 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/plugins/inventory/ec2.ini b/plugins/inventory/ec2.ini index 703f07081a5..170505fe331 100644 --- a/plugins/inventory/ec2.ini +++ b/plugins/inventory/ec2.ini @@ -34,6 +34,14 @@ destination_variable = public_dns_name # be run from with EC2. vpc_destination_variable = ip_address +# To tag instances on EC2 with the resource records that point to them from +# Route53, uncomment and set 'route53' to True. +# +# Optionally, you can specify the list of zones to exclude looking up in +# 'route53_excluded_zones' as a comma-seperated list. +route53 = False +route53_excluded_zones = + # API calls to EC2 are slow. For this reason, we cache the results of an API # call. Set this to the path you want cache files to be written to. Two files # will be written to this directory: @@ -44,6 +52,3 @@ cache_path = /tmp # The number of seconds a cache file is considered valid. After this many # seconds, a new API call will be made, and the cache file will be updated. cache_max_age = 300 - - - diff --git a/plugins/inventory/ec2.py b/plugins/inventory/ec2.py index 913ddb5f84a..41b87294066 100755 --- a/plugins/inventory/ec2.py +++ b/plugins/inventory/ec2.py @@ -116,6 +116,7 @@ from time import time import boto from boto import ec2 from boto import rds +from boto import route53 import ConfigParser try: @@ -204,6 +205,10 @@ class Ec2Inventory(object): self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') + # Route53 + self.route53_enabled = config.getboolean('ec2', 'route53') + self.route53_excluded_zones = config.get('ec2', 'route53_excluded_zones', '').split(',') + # Cache related cache_path = config.get('ec2', 'cache_path') self.cache_path_cache = cache_path + "/ansible-ec2.cache" @@ -228,6 +233,9 @@ class Ec2Inventory(object): def do_api_calls_update_cache(self): ''' Do API calls to each region, and save data in cache files ''' + if self.route53_enabled: + self.get_route53_records() + for region in self.regions: self.get_instances_by_region(region) self.get_rds_instances_by_region(region) @@ -329,7 +337,7 @@ class Ec2Inventory(object): # Inventory: Group by instance type self.push(self.inventory, self.to_safe('type_' + instance.instance_type), dest) - + # Inventory: Group by key pair if instance.key_name: self.push(self.inventory, self.to_safe('key_' + instance.key_name), dest) @@ -349,6 +357,12 @@ class Ec2Inventory(object): key = self.to_safe("tag_" + k + "=" + v) self.push(self.inventory, key, dest) + # Inventory: Group by Route53 domain names if enabled + if self.route53_enabled: + route53_names = self.get_instance_route53_names(instance) + for name in route53_names: + self.push(self.inventory, name, dest) + def add_rds_instance(self, instance, region): ''' Adds an RDS instance to the inventory and index, as long as it is @@ -401,6 +415,55 @@ class Ec2Inventory(object): self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest) + def get_route53_records(self): + ''' Get and store the map of resource records to domain names that + point to them. ''' + + r53_conn = route53.Route53Connection() + all_zones = r53_conn.get_zones() + + is_valid_zone = lambda zone: not zone.name in self.route53_excluded_zones + + route53_zones = filter(is_valid_zone, all_zones) + + self.route53_records = {} + + for zone in route53_zones: + rrsets = r53_conn.get_all_rrsets(zone.id) + + for record_set in rrsets: + record_name = record_set.name + + if record_name.endswith('.'): + record_name = record_name[:-1] + + for resource in record_set.resource_records: + self.route53_records.setdefault(resource, set()) + self.route53_records[resource].add(record_name) + + + def get_instance_route53_names(self, instance): + ''' Check if an instance is referenced in the records we have from + Route53. If it is, return the list of domain names pointing to said + instance. If nothing points to it, return an empty list. ''' + + instance_attributes = [ 'public_dns_name', 'private_dns_name', + 'ip_address', 'private_ip_address' ] + + name_list = set() + + for attrib in instance_attributes: + try: + value = getattr(instance, attrib) + except AttributeError: + continue + + if value in self.route53_records: + name_list.update(self.route53_records[value]) + + return list(name_list) + + def get_host_info(self): ''' Get variables about a specific host ''' From 6ddebb3fadda0b68c3e357998aff5ba5415a27b0 Mon Sep 17 00:00:00 2001 From: Herby Gillot Date: Mon, 23 Sep 2013 21:40:18 -0400 Subject: [PATCH 2/3] ec2 inventory route53 tagging: * make filtering more comprehensive * add checks against missing route53 zone exclusion option --- plugins/inventory/ec2.ini | 8 ++++---- plugins/inventory/ec2.py | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/inventory/ec2.ini b/plugins/inventory/ec2.ini index 170505fe331..01a4982d624 100644 --- a/plugins/inventory/ec2.ini +++ b/plugins/inventory/ec2.ini @@ -36,11 +36,11 @@ vpc_destination_variable = ip_address # To tag instances on EC2 with the resource records that point to them from # Route53, uncomment and set 'route53' to True. -# -# Optionally, you can specify the list of zones to exclude looking up in -# 'route53_excluded_zones' as a comma-seperated list. route53 = False -route53_excluded_zones = + +# Additionally, you can specify the list of zones to exclude looking up in +# 'route53_excluded_zones' as a comma-seperated list. +# route53_excluded_zones = samplezone1.com, samplezone2.com # API calls to EC2 are slow. For this reason, we cache the results of an API # call. Set this to the path you want cache files to be written to. Two files diff --git a/plugins/inventory/ec2.py b/plugins/inventory/ec2.py index 41b87294066..c533a77b385 100755 --- a/plugins/inventory/ec2.py +++ b/plugins/inventory/ec2.py @@ -207,7 +207,10 @@ class Ec2Inventory(object): # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') - self.route53_excluded_zones = config.get('ec2', 'route53_excluded_zones', '').split(',') + self.route53_excluded_zones = [] + if config.has_option('ec2', 'route53_excluded_zones'): + self.route53_excluded_zones.extend( + config.get('ec2', 'route53_excluded_zones', '').split(',')) # Cache related cache_path = config.get('ec2', 'cache_path') @@ -422,9 +425,8 @@ class Ec2Inventory(object): r53_conn = route53.Route53Connection() all_zones = r53_conn.get_zones() - is_valid_zone = lambda zone: not zone.name in self.route53_excluded_zones - - route53_zones = filter(is_valid_zone, all_zones) + route53_zones = [ zone for zone in all_zones if zone.name[:-1] + not in self.route53_excluded_zones ] self.route53_records = {} From 8b3674bf58bbdd6869b3e92f81bd02f8d4826dee Mon Sep 17 00:00:00 2001 From: Herby Gillot Date: Mon, 23 Sep 2013 21:43:00 -0400 Subject: [PATCH 3/3] minor spacing cleanup --- plugins/inventory/ec2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inventory/ec2.py b/plugins/inventory/ec2.py index c533a77b385..383dad95e03 100755 --- a/plugins/inventory/ec2.py +++ b/plugins/inventory/ec2.py @@ -337,7 +337,7 @@ class Ec2Inventory(object): # Inventory: Group by availability zone self.push(self.inventory, instance.placement, dest) - + # Inventory: Group by instance type self.push(self.inventory, self.to_safe('type_' + instance.instance_type), dest) @@ -425,7 +425,7 @@ class Ec2Inventory(object): r53_conn = route53.Route53Connection() all_zones = r53_conn.get_zones() - route53_zones = [ zone for zone in all_zones if zone.name[:-1] + route53_zones = [ zone for zone in all_zones if zone.name[:-1] not in self.route53_excluded_zones ] self.route53_records = {}