From bb5a1f7440a6e56ba8bf833d5d1d25eedb4dbd2e Mon Sep 17 00:00:00 2001 From: Tom Paine Date: Thu, 9 Jun 2016 13:54:50 +0100 Subject: [PATCH] Add RDS cluster info to EC2 dynamic inventory Add db_clusters to the ec2 inventory. Show tags. Only show clusters matching tags in the `.ini`. Set `include_rds_clusters = True` option to enable RDS cluster inventory collection. Example inventory output: ``` { "db_clusters": { "ryansb-cluster-test": { "AllocatedStorage": 1, "AvailabilityZones": [ "us-west-2a", "us-west-2b", "us-west-2c" ], "BackupRetentionPeriod": 1, "DBClusterIdentifier": "ryansb-cluster-test", "DBClusterMembers": [ { "DBClusterParameterGroupStatus": "in-sync", "DBInstanceIdentifier": "ryansb-test", "IsClusterWriter": true, "PromotionTier": 1 }, { "DBClusterParameterGroupStatus": "in-sync", "DBInstanceIdentifier": "ryansb-test-us-west-2b", "IsClusterWriter": false, "PromotionTier": 1 } ], "DBClusterParameterGroup": "default.aurora5.6", "DBSubnetGroup": "default", "DatabaseName": "mydb", "DbClusterResourceId": "cluster-OB6H7JQURFKFD4BYNHG5HSRLBA", "Endpoint": "ryansb-cluster-test.cluster-c9ntgaejgqln.us-west-2.rds.amazonaws.com", "Engine": "aurora", "EngineVersion": "5.6.10a", "MasterUsername": "admin", "Port": 3306, "PreferredBackupWindow": "06:09-06:39", "PreferredMaintenanceWindow": "mon:11:22-mon:11:52", "ReadReplicaIdentifiers": [], "Status": "available", "StorageEncrypted": false, "VpcSecurityGroups": [ { "Status": "active", "VpcSecurityGroupId": "sg-47eaea20" } ] } }, "rds": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "rds_aurora": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "rds_parameter_group_default_aurora5_6": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "ryansb-test": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "ryansb-test-us-west-2b": [ "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "type_db_r3_large": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "us-west-2": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "us-west-2a": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "us-west-2b": [ "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "vpc_id_vpc_3ca34459": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ] } ``` --- contrib/inventory/ec2.ini | 3 ++ contrib/inventory/ec2.py | 58 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/contrib/inventory/ec2.ini b/contrib/inventory/ec2.ini index de6d2d912bf..984251cb2d2 100644 --- a/contrib/inventory/ec2.ini +++ b/contrib/inventory/ec2.ini @@ -82,6 +82,9 @@ all_instances = False # 'all_rds_instances' to True return all RDS instances regardless of state. all_rds_instances = False +# Include RDS cluster information (Aurora etc.) +include_rds_clusters = True + # By default, only ElastiCache clusters and nodes in the 'available' state # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' # to True return all ElastiCache clusters and nodes, regardless of state. diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index 9c565fdf798..c0cddca088f 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -130,6 +130,7 @@ from boto import rds from boto import elasticache from boto import route53 import six +import boto3 from six.moves import configparser from collections import defaultdict @@ -265,6 +266,12 @@ class Ec2Inventory(object): if config.has_option('ec2', 'rds'): self.rds_enabled = config.getboolean('ec2', 'rds') + # Include RDS cluster instances? + if config.has_option('ec2', 'include_rds_clusters'): + self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters') + else: + self.include_rds_clusters = False + # Include ElastiCache instances? self.elasticache_enabled = True if config.has_option('ec2', 'elasticache'): @@ -474,6 +481,8 @@ class Ec2Inventory(object): if self.elasticache_enabled: self.get_elasticache_clusters_by_region(region) self.get_elasticache_replication_groups_by_region(region) + if self.include_rds_clusters: + self.include_rds_clusters_by_region(region) self.write_to_cache(self.inventory, self.cache_path_cache) self.write_to_cache(self.index, self.cache_path_index) @@ -574,6 +583,55 @@ class Ec2Inventory(object): error = "Looks like AWS RDS is down:\n%s" % e.message self.fail_with_error(error, 'getting RDS instances') + def include_rds_clusters_by_region(self, region): + client = boto3.client('rds', region_name=region) + clusters = client.describe_db_clusters()["DBClusters"] + account_id = boto.connect_iam().get_user().arn.split(':')[4] + c_dict = {} + for c in clusters: + # remove these datetime objects as there is no serialisation to json + # currently in place and we don't need the data yet + if 'EarliestRestorableTime' in c: + del c['EarliestRestorableTime'] + if 'LatestRestorableTime' in c: + del c['LatestRestorableTime'] + + if self.ec2_instance_filters == {}: + matches_filter = True + else: + matches_filter = False + + try: + # arn:aws:rds:::: + tags = client.list_tags_for_resource( + ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier']) + c['Tags'] = tags['TagList'] + + if self.ec2_instance_filters: + for filter_key, filter_values in self.ec2_instance_filters.items(): + # get AWS tag key e.g. tag:env will be 'env' + tag_name = filter_key.split(":", 1)[1] + # Filter values is a list (if you put multiple values for the same tag name) + matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) + + if matches_filter: + # it matches a filter, so stop looking for further matches + break + + except Exception as e: + if e.message.find('DBInstanceNotFound') >= 0: + # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster. + # Ignore errors when trying to find tags for these + pass + + # ignore empty clusters caused by AWS bug + if len(c['DBClusterMembers']) == 0: + continue + elif matches_filter: + c_dict[c['DBClusterIdentifier']] = c + + self.inventory['db_clusters'] = c_dict + def get_elasticache_clusters_by_region(self, region): ''' Makes an AWS API call to the list of ElastiCache clusters (with nodes' info) in a particular region.'''