From ddf2a736400a5ff766b786273e8460d4c267a12e Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Wed, 6 Jul 2016 10:28:54 -0400 Subject: [PATCH 1/5] Make it possible to use boto3_conn outside modules The `boto3_conn` function requires a module argument, and calls `module.fail_json` if the connection doesn't receive enough arguments. In non-module settings like inventory scripts, there is no module to be passed. The `boto3_inventory_conn` function takes the same arguments except for `module`, and both call _boto3_conn which doesn't require a module be passed. --- lib/ansible/module_utils/ec2.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index 494d118f64c..aabf78414fd 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -55,10 +55,19 @@ class AnsibleAWSError(Exception): def boto3_conn(module, conn_type=None, resource=None, region=None, endpoint=None, **params): + try: + return _boto3_conn(conn_type=None, resource=None, region=None, endpoint=None, **params) + except ValueError: + module.fail_json(msg='There is an issue in the code of the module. You must specify either both, resource or client to the conn_type parameter in the boto3_conn function call') + +def _boto3_conn(conn_type=None, resource=None, region=None, endpoint=None, **params): profile = params.pop('profile_name', None) if conn_type not in ['both', 'resource', 'client']: - module.fail_json(msg='There is an issue in the code of the module. You must specify either both, resource or client to the conn_type parameter in the boto3_conn function call') + raise ValueError('There is an issue in the calling code. You ' + 'must specify either both, resource, or client to ' + 'the conn_type parameter in the boto3_conn function ' + 'call') if conn_type == 'resource': resource = boto3.session.Session(profile_name=profile).resource(resource, region_name=region, endpoint_url=endpoint, **params) @@ -71,6 +80,7 @@ def boto3_conn(module, conn_type=None, resource=None, region=None, endpoint=None client = boto3.session.Session(profile_name=profile).client(resource, region_name=region, endpoint_url=endpoint, **params) return client, resource +boto3_inventory_conn = _boto3_conn def aws_common_argument_spec(): return dict( From bb5a1f7440a6e56ba8bf833d5d1d25eedb4dbd2e Mon Sep 17 00:00:00 2001 From: Tom Paine Date: Thu, 9 Jun 2016 13:54:50 +0100 Subject: [PATCH 2/5] Add RDS cluster info to EC2 dynamic inventory Add db_clusters to the ec2 inventory. Show tags. Only show clusters matching tags in the `.ini`. Set `include_rds_clusters = True` option to enable RDS cluster inventory collection. Example inventory output: ``` { "db_clusters": { "ryansb-cluster-test": { "AllocatedStorage": 1, "AvailabilityZones": [ "us-west-2a", "us-west-2b", "us-west-2c" ], "BackupRetentionPeriod": 1, "DBClusterIdentifier": "ryansb-cluster-test", "DBClusterMembers": [ { "DBClusterParameterGroupStatus": "in-sync", "DBInstanceIdentifier": "ryansb-test", "IsClusterWriter": true, "PromotionTier": 1 }, { "DBClusterParameterGroupStatus": "in-sync", "DBInstanceIdentifier": "ryansb-test-us-west-2b", "IsClusterWriter": false, "PromotionTier": 1 } ], "DBClusterParameterGroup": "default.aurora5.6", "DBSubnetGroup": "default", "DatabaseName": "mydb", "DbClusterResourceId": "cluster-OB6H7JQURFKFD4BYNHG5HSRLBA", "Endpoint": "ryansb-cluster-test.cluster-c9ntgaejgqln.us-west-2.rds.amazonaws.com", "Engine": "aurora", "EngineVersion": "5.6.10a", "MasterUsername": "admin", "Port": 3306, "PreferredBackupWindow": "06:09-06:39", "PreferredMaintenanceWindow": "mon:11:22-mon:11:52", "ReadReplicaIdentifiers": [], "Status": "available", "StorageEncrypted": false, "VpcSecurityGroups": [ { "Status": "active", "VpcSecurityGroupId": "sg-47eaea20" } ] } }, "rds": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "rds_aurora": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "rds_parameter_group_default_aurora5_6": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "ryansb-test": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "ryansb-test-us-west-2b": [ "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "type_db_r3_large": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "us-west-2": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "us-west-2a": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "us-west-2b": [ "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ], "vpc_id_vpc_3ca34459": [ "ryansb_test_c9ntgaejgqln_us_west_2_rds_amazonaws_com", "ryansb_test_us_west_2b_c9ntgaejgqln_us_west_2_rds_amazonaws_com" ] } ``` --- contrib/inventory/ec2.ini | 3 ++ contrib/inventory/ec2.py | 58 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/contrib/inventory/ec2.ini b/contrib/inventory/ec2.ini index de6d2d912bf..984251cb2d2 100644 --- a/contrib/inventory/ec2.ini +++ b/contrib/inventory/ec2.ini @@ -82,6 +82,9 @@ all_instances = False # 'all_rds_instances' to True return all RDS instances regardless of state. all_rds_instances = False +# Include RDS cluster information (Aurora etc.) +include_rds_clusters = True + # By default, only ElastiCache clusters and nodes in the 'available' state # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' # to True return all ElastiCache clusters and nodes, regardless of state. diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index 9c565fdf798..c0cddca088f 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -130,6 +130,7 @@ from boto import rds from boto import elasticache from boto import route53 import six +import boto3 from six.moves import configparser from collections import defaultdict @@ -265,6 +266,12 @@ class Ec2Inventory(object): if config.has_option('ec2', 'rds'): self.rds_enabled = config.getboolean('ec2', 'rds') + # Include RDS cluster instances? + if config.has_option('ec2', 'include_rds_clusters'): + self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters') + else: + self.include_rds_clusters = False + # Include ElastiCache instances? self.elasticache_enabled = True if config.has_option('ec2', 'elasticache'): @@ -474,6 +481,8 @@ class Ec2Inventory(object): if self.elasticache_enabled: self.get_elasticache_clusters_by_region(region) self.get_elasticache_replication_groups_by_region(region) + if self.include_rds_clusters: + self.include_rds_clusters_by_region(region) self.write_to_cache(self.inventory, self.cache_path_cache) self.write_to_cache(self.index, self.cache_path_index) @@ -574,6 +583,55 @@ class Ec2Inventory(object): error = "Looks like AWS RDS is down:\n%s" % e.message self.fail_with_error(error, 'getting RDS instances') + def include_rds_clusters_by_region(self, region): + client = boto3.client('rds', region_name=region) + clusters = client.describe_db_clusters()["DBClusters"] + account_id = boto.connect_iam().get_user().arn.split(':')[4] + c_dict = {} + for c in clusters: + # remove these datetime objects as there is no serialisation to json + # currently in place and we don't need the data yet + if 'EarliestRestorableTime' in c: + del c['EarliestRestorableTime'] + if 'LatestRestorableTime' in c: + del c['LatestRestorableTime'] + + if self.ec2_instance_filters == {}: + matches_filter = True + else: + matches_filter = False + + try: + # arn:aws:rds:::: + tags = client.list_tags_for_resource( + ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier']) + c['Tags'] = tags['TagList'] + + if self.ec2_instance_filters: + for filter_key, filter_values in self.ec2_instance_filters.items(): + # get AWS tag key e.g. tag:env will be 'env' + tag_name = filter_key.split(":", 1)[1] + # Filter values is a list (if you put multiple values for the same tag name) + matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) + + if matches_filter: + # it matches a filter, so stop looking for further matches + break + + except Exception as e: + if e.message.find('DBInstanceNotFound') >= 0: + # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster. + # Ignore errors when trying to find tags for these + pass + + # ignore empty clusters caused by AWS bug + if len(c['DBClusterMembers']) == 0: + continue + elif matches_filter: + c_dict[c['DBClusterIdentifier']] = c + + self.inventory['db_clusters'] = c_dict + def get_elasticache_clusters_by_region(self, region): ''' Makes an AWS API call to the list of ElastiCache clusters (with nodes' info) in a particular region.''' From 418f91d0e25c7caea96f801b533af19f05ad3508 Mon Sep 17 00:00:00 2001 From: Tom Paine Date: Sun, 19 Jun 2016 15:55:42 +0100 Subject: [PATCH 3/5] Fail softly when boto3 is not installed Updated as per @ryansb comments. The EC2 inventory script will now fail with a useful message when boto3 is not installed and the user is trying to read RDS cluster information. --- contrib/inventory/ec2.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index c0cddca088f..f2622ef6f14 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -130,7 +130,13 @@ from boto import rds from boto import elasticache from boto import route53 import six -import boto3 + +HAS_BOTO3 = False +try: + import boto3 + HAS_BOTO3 = True +except ImportError: + pass from six.moves import configparser from collections import defaultdict @@ -584,7 +590,10 @@ class Ec2Inventory(object): self.fail_with_error(error, 'getting RDS instances') def include_rds_clusters_by_region(self, region): - client = boto3.client('rds', region_name=region) + if not HAS_BOTO3: + module.fail_json(message="This module requires boto3 be installed - please install boto3 and try again") + + client = self.connect_to_aws(rds, region) clusters = client.describe_db_clusters()["DBClusters"] account_id = boto.connect_iam().get_user().arn.split(':')[4] c_dict = {} From 59e499f8f00e4068b68c147b8f3002535ad706e0 Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Wed, 6 Jul 2016 10:38:18 -0400 Subject: [PATCH 4/5] Respect profiles & credentials for boto3 inventory Using boto3 directly wasn't properly using profiles set in the `ec2.ini` file, this change uses the `module_utils` boto3_conn instead. --- contrib/inventory/ec2.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index f2622ef6f14..a870b8fcfed 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -131,6 +131,8 @@ from boto import elasticache from boto import route53 import six +from ansible.module_utils import ec2 as ec2_utils + HAS_BOTO3 = False try: import boto3 @@ -591,9 +593,10 @@ class Ec2Inventory(object): def include_rds_clusters_by_region(self, region): if not HAS_BOTO3: - module.fail_json(message="This module requires boto3 be installed - please install boto3 and try again") - - client = self.connect_to_aws(rds, region) + self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again", + "getting RDS clusters") + + client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) clusters = client.describe_db_clusters()["DBClusters"] account_id = boto.connect_iam().get_user().arn.split(':')[4] c_dict = {} From 0783c172d781acc0ce38fe1afa662ea126dabd90 Mon Sep 17 00:00:00 2001 From: "Ryan S. Brown" Date: Mon, 15 Aug 2016 14:27:48 -0400 Subject: [PATCH 5/5] Paginate DB cluster responses in AWS RDS dynamic inventory --- contrib/inventory/ec2.ini | 2 +- contrib/inventory/ec2.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/inventory/ec2.ini b/contrib/inventory/ec2.ini index 984251cb2d2..9ae2ecea444 100644 --- a/contrib/inventory/ec2.ini +++ b/contrib/inventory/ec2.ini @@ -83,7 +83,7 @@ all_instances = False all_rds_instances = False # Include RDS cluster information (Aurora etc.) -include_rds_clusters = True +include_rds_clusters = False # By default, only ElastiCache clusters and nodes in the 'available' state # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py index a870b8fcfed..77e8128d035 100755 --- a/contrib/inventory/ec2.py +++ b/contrib/inventory/ec2.py @@ -597,7 +597,13 @@ class Ec2Inventory(object): "getting RDS clusters") client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) - clusters = client.describe_db_clusters()["DBClusters"] + + marker, clusters = '', [] + while marker is not None: + resp = client.describe_db_clusters(Marker=marker) + clusters.extend(resp["DBClusters"]) + marker = resp.get('Marker', None) + account_id = boto.connect_iam().get_user().arn.split(':')[4] c_dict = {} for c in clusters: