From f6a0f9874ddeb4b79012272eb5a4a92c0da76756 Mon Sep 17 00:00:00 2001 From: Strahinja Kustudic Date: Wed, 12 Jun 2019 22:38:48 +0200 Subject: [PATCH] Improve speed of the gpc_compute dynamic inventory (#57591) To get all instances gcp_compute made a call to the Google API for each zone separately. Because of this if all zones needed to be queried fetching hosts lasted 30+ seconds. Now the module will use a single query that will return all the instances, so the execution should last just a few seconds. This commit also suppresses a warning from the google-auth library about using user credentials because if an Ansible user wants to use user credentials, there is no need to warn him about it. --- ...peed-up-gcp-compute-dynamic-inventory.yaml | 3 ++ lib/ansible/module_utils/gcp_utils.py | 7 +++- lib/ansible/plugins/inventory/gcp_compute.py | 39 +++++++------------ 3 files changed, 23 insertions(+), 26 deletions(-) create mode 100644 changelogs/fragments/57591-speed-up-gcp-compute-dynamic-inventory.yaml diff --git a/changelogs/fragments/57591-speed-up-gcp-compute-dynamic-inventory.yaml b/changelogs/fragments/57591-speed-up-gcp-compute-dynamic-inventory.yaml new file mode 100644 index 00000000000..be24a6010de --- /dev/null +++ b/changelogs/fragments/57591-speed-up-gcp-compute-dynamic-inventory.yaml @@ -0,0 +1,3 @@ +--- +bugfixes: + - "gcp_compute - Speed up dynamic invetory up to 30x." diff --git a/lib/ansible/module_utils/gcp_utils.py b/lib/ansible/module_utils/gcp_utils.py index ae4f88582b7..b7a897b58b6 100644 --- a/lib/ansible/module_utils/gcp_utils.py +++ b/lib/ansible/module_utils/gcp_utils.py @@ -76,7 +76,12 @@ class GcpSession(object): def get(self, url, body=None, **kwargs): kwargs.update({'json': body, 'headers': self._headers()}) try: - return self.session().get(url, **kwargs) + # Ignore the google-auth library warning for user credentials. More + # details: https://github.com/googleapis/google-auth-library-python/issues/271 + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials") + return self.session().get(url, **kwargs) except getattr(requests.exceptions, 'RequestException') as inst: self.module.fail_json(msg=inst.message) diff --git a/lib/ansible/plugins/inventory/gcp_compute.py b/lib/ansible/plugins/inventory/gcp_compute.py index 84fc69d36ad..9250c2914c8 100644 --- a/lib/ansible/plugins/inventory/gcp_compute.py +++ b/lib/ansible/plugins/inventory/gcp_compute.py @@ -33,7 +33,7 @@ DOCUMENTATION = ''' filters: description: > A list of filter value pairs. Available filters are listed here - U(https://cloud.google.com/compute/docs/reference/rest/v1/instances/list). + U(https://cloud.google.com/compute/docs/reference/rest/v1/instances/aggregatedList). Each additional filter in the list will act be added as an AND condition (filter1 and filter2) type: list @@ -139,7 +139,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): NAME = 'gcp_compute' - _instances = r"https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances" + _instances = r"https://www.googleapis.com/compute/v1/projects/%s/aggregated/instances" def __init__(self): super(InventoryModule, self).__init__() @@ -181,18 +181,6 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): response = self.auth_session.get(link, params={'filter': query}) return self._return_if_object(self.fake_module, response) - def _get_zones(self, project, config_data): - ''' - :param config_data: dict of info from inventory file - :return an array of zones that this project has access to - ''' - link = "https://www.googleapis.com/compute/v1/projects/%s/zones" % project - zones = [] - zones_response = self.fetch_list(config_data, link, '') - for item in zones_response['items']: - zones.append(item['name']) - return zones - def _get_query_options(self, filters): ''' :param config_data: contents of the inventory config file @@ -238,7 +226,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): if navigate_hash(result, ['error', 'errors']): module.fail_json(msg=navigate_hash(result, ['error', 'errors'])) - if result['kind'] != 'compute#instanceList' and result['kind'] != 'compute#zoneList': + if result['kind'] != 'compute#instanceAggregatedList' and result['kind'] != 'compute#zoneList': module.fail_json(msg="Incorrect result: {kind}".format(**result)) return result @@ -471,16 +459,17 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): for project in params['projects']: cached_data[project] = {} params['project'] = project - if not params['zones']: - zones = self._get_zones(project, params) - else: - zones = params['zones'] - for zone in zones: - link = self._instances % (project, zone) - params['zone'] = zone - resp = self.fetch_list(params, link, query) - self._add_hosts(resp.get('items'), config_data, project_disks=project_disks) - cached_data[project][zone] = resp.get('items') + zones = params['zones'] + # Fetch all instances + link = self._instances % project + resp = self.fetch_list(params, link, query) + for key, value in resp.get('items').items(): + if 'instances' in value: + # Key is in format: "zones/europe-west1-b" + zone = key[6:] + if not zones or zone in zones: + self._add_hosts(value['instances'], config_data, project_disks=project_disks) + cached_data[project][zone] = value['instances'] if cache_needs_update: self._cache[cache_key] = cached_data