Improve speed of the gpc_compute dynamic inventory (#57591)

To get all instances gcp_compute made a call to the Google API for each
zone separately. Because of this if all zones needed to be queried
fetching hosts lasted 30+ seconds. Now the module will use a single
query that will return all the instances, so the execution should last
just a few seconds.

This commit also suppresses a warning from the google-auth library about
using user credentials because if an Ansible user wants to use user
credentials, there is no need to warn him about it.

changelogs/fragments/57591-speed-up-gcp-compute-dynamic-inventory.yaml

@@ -0,0 +1,3 @@
+---
+bugfixes:
+  - "gcp_compute - Speed up dynamic invetory up to 30x."

lib/ansible/module_utils/gcp_utils.py

@@ -76,7 +76,12 @@ class GcpSession(object):
     def get(self, url, body=None, **kwargs):
         kwargs.update({'json': body, 'headers': self._headers()})
         try:
-            return self.session().get(url, **kwargs)
+            # Ignore the google-auth library warning for user credentials. More
+            # details: https://github.com/googleapis/google-auth-library-python/issues/271
+            import warnings
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials")
+                return self.session().get(url, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 

lib/ansible/plugins/inventory/gcp_compute.py

@@ -33,7 +33,7 @@ DOCUMENTATION = '''
         filters:
           description: >
             A list of filter value pairs. Available filters are listed here
-            U(https://cloud.google.com/compute/docs/reference/rest/v1/instances/list).
+            U(https://cloud.google.com/compute/docs/reference/rest/v1/instances/aggregatedList).
             Each additional filter in the list will act be added as an AND condition
             (filter1 and filter2)
           type: list
@@ -139,7 +139,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
 
     NAME = 'gcp_compute'
 
-    _instances = r"https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances"
+    _instances = r"https://www.googleapis.com/compute/v1/projects/%s/aggregated/instances"
 
     def __init__(self):
         super(InventoryModule, self).__init__()
@@ -181,18 +181,6 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
         response = self.auth_session.get(link, params={'filter': query})
         return self._return_if_object(self.fake_module, response)
 
-    def _get_zones(self, project, config_data):
-        '''
-            :param config_data: dict of info from inventory file
-            :return an array of zones that this project has access to
-        '''
-        link = "https://www.googleapis.com/compute/v1/projects/%s/zones" % project
-        zones = []
-        zones_response = self.fetch_list(config_data, link, '')
-        for item in zones_response['items']:
-            zones.append(item['name'])
-        return zones
-
     def _get_query_options(self, filters):
         '''
             :param config_data: contents of the inventory config file
@@ -238,7 +226,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
 
         if navigate_hash(result, ['error', 'errors']):
             module.fail_json(msg=navigate_hash(result, ['error', 'errors']))
-        if result['kind'] != 'compute#instanceList' and result['kind'] != 'compute#zoneList':
+        if result['kind'] != 'compute#instanceAggregatedList' and result['kind'] != 'compute#zoneList':
             module.fail_json(msg="Incorrect result: {kind}".format(**result))
 
         return result
@@ -471,16 +459,17 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             for project in params['projects']:
                 cached_data[project] = {}
                 params['project'] = project
-                if not params['zones']:
-                    zones = self._get_zones(project, params)
-                else:
-                    zones = params['zones']
-                for zone in zones:
-                    link = self._instances % (project, zone)
-                    params['zone'] = zone
-                    resp = self.fetch_list(params, link, query)
-                    self._add_hosts(resp.get('items'), config_data, project_disks=project_disks)
-                    cached_data[project][zone] = resp.get('items')
+                zones = params['zones']
+                # Fetch all instances
+                link = self._instances % project
+                resp = self.fetch_list(params, link, query)
+                for key, value in resp.get('items').items():
+                    if 'instances' in value:
+                        # Key is in format: "zones/europe-west1-b"
+                        zone = key[6:]
+                        if not zones or zone in zones:
+                            self._add_hosts(value['instances'], config_data, project_disks=project_disks)
+                            cached_data[project][zone] = value['instances']
 
         if cache_needs_update:
             self._cache[cache_key] = cached_data