Improve consul_io.py execution speed (#33737)

* Improve execution speed

* fix missing whitespace

contrib/inventory/consul_io.ini

@@ -2,6 +2,12 @@
 
 [consul]
 
+#
+# Bulk load. Load all possible data before building inventory JSON
+# If true, script processes in-memory data. JSON generation reduces drastically
+#
+bulk_load = false
+
 # restrict included nodes to those from this datacenter
 #datacenter = nyc1
 
@@ -13,6 +19,14 @@ url = http://localhost:8500
 # a suffix of '_servers' will add each address to the group name 'redis_servers'
 servers_suffix = _servers
 
+#
+# By default, final JSON is built based on all available info in consul.
+# Suffixes means that services groups will be added in addition to basic infromation. See servers_suffix for additional info
+# There are cases when speed is preferable than having services groups
+# False value will reduce script execution time dragtically.
+#
+suffixes = true
+
 # if specified then the inventory will generate domain names that will resolve
 # via Consul's inbuilt DNS. 
 #domain=consul
@@ -32,6 +46,9 @@ kv_metadata=ansible/metadata
 
 # looks up the health of each service and adds the node to 'up' and 'down' groups
 # based on the service availibility
+#
+# !!!! if availability is true, suffixes also must be true. !!!!
+#
 availability = true
 available_suffix = _up
 unavailable_suffix = _down

contrib/inventory/consul_io.py

@@ -218,6 +218,8 @@ class ConsulInventory(object):
         self.nodes_by_kv = {}
         self.nodes_by_availability = {}
         self.current_dc = None
+        self.inmemory_kv = []
+        self.inmemory_nodes = []
 
         config = ConsulConfig()
         self.config = config
@@ -235,12 +237,21 @@ class ConsulInventory(object):
         self.combine_all_results()
         print(json.dumps(self.inventory, sort_keys=True, indent=2))
 
+    def bulk_load(self, datacenter):
+        index, groups_list = self.consul_api.kv.get(self.config.kv_groups, recurse=True, dc=datacenter)
+        index, metadata_list = self.consul_api.kv.get(self.config.kv_metadata, recurse=True, dc=datacenter)
+        index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
+        self.inmemory_kv += groups_list
+        self.inmemory_kv += metadata_list
+        self.inmemory_nodes += nodes
+
     def load_all_data_consul(self):
         ''' cycle through each of the datacenters in the consul catalog and process
             the nodes in each '''
         self.datacenters = self.consul_api.catalog.datacenters()
         for datacenter in self.datacenters:
             self.current_dc = datacenter
+            self.bulk_load(datacenter)
             self.load_data_for_datacenter(datacenter)
 
     def load_availability_groups(self, node, datacenter):
@@ -262,8 +273,19 @@ class ConsulInventory(object):
                             self.add_node_to_map(self.nodes_by_availability,
                                                  service_name + suffix, node['Node'])
 
+    def consul_get_kv_inmemory(self, key):
+        result = filter(lambda x: x['Key'] == key, self.inmemory_kv)
+        return result.pop() if result else None
+
+    def consul_get_node_inmemory(self, node):
+        result = filter(lambda x: x['Node'] == node, self.inmemory_nodes)
+        return {"Node": result.pop(), "Services": {}} if result else None
+
     def load_data_for_datacenter(self, datacenter):
         '''processes all the nodes in a particular datacenter'''
+        if self.config.bulk_load == 'true':
+            nodes = self.inmemory_nodes
+        else:
             index, nodes = self.consul_api.catalog.nodes(dc=datacenter)
         for node in nodes:
             self.add_node_to_map(self.nodes_by_datacenter, datacenter, node)
@@ -273,16 +295,20 @@ class ConsulInventory(object):
         '''loads the data for a sinle node adding it to various groups based on
         metadata retrieved from the kv store and service availability'''
 
+        if self.config.suffixes == 'true':
             index, node_data = self.consul_api.catalog.node(node, dc=datacenter)
+        else:
+            node_data = self.consul_get_node_inmemory(node)
         node = node_data['Node']
+
         self.add_node_to_map(self.nodes, 'all', node)
         self.add_metadata(node_data, "consul_datacenter", datacenter)
         self.add_metadata(node_data, "consul_nodename", node['Node'])
 
         self.load_groups_from_kv(node_data)
         self.load_node_metadata_from_kv(node_data)
+        if self.config.suffixes == 'true':
             self.load_availability_groups(node_data, datacenter)
-
             for name, service in node_data['Services'].items():
                 self.load_data_from_service(name, service, node_data)
 
@@ -293,6 +319,9 @@ class ConsulInventory(object):
         node = node_data['Node']
         if self.config.has_config('kv_metadata'):
             key = "%s/%s/%s" % (self.config.kv_metadata, self.current_dc, node['Node'])
+            if self.config.bulk_load == 'true':
+                metadata = self.consul_get_kv_inmemory(key)
+            else:
                 index, metadata = self.consul_api.kv.get(key)
             if metadata and metadata['Value']:
                 try:
@@ -309,6 +338,9 @@ class ConsulInventory(object):
         node = node_data['Node']
         if self.config.has_config('kv_groups'):
             key = "%s/%s/%s" % (self.config.kv_groups, self.current_dc, node['Node'])
+            if self.config.bulk_load == 'true':
+                groups = self.consul_get_kv_inmemory(key)
+            else:
                 index, groups = self.consul_api.kv.get(key)
             if groups and groups['Value']:
                 for group in groups['Value'].split(','):
@@ -434,11 +466,11 @@ class ConsulConfig(dict):
         config_options = ['host', 'token', 'datacenter', 'servers_suffix',
                           'tags', 'kv_metadata', 'kv_groups', 'availability',
                           'unavailable_suffix', 'available_suffix', 'url',
-                          'domain']
+                          'domain', 'suffixes', 'bulk_load']
         for option in config_options:
             value = None
             if config.has_option('consul', option):
-                value = config.get('consul', option)
+                value = config.get('consul', option).lower()
             setattr(self, option, value)
 
     def read_cli_args(self):