Make hostvars more dynamic again to improve performance with large inventories

Fixes #12477

lib/ansible/inventory/group.py

@@ -66,6 +66,7 @@ class Group:
         self.__init__()
         self.name = data.get('name')
         self.vars = data.get('vars', dict())
+        self.depth = data.get('depth', 0)
 
         parent_groups = data.get('parent_groups', [])
         for parent_data in parent_groups:

lib/ansible/vars/__init__.py

@@ -43,7 +43,8 @@ from ansible.utils.vars import combine_vars
 from ansible.vars.hostvars import HostVars
 from ansible.vars.unsafe_proxy import UnsafeProxy
 
-CACHED_VARS = dict()
+VARIABLE_CACHE = dict()
+HOSTVARS_CACHE = dict()
 
 try:
     from __main__ import display
@@ -85,6 +86,28 @@ class VariableManager:
         self._inventory = None
         self._omit_token = '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest()
 
+    def __getstate__(self):
+        data = dict(
+            fact_cache = self._fact_cache.copy(),
+            np_fact_cache = self._nonpersistent_fact_cache.copy(),
+            vars_cache = self._vars_cache.copy(),
+            extra_vars = self._extra_vars.copy(),
+            host_vars_files = self._host_vars_files.copy(),
+            group_vars_files = self._group_vars_files.copy(),
+            omit_token = self._omit_token,
+        )
+        return data
+
+    def __setstate__(self, data):
+        self._fact_cache = data.get('fact_cache', defaultdict(dict))
+        self._nonpersistent_fact_cache = data.get('np_fact_cache', defaultdict(dict))
+        self._vars_cache = data.get('vars_cache', defaultdict(dict))
+        self._extra_vars = data.get('extra_vars', dict())
+        self._host_vars_files = data.get('host_vars_files', defaultdict(dict))
+        self._group_vars_files = data.get('group_vars_files', defaultdict(dict))
+        self._omit_token = data.get('omit_token', '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest())
+        self._inventory = None
+
     def _get_cache_entry(self, play=None, host=None, task=None):
         play_id = "NONE"
         if play:
@@ -157,9 +180,9 @@ class VariableManager:
 
         debug("in VariableManager get_vars()")
         cache_entry = self._get_cache_entry(play=play, host=host, task=task)
-        if cache_entry in CACHED_VARS and use_cache:
+        if cache_entry in VARIABLE_CACHE and use_cache:
             debug("vars are cached, returning them now")
-            return CACHED_VARS[cache_entry]
+            return VARIABLE_CACHE[cache_entry]
 
         all_vars = defaultdict(dict)
 
@@ -289,7 +312,12 @@ class VariableManager:
                     all_vars['groups'][group_name] = [h.name for h in group.get_hosts()]
 
                 if include_hostvars:
-                    hostvars = HostVars(vars_manager=self, play=play, inventory=self._inventory, loader=loader)
+                    hostvars_cache_entry = self._get_cache_entry(play=play)
+                    if hostvars_cache_entry in HOSTVARS_CACHE:
+                        hostvars = HOSTVARS_CACHE[hostvars_cache_entry]
+                    else:
+                        hostvars = HostVars(play=play, inventory=self._inventory, loader=loader, variable_manager=self)
+                        HOSTVARS_CACHE[hostvars_cache_entry] = hostvars
                     all_vars['hostvars'] = hostvars
 
         if task:
@@ -355,7 +383,7 @@ class VariableManager:
         if 'hostvars' in all_vars and host:
             all_vars['vars'] = all_vars['hostvars'][host.get_name()]
 
-        #CACHED_VARS[cache_entry] = all_vars
+        #VARIABLE_CACHE[cache_entry] = all_vars
 
         debug("done with get_vars()")
         return all_vars

lib/ansible/vars/hostvars.py

@@ -34,9 +34,11 @@ __all__ = ['HostVars']
 class HostVars(collections.Mapping):
     ''' A special view of vars_cache that adds values from the inventory when needed. '''
 
-    def __init__(self, vars_manager, play, inventory, loader):
-        self._lookup = {}
+    def __init__(self, play, inventory, variable_manager, loader):
+        self._lookup = dict()
         self._loader = loader
+        self._play = play
+        self._variable_manager = variable_manager
 
         hosts = inventory.get_hosts(ignore_limits_and_restrictions=True)
 
@@ -49,9 +51,6 @@ class HostVars(collections.Mapping):
                 has_localhost = True
                 break
 
-        # we don't use the method in inventory to create the implicit host,
-        # because it also adds it to the 'ungrouped' group, and we want to
-        # avoid any side-effects
         if not has_localhost:
             new_host =  Host(name='localhost')
             new_host.set_variable("ansible_python_interpreter", sys.executable)
@@ -60,14 +59,15 @@ class HostVars(collections.Mapping):
             hosts.append(new_host)
 
         for host in hosts:
-            self._lookup[host.name] = vars_manager.get_vars(loader=loader, play=play, host=host, include_hostvars=False)
+            self._lookup[host.name] = host
 
     def __getitem__(self, host_name):
 
         if host_name not in self._lookup:
             return j2undefined
 
-        data = self._lookup.get(host_name)
+        host = self._lookup.get(host_name)
+        data = self._variable_manager.get_vars(loader=self._loader, host=host, play=self._play, include_hostvars=False)
         templar = Templar(variables=data, loader=self._loader)
         return templar.template(data, fail_on_undefined=False)
 
@@ -84,9 +84,10 @@ class HostVars(collections.Mapping):
         raise NotImplementedError('HostVars does not support len.  hosts entries are discovered dynamically as needed')
 
     def __getstate__(self):
-        data = self._lookup.copy()
-        return dict(loader=self._loader, data=data)
+        return dict(loader=self._loader, lookup=self._lookup, play=self._play, var_manager=self._variable_manager)
 
     def __setstate__(self, data):
-        self._lookup = data.get('data')
+        self._play = data.get('play')
         self._loader = data.get('loader')
+        self._lookup = data.get('lookup')
+        self._variable_manager = data.get('var_manager')