From c23b11d21206b9409c494401598809672494966e Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 30 Aug 2016 14:27:06 +0200 Subject: [PATCH] [Inventory] Cache the result of enumerating groups and host names for `VariableManager._get_magic_variables()`. This saves a lot of time re-iterating the nearly always constant global list of groups and their members. Generate once and cache, and invalidate cache in case `add_host:` or `group_by:` are used. --- lib/ansible/inventory/__init__.py | 35 +++++++++++++++---- lib/ansible/plugins/strategy/__init__.py | 6 ++++ lib/ansible/vars/__init__.py | 7 ++-- .../plugins/strategies/test_strategy_base.py | 1 + 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index f76702133db..f8a98929f4d 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -25,7 +25,7 @@ import sys import re import itertools -from ansible.compat.six import string_types +from ansible.compat.six import string_types, iteritems from ansible import constants as C from ansible.errors import AnsibleError @@ -63,11 +63,12 @@ class Inventory(object): # caching to avoid repeated calculations, particularly with # external inventory scripts. - self._vars_per_host = {} - self._vars_per_group = {} - self._hosts_cache = {} - self._pattern_cache = {} - self._vars_plugins = [] + self._vars_per_host = {} + self._vars_per_group = {} + self._hosts_cache = {} + self._pattern_cache = {} + self._group_dict_cache = {} + self._vars_plugins = [] self._basedir = self.basedir() @@ -88,6 +89,7 @@ class Inventory(object): # clear the cache here, which is only useful if more than # one Inventory objects are created when using the API directly self.clear_pattern_cache() + self.clear_group_dict_cache() self.parse_inventory(host_list) @@ -220,7 +222,7 @@ class Inventory(object): hosts = [ h for h in hosts if h in subset ] # exclude hosts mentioned in any restriction (ex: failed hosts) - if self._restriction is not None: + if self._restriction: hosts = [ h for h in hosts if h.name in self._restriction ] seen = set() @@ -502,6 +504,10 @@ class Inventory(object): HOSTS_PATTERNS_CACHE = {} self._pattern_cache = {} + def clear_group_dict_cache(self): + ''' called exclusively by the add_host and group_by plugins ''' + self._group_dict_cache = {} + def groups_for_host(self, host): if host in self._hosts_cache: return self._hosts_cache[host].get_groups() @@ -568,6 +574,20 @@ class Inventory(object): return vars + def get_group_dict(self): + """ + In get_vars() we merge a 'magic' dictionary 'groups' with group name + keys and hostname list values into every host variable set. + + Cache the creation of this structure here + """ + + if not self._group_dict_cache: + for (group_name, group) in iteritems(self.groups): + self._group_dict_cache[group_name] = [h.name for h in group.get_hosts()] + + return self._group_dict_cache + def get_vars(self, hostname, update_cached=False, vault_password=None): host = self.get_host(hostname) @@ -829,6 +849,7 @@ class Inventory(object): def refresh_inventory(self): self.clear_pattern_cache() + self.clear_group_dict_cache() self._hosts_cache = {} self._vars_per_host = {} diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index 11cab2727aa..e4b9d1e9716 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -475,6 +475,9 @@ class StrategyBase: # patterns may have referenced the group self._inventory.clear_pattern_cache() + # clear cache of group dict, which is used in magic host variables + self._inventory.clear_group_dict_cache() + # also clear the hostvar cache entry for the given play, so that # the new hosts are available if hostvars are referenced self._variable_manager.invalidate_hostvars_cache(play=iterator._play) @@ -495,6 +498,9 @@ class StrategyBase: group_name = result_item.get('add_group') new_group = self._inventory.get_group(group_name) if not new_group: + # clear cache of group dict, which is used in magic host variables + self._inventory.clear_group_dict_cache() + # create the new group and add it to inventory new_group = Group(name=group_name) self._inventory.add_group(new_group) diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py index 5f714ce1166..2c5a4b4f60a 100644 --- a/lib/ansible/vars/__init__.py +++ b/lib/ansible/vars/__init__.py @@ -393,10 +393,9 @@ class VariableManager: if host: variables['group_names'] = sorted([group.name for group in host.get_groups() if group.name != 'all']) - if self._inventory is not None: - variables['groups'] = dict() - for (group_name, group) in iteritems(self._inventory.groups): - variables['groups'][group_name] = [h.name for h in group.get_hosts()] + if self._inventory: + variables['groups'] = self._inventory.get_group_dict() + if play: variables['role_names'] = [r._role_name for r in play.roles] diff --git a/test/units/plugins/strategies/test_strategy_base.py b/test/units/plugins/strategies/test_strategy_base.py index 231897d0edc..2f7a13f1ccc 100644 --- a/test/units/plugins/strategies/test_strategy_base.py +++ b/test/units/plugins/strategies/test_strategy_base.py @@ -237,6 +237,7 @@ class TestStrategyBase(unittest.TestCase): mock_inventory.get_host.side_effect = _get_host mock_inventory.get_group.side_effect = _get_group mock_inventory.clear_pattern_cache.return_value = None + mock_inventory.clear_group_dict_cache.return_value = None mock_inventory.get_host_vars.return_value = {} mock_var_mgr = MagicMock()