add inventory caching & use in virtualbox inventory plugin (#34510)
* Inventory caching * Add inventory caching for virtualbox * Don't populate cache for virtualbox with stdout, use a dict of inventory instead * Fix error creating the cache dir if it doesn't exist * Keep cache default False and set to True in VariableManager __init__ * Check all groups before determining if a host is ungrouped.
This commit is contained in:
parent
9fdaa86c9f
commit
4a1cc661c4
5 changed files with 153 additions and 16 deletions
|
@ -142,7 +142,7 @@ class InventoryManager(object):
|
||||||
self._sources = sources
|
self._sources = sources
|
||||||
|
|
||||||
# get to work!
|
# get to work!
|
||||||
self.parse_sources()
|
self.parse_sources(cache=True)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def localhost(self):
|
def localhost(self):
|
||||||
|
@ -234,7 +234,7 @@ class InventoryManager(object):
|
||||||
|
|
||||||
# recursively deal with directory entries
|
# recursively deal with directory entries
|
||||||
fullpath = os.path.join(b_source, i)
|
fullpath = os.path.join(b_source, i)
|
||||||
parsed_this_one = self.parse_source(to_native(fullpath))
|
parsed_this_one = self.parse_source(to_native(fullpath), cache=cache)
|
||||||
display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one))
|
display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one))
|
||||||
if not parsed:
|
if not parsed:
|
||||||
parsed = parsed_this_one
|
parsed = parsed_this_one
|
||||||
|
|
71
lib/ansible/plugins/cache/__init__.py
vendored
71
lib/ansible/plugins/cache/__init__.py
vendored
|
@ -79,12 +79,24 @@ class BaseFileCacheModule(BaseCacheModule):
|
||||||
self.plugin_name = self.__module__.split('.')[-1]
|
self.plugin_name = self.__module__.split('.')[-1]
|
||||||
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
|
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
|
||||||
self._cache = {}
|
self._cache = {}
|
||||||
self._cache_dir = None
|
self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
|
||||||
|
self._set_inventory_cache_override(**kwargs)
|
||||||
|
self.validate_cache_connection()
|
||||||
|
|
||||||
if C.CACHE_PLUGIN_CONNECTION:
|
def _get_cache_connection(self, source):
|
||||||
# expects a dir path
|
if source:
|
||||||
self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION))
|
try:
|
||||||
|
return os.path.expanduser(os.path.expandvars(source))
|
||||||
|
except TypeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _set_inventory_cache_override(self, **kwargs):
|
||||||
|
if kwargs.get('cache_timeout'):
|
||||||
|
self._timeout = kwargs.get('cache_timeout')
|
||||||
|
if kwargs.get('cache_connection'):
|
||||||
|
self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
|
||||||
|
|
||||||
|
def validate_cache_connection(self):
|
||||||
if not self._cache_dir:
|
if not self._cache_dir:
|
||||||
raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
|
raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
|
||||||
"to be set (to a writeable directory path)" % self.plugin_name)
|
"to be set (to a writeable directory path)" % self.plugin_name)
|
||||||
|
@ -141,7 +153,7 @@ class BaseFileCacheModule(BaseCacheModule):
|
||||||
def has_expired(self, key):
|
def has_expired(self, key):
|
||||||
|
|
||||||
if self._timeout == 0:
|
if self._timeout == 0:
|
||||||
return False
|
return True
|
||||||
|
|
||||||
cachefile = "%s/%s" % (self._cache_dir, key)
|
cachefile = "%s/%s" % (self._cache_dir, key)
|
||||||
try:
|
try:
|
||||||
|
@ -284,3 +296,52 @@ class FactCache(MutableMapping):
|
||||||
host_cache = self._plugin.get(key)
|
host_cache = self._plugin.get(key)
|
||||||
host_cache.update(value)
|
host_cache.update(value)
|
||||||
self._plugin.set(key, host_cache)
|
self._plugin.set(key, host_cache)
|
||||||
|
|
||||||
|
|
||||||
|
class InventoryFileCacheModule(BaseFileCacheModule):
|
||||||
|
"""
|
||||||
|
A caching module backed by file based storage.
|
||||||
|
"""
|
||||||
|
def __init__(self, plugin_name, timeout, cache_dir):
|
||||||
|
|
||||||
|
self.plugin_name = plugin_name
|
||||||
|
self._timeout = timeout
|
||||||
|
self._cache = {}
|
||||||
|
self._cache_dir = self._get_cache_connection(cache_dir)
|
||||||
|
self.validate_cache_connection()
|
||||||
|
self._plugin = self.get_plugin(plugin_name)
|
||||||
|
|
||||||
|
def validate_cache_connection(self):
|
||||||
|
try:
|
||||||
|
super(InventoryFileCacheModule, self).validate_cache_connection()
|
||||||
|
except AnsibleError as e:
|
||||||
|
cache_connection_set = False
|
||||||
|
else:
|
||||||
|
cache_connection_set = True
|
||||||
|
|
||||||
|
if not cache_connection_set:
|
||||||
|
raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
|
||||||
|
"ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
|
||||||
|
"Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
|
||||||
|
"to be set to a writeable directory path" % self.plugin_name)
|
||||||
|
|
||||||
|
def get(self, cache_key):
|
||||||
|
|
||||||
|
if not self.contains(cache_key):
|
||||||
|
# Check if cache file exists
|
||||||
|
raise KeyError
|
||||||
|
|
||||||
|
return super(InventoryFileCacheModule, self).get(cache_key)
|
||||||
|
|
||||||
|
def get_plugin(self, plugin_name):
|
||||||
|
plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
|
||||||
|
if not plugin:
|
||||||
|
raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
|
||||||
|
self._cache = {}
|
||||||
|
return plugin
|
||||||
|
|
||||||
|
def _load(self, path):
|
||||||
|
return self._plugin._load(path)
|
||||||
|
|
||||||
|
def _dump(self, value, path):
|
||||||
|
return self._plugin._dump(value, path)
|
||||||
|
|
|
@ -28,6 +28,7 @@ from collections import MutableMapping
|
||||||
|
|
||||||
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
|
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
|
||||||
from ansible.plugins import AnsiblePlugin
|
from ansible.plugins import AnsiblePlugin
|
||||||
|
from ansible.plugins.cache import InventoryFileCacheModule
|
||||||
from ansible.module_utils._text import to_bytes, to_native
|
from ansible.module_utils._text import to_bytes, to_native
|
||||||
from ansible.module_utils.parsing.convert_bool import boolean
|
from ansible.module_utils.parsing.convert_bool import boolean
|
||||||
from ansible.module_utils.six import string_types
|
from ansible.module_utils.six import string_types
|
||||||
|
@ -142,6 +143,7 @@ class BaseInventoryPlugin(AnsiblePlugin):
|
||||||
self._options = {}
|
self._options = {}
|
||||||
self.inventory = None
|
self.inventory = None
|
||||||
self.display = display
|
self.display = display
|
||||||
|
self.cache = None
|
||||||
|
|
||||||
def parse(self, inventory, loader, path, cache=True):
|
def parse(self, inventory, loader, path, cache=True):
|
||||||
''' Populates self.groups from the given data. Raises an error on any parse failure. '''
|
''' Populates self.groups from the given data. Raises an error on any parse failure. '''
|
||||||
|
@ -185,9 +187,16 @@ class BaseInventoryPlugin(AnsiblePlugin):
|
||||||
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
|
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
|
||||||
|
|
||||||
self.set_options(direct=config)
|
self.set_options(direct=config)
|
||||||
|
if self._options.get('cache'):
|
||||||
|
self._set_cache_options(self._options)
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
def _set_cache_options(self, options):
|
||||||
|
self.cache = InventoryFileCacheModule(plugin_name=options.get('cache_plugin'),
|
||||||
|
timeout=options.get('cache_timeout'),
|
||||||
|
cache_dir=options.get('cache_connection'))
|
||||||
|
|
||||||
def _consume_options(self, data):
|
def _consume_options(self, data):
|
||||||
''' update existing options from file data'''
|
''' update existing options from file data'''
|
||||||
|
|
||||||
|
@ -213,6 +222,9 @@ class Cacheable(object):
|
||||||
|
|
||||||
_cache = {}
|
_cache = {}
|
||||||
|
|
||||||
|
def get_cache_key(self, path):
|
||||||
|
return "{0}_{1}_{2}".format(self.NAME, self._get_cache_prefix(path), self._get_config_identifier(path))
|
||||||
|
|
||||||
def _get_cache_prefix(self, path):
|
def _get_cache_prefix(self, path):
|
||||||
''' create predictable unique prefix for plugin/inventory '''
|
''' create predictable unique prefix for plugin/inventory '''
|
||||||
|
|
||||||
|
@ -226,6 +238,11 @@ class Cacheable(object):
|
||||||
|
|
||||||
return 's_'.join([d1[:5], d2[:5]])
|
return 's_'.join([d1[:5], d2[:5]])
|
||||||
|
|
||||||
|
def _get_config_identifier(self, path):
|
||||||
|
''' create predictable config-specific prefix for plugin/inventory '''
|
||||||
|
|
||||||
|
return hashlib.md5(path.encode()).hexdigest()
|
||||||
|
|
||||||
def clear_cache(self):
|
def clear_cache(self):
|
||||||
self._cache = {}
|
self._cache = {}
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,7 @@ DOCUMENTATION = '''
|
||||||
- The inventory_hostname is always the 'Name' of the virtualbox instance.
|
- The inventory_hostname is always the 'Name' of the virtualbox instance.
|
||||||
extends_documentation_fragment:
|
extends_documentation_fragment:
|
||||||
- constructed
|
- constructed
|
||||||
|
- inventory_cache
|
||||||
options:
|
options:
|
||||||
running_only:
|
running_only:
|
||||||
description: toggles showing all vms vs only those currently running
|
description: toggles showing all vms vs only those currently running
|
||||||
|
@ -91,7 +92,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
# constructed groups based on conditionals
|
# constructed groups based on conditionals
|
||||||
self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host)
|
self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host)
|
||||||
|
|
||||||
def _populate_from_source(self, source_data):
|
def _populate_from_cache(self, source_data):
|
||||||
|
hostvars = source_data.pop('_meta', {}).get('hostvars', {})
|
||||||
|
for group in source_data:
|
||||||
|
if group == 'all':
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
self.inventory.add_group(group)
|
||||||
|
hosts = source_data[group].get('hosts', [])
|
||||||
|
for host in hosts:
|
||||||
|
self._populate_host_vars([host], hostvars.get(host, {}), group)
|
||||||
|
self.inventory.add_child('all', group)
|
||||||
|
if not source_data:
|
||||||
|
for host in hostvars:
|
||||||
|
self.inventory.add_host(host)
|
||||||
|
self._populate_host_vars([host], hostvars.get(host, {}))
|
||||||
|
|
||||||
|
def _populate_from_source(self, source_data, using_current_cache=False):
|
||||||
|
if using_current_cache:
|
||||||
|
self._populate_from_cache(source_data)
|
||||||
|
return source_data
|
||||||
|
|
||||||
|
cacheable_results = {'_meta': {'hostvars': {}}}
|
||||||
|
|
||||||
hostvars = {}
|
hostvars = {}
|
||||||
prevkey = pref_k = ''
|
prevkey = pref_k = ''
|
||||||
current_host = None
|
current_host = None
|
||||||
|
@ -100,6 +123,9 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
netinfo = self.get_option('network_info_path')
|
netinfo = self.get_option('network_info_path')
|
||||||
|
|
||||||
for line in source_data:
|
for line in source_data:
|
||||||
|
line = to_text(line)
|
||||||
|
if ':' not in line:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
k, v = line.split(':', 1)
|
k, v = line.split(':', 1)
|
||||||
except:
|
except:
|
||||||
|
@ -127,8 +153,11 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
elif k == 'Groups':
|
elif k == 'Groups':
|
||||||
for group in v.split('/'):
|
for group in v.split('/'):
|
||||||
if group:
|
if group:
|
||||||
|
if group not in cacheable_results:
|
||||||
|
cacheable_results[group] = {'hosts': []}
|
||||||
self.inventory.add_group(group)
|
self.inventory.add_group(group)
|
||||||
self.inventory.add_child(group, current_host)
|
self.inventory.add_child(group, current_host)
|
||||||
|
cacheable_results[group]['hosts'].append(current_host)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
@ -141,10 +170,32 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
else:
|
else:
|
||||||
if v != '':
|
if v != '':
|
||||||
hostvars[current_host][pref_k] = v
|
hostvars[current_host][pref_k] = v
|
||||||
|
if self._ungrouped_host(current_host, cacheable_results):
|
||||||
|
if 'ungrouped' not in cacheable_results:
|
||||||
|
cacheable_results['ungrouped'] = {'hosts': []}
|
||||||
|
cacheable_results['ungrouped']['hosts'].append(current_host)
|
||||||
|
|
||||||
prevkey = pref_k
|
prevkey = pref_k
|
||||||
|
|
||||||
self._set_variables(hostvars)
|
self._set_variables(hostvars)
|
||||||
|
for host in hostvars:
|
||||||
|
h = self.inventory.get_host(host)
|
||||||
|
cacheable_results['_meta']['hostvars'][h.name] = h.vars
|
||||||
|
|
||||||
|
return cacheable_results
|
||||||
|
|
||||||
|
def _ungrouped_host(self, host, inventory):
|
||||||
|
def find_host(host, inventory):
|
||||||
|
for k, v in inventory.items():
|
||||||
|
if k == '_meta':
|
||||||
|
continue
|
||||||
|
if isinstance(v, dict):
|
||||||
|
yield self._ungrouped_host(host, v)
|
||||||
|
elif isinstance(v, list):
|
||||||
|
yield host not in v
|
||||||
|
yield True
|
||||||
|
|
||||||
|
return all([found_host for found_host in find_host(host, inventory)])
|
||||||
|
|
||||||
def verify_file(self, path):
|
def verify_file(self, path):
|
||||||
|
|
||||||
|
@ -158,7 +209,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
|
|
||||||
super(InventoryModule, self).parse(inventory, loader, path)
|
super(InventoryModule, self).parse(inventory, loader, path)
|
||||||
|
|
||||||
cache_key = self._get_cache_prefix(path)
|
cache_key = self.get_cache_key(path)
|
||||||
|
|
||||||
config_data = self._read_config_data(path)
|
config_data = self._read_config_data(path)
|
||||||
|
|
||||||
|
@ -166,11 +217,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
self._consume_options(config_data)
|
self._consume_options(config_data)
|
||||||
|
|
||||||
source_data = None
|
source_data = None
|
||||||
if cache and cache_key in self._cache:
|
if cache:
|
||||||
|
cache = self._options.get('cache')
|
||||||
|
|
||||||
|
update_cache = False
|
||||||
|
if cache:
|
||||||
try:
|
try:
|
||||||
source_data = self._cache[cache_key]
|
source_data = self.cache.get(cache_key)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
update_cache = True
|
||||||
|
|
||||||
if not source_data:
|
if not source_data:
|
||||||
b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict')
|
b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict')
|
||||||
|
@ -192,7 +247,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
AnsibleParserError(to_native(e))
|
AnsibleParserError(to_native(e))
|
||||||
|
|
||||||
source_data = p.stdout.read()
|
source_data = p.stdout.read().splitlines()
|
||||||
self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
|
|
||||||
|
|
||||||
self._populate_from_source(source_data.splitlines())
|
using_current_cache = cache and not update_cache
|
||||||
|
cacheable_results = self._populate_from_source(source_data, using_current_cache)
|
||||||
|
|
||||||
|
if update_cache:
|
||||||
|
self.cache.set(cache_key, cacheable_results)
|
||||||
|
|
|
@ -179,6 +179,7 @@ class StrategyBase:
|
||||||
self._final_q = tqm._final_q
|
self._final_q = tqm._final_q
|
||||||
self._step = getattr(tqm._options, 'step', False)
|
self._step = getattr(tqm._options, 'step', False)
|
||||||
self._diff = getattr(tqm._options, 'diff', False)
|
self._diff = getattr(tqm._options, 'diff', False)
|
||||||
|
self.flush_cache = getattr(tqm._options, 'flush_cache', False)
|
||||||
|
|
||||||
# the task cache is a dictionary of tuples of (host.name, task._uuid)
|
# the task cache is a dictionary of tuples of (host.name, task._uuid)
|
||||||
# used to find the original task object of in-flight tasks and to store
|
# used to find the original task object of in-flight tasks and to store
|
||||||
|
@ -956,7 +957,7 @@ class StrategyBase:
|
||||||
elif meta_action == 'flush_handlers':
|
elif meta_action == 'flush_handlers':
|
||||||
self.run_handlers(iterator, play_context)
|
self.run_handlers(iterator, play_context)
|
||||||
msg = "ran handlers"
|
msg = "ran handlers"
|
||||||
elif meta_action == 'refresh_inventory':
|
elif meta_action == 'refresh_inventory' or self.flush_cache:
|
||||||
self._inventory.refresh_inventory()
|
self._inventory.refresh_inventory()
|
||||||
msg = "inventory successfully refreshed"
|
msg = "inventory successfully refreshed"
|
||||||
elif meta_action == 'clear_facts':
|
elif meta_action == 'clear_facts':
|
||||||
|
|
Loading…
Reference in a new issue