add inventory caching & use in virtualbox inventory plugin (#34510)
* Inventory caching * Add inventory caching for virtualbox * Don't populate cache for virtualbox with stdout, use a dict of inventory instead * Fix error creating the cache dir if it doesn't exist * Keep cache default False and set to True in VariableManager __init__ * Check all groups before determining if a host is ungrouped.
This commit is contained in:
Sloane Hertel
Ryan Brown
parent
9fdaa86c9f
commit
4a1cc661c4
5 changed files with 153 additions and 16 deletions
|
|
@ -142,7 +142,7 @@ class InventoryManager(object):
|
|||
self._sources = sources
|
||||
|
||||
# get to work!
|
||||
self.parse_sources()
|
||||
self.parse_sources(cache=True)
|
||||
|
||||
@property
|
||||
def localhost(self):
|
||||
|
|
@ -234,7 +234,7 @@ class InventoryManager(object):
|
|||
|
||||
# recursively deal with directory entries
|
||||
fullpath = os.path.join(b_source, i)
|
||||
parsed_this_one = self.parse_source(to_native(fullpath))
|
||||
parsed_this_one = self.parse_source(to_native(fullpath), cache=cache)
|
||||
display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one))
|
||||
if not parsed:
|
||||
parsed = parsed_this_one
|
||||
|
|
|
|||
71
lib/ansible/plugins/cache/__init__.py
vendored
71
lib/ansible/plugins/cache/__init__.py
vendored
|
|
@ -79,12 +79,24 @@ class BaseFileCacheModule(BaseCacheModule):
|
|||
self.plugin_name = self.__module__.split('.')[-1]
|
||||
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
|
||||
self._cache = {}
|
||||
self._cache_dir = None
|
||||
self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
|
||||
self._set_inventory_cache_override(**kwargs)
|
||||
self.validate_cache_connection()
|
||||
|
||||
if C.CACHE_PLUGIN_CONNECTION:
|
||||
# expects a dir path
|
||||
self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION))
|
||||
def _get_cache_connection(self, source):
|
||||
if source:
|
||||
try:
|
||||
return os.path.expanduser(os.path.expandvars(source))
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
def _set_inventory_cache_override(self, **kwargs):
|
||||
if kwargs.get('cache_timeout'):
|
||||
self._timeout = kwargs.get('cache_timeout')
|
||||
if kwargs.get('cache_connection'):
|
||||
self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
|
||||
|
||||
def validate_cache_connection(self):
|
||||
if not self._cache_dir:
|
||||
raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
|
||||
"to be set (to a writeable directory path)" % self.plugin_name)
|
||||
|
|
@ -141,7 +153,7 @@ class BaseFileCacheModule(BaseCacheModule):
|
|||
def has_expired(self, key):
|
||||
|
||||
if self._timeout == 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
cachefile = "%s/%s" % (self._cache_dir, key)
|
||||
try:
|
||||
|
|
@ -284,3 +296,52 @@ class FactCache(MutableMapping):
|
|||
host_cache = self._plugin.get(key)
|
||||
host_cache.update(value)
|
||||
self._plugin.set(key, host_cache)
|
||||
|
||||
|
||||
class InventoryFileCacheModule(BaseFileCacheModule):
|
||||
"""
|
||||
A caching module backed by file based storage.
|
||||
"""
|
||||
def __init__(self, plugin_name, timeout, cache_dir):
|
||||
|
||||
self.plugin_name = plugin_name
|
||||
self._timeout = timeout
|
||||
self._cache = {}
|
||||
self._cache_dir = self._get_cache_connection(cache_dir)
|
||||
self.validate_cache_connection()
|
||||
self._plugin = self.get_plugin(plugin_name)
|
||||
|
||||
def validate_cache_connection(self):
|
||||
try:
|
||||
super(InventoryFileCacheModule, self).validate_cache_connection()
|
||||
except AnsibleError as e:
|
||||
cache_connection_set = False
|
||||
else:
|
||||
cache_connection_set = True
|
||||
|
||||
if not cache_connection_set:
|
||||
raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
|
||||
"ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
|
||||
"Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
|
||||
"to be set to a writeable directory path" % self.plugin_name)
|
||||
|
||||
def get(self, cache_key):
|
||||
|
||||
if not self.contains(cache_key):
|
||||
# Check if cache file exists
|
||||
raise KeyError
|
||||
|
||||
return super(InventoryFileCacheModule, self).get(cache_key)
|
||||
|
||||
def get_plugin(self, plugin_name):
|
||||
plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
|
||||
if not plugin:
|
||||
raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
|
||||
self._cache = {}
|
||||
return plugin
|
||||
|
||||
def _load(self, path):
|
||||
return self._plugin._load(path)
|
||||
|
||||
def _dump(self, value, path):
|
||||
return self._plugin._dump(value, path)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from collections import MutableMapping
|
|||
|
||||
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
|
||||
from ansible.plugins import AnsiblePlugin
|
||||
from ansible.plugins.cache import InventoryFileCacheModule
|
||||
from ansible.module_utils._text import to_bytes, to_native
|
||||
from ansible.module_utils.parsing.convert_bool import boolean
|
||||
from ansible.module_utils.six import string_types
|
||||
|
|
@ -142,6 +143,7 @@ class BaseInventoryPlugin(AnsiblePlugin):
|
|||
self._options = {}
|
||||
self.inventory = None
|
||||
self.display = display
|
||||
self.cache = None
|
||||
|
||||
def parse(self, inventory, loader, path, cache=True):
|
||||
''' Populates self.groups from the given data. Raises an error on any parse failure. '''
|
||||
|
|
@ -185,9 +187,16 @@ class BaseInventoryPlugin(AnsiblePlugin):
|
|||
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
|
||||
|
||||
self.set_options(direct=config)
|
||||
if self._options.get('cache'):
|
||||
self._set_cache_options(self._options)
|
||||
|
||||
return config
|
||||
|
||||
def _set_cache_options(self, options):
|
||||
self.cache = InventoryFileCacheModule(plugin_name=options.get('cache_plugin'),
|
||||
timeout=options.get('cache_timeout'),
|
||||
cache_dir=options.get('cache_connection'))
|
||||
|
||||
def _consume_options(self, data):
|
||||
''' update existing options from file data'''
|
||||
|
||||
|
|
@ -213,6 +222,9 @@ class Cacheable(object):
|
|||
|
||||
_cache = {}
|
||||
|
||||
def get_cache_key(self, path):
|
||||
return "{0}_{1}_{2}".format(self.NAME, self._get_cache_prefix(path), self._get_config_identifier(path))
|
||||
|
||||
def _get_cache_prefix(self, path):
|
||||
''' create predictable unique prefix for plugin/inventory '''
|
||||
|
||||
|
|
@ -226,6 +238,11 @@ class Cacheable(object):
|
|||
|
||||
return 's_'.join([d1[:5], d2[:5]])
|
||||
|
||||
def _get_config_identifier(self, path):
|
||||
''' create predictable config-specific prefix for plugin/inventory '''
|
||||
|
||||
return hashlib.md5(path.encode()).hexdigest()
|
||||
|
||||
def clear_cache(self):
|
||||
self._cache = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ DOCUMENTATION = '''
|
|||
- The inventory_hostname is always the 'Name' of the virtualbox instance.
|
||||
extends_documentation_fragment:
|
||||
- constructed
|
||||
- inventory_cache
|
||||
options:
|
||||
running_only:
|
||||
description: toggles showing all vms vs only those currently running
|
||||
|
|
@ -91,7 +92,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
# constructed groups based on conditionals
|
||||
self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host)
|
||||
|
||||
def _populate_from_source(self, source_data):
|
||||
def _populate_from_cache(self, source_data):
|
||||
hostvars = source_data.pop('_meta', {}).get('hostvars', {})
|
||||
for group in source_data:
|
||||
if group == 'all':
|
||||
continue
|
||||
else:
|
||||
self.inventory.add_group(group)
|
||||
hosts = source_data[group].get('hosts', [])
|
||||
for host in hosts:
|
||||
self._populate_host_vars([host], hostvars.get(host, {}), group)
|
||||
self.inventory.add_child('all', group)
|
||||
if not source_data:
|
||||
for host in hostvars:
|
||||
self.inventory.add_host(host)
|
||||
self._populate_host_vars([host], hostvars.get(host, {}))
|
||||
|
||||
def _populate_from_source(self, source_data, using_current_cache=False):
|
||||
if using_current_cache:
|
||||
self._populate_from_cache(source_data)
|
||||
return source_data
|
||||
|
||||
cacheable_results = {'_meta': {'hostvars': {}}}
|
||||
|
||||
hostvars = {}
|
||||
prevkey = pref_k = ''
|
||||
current_host = None
|
||||
|
|
@ -100,6 +123,9 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
netinfo = self.get_option('network_info_path')
|
||||
|
||||
for line in source_data:
|
||||
line = to_text(line)
|
||||
if ':' not in line:
|
||||
continue
|
||||
try:
|
||||
k, v = line.split(':', 1)
|
||||
except:
|
||||
|
|
@ -127,8 +153,11 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
elif k == 'Groups':
|
||||
for group in v.split('/'):
|
||||
if group:
|
||||
if group not in cacheable_results:
|
||||
cacheable_results[group] = {'hosts': []}
|
||||
self.inventory.add_group(group)
|
||||
self.inventory.add_child(group, current_host)
|
||||
cacheable_results[group]['hosts'].append(current_host)
|
||||
continue
|
||||
|
||||
else:
|
||||
|
|
@ -141,10 +170,32 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
else:
|
||||
if v != '':
|
||||
hostvars[current_host][pref_k] = v
|
||||
if self._ungrouped_host(current_host, cacheable_results):
|
||||
if 'ungrouped' not in cacheable_results:
|
||||
cacheable_results['ungrouped'] = {'hosts': []}
|
||||
cacheable_results['ungrouped']['hosts'].append(current_host)
|
||||
|
||||
prevkey = pref_k
|
||||
|
||||
self._set_variables(hostvars)
|
||||
for host in hostvars:
|
||||
h = self.inventory.get_host(host)
|
||||
cacheable_results['_meta']['hostvars'][h.name] = h.vars
|
||||
|
||||
return cacheable_results
|
||||
|
||||
def _ungrouped_host(self, host, inventory):
|
||||
def find_host(host, inventory):
|
||||
for k, v in inventory.items():
|
||||
if k == '_meta':
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
yield self._ungrouped_host(host, v)
|
||||
elif isinstance(v, list):
|
||||
yield host not in v
|
||||
yield True
|
||||
|
||||
return all([found_host for found_host in find_host(host, inventory)])
|
||||
|
||||
def verify_file(self, path):
|
||||
|
||||
|
|
@ -158,7 +209,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
|
||||
super(InventoryModule, self).parse(inventory, loader, path)
|
||||
|
||||
cache_key = self._get_cache_prefix(path)
|
||||
cache_key = self.get_cache_key(path)
|
||||
|
||||
config_data = self._read_config_data(path)
|
||||
|
||||
|
|
@ -166,11 +217,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
self._consume_options(config_data)
|
||||
|
||||
source_data = None
|
||||
if cache and cache_key in self._cache:
|
||||
if cache:
|
||||
cache = self._options.get('cache')
|
||||
|
||||
update_cache = False
|
||||
if cache:
|
||||
try:
|
||||
source_data = self._cache[cache_key]
|
||||
source_data = self.cache.get(cache_key)
|
||||
except KeyError:
|
||||
pass
|
||||
update_cache = True
|
||||
|
||||
if not source_data:
|
||||
b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict')
|
||||
|
|
@ -192,7 +247,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
|||
except Exception as e:
|
||||
AnsibleParserError(to_native(e))
|
||||
|
||||
source_data = p.stdout.read()
|
||||
self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
|
||||
source_data = p.stdout.read().splitlines()
|
||||
|
||||
self._populate_from_source(source_data.splitlines())
|
||||
using_current_cache = cache and not update_cache
|
||||
cacheable_results = self._populate_from_source(source_data, using_current_cache)
|
||||
|
||||
if update_cache:
|
||||
self.cache.set(cache_key, cacheable_results)
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ class StrategyBase:
|
|||
self._final_q = tqm._final_q
|
||||
self._step = getattr(tqm._options, 'step', False)
|
||||
self._diff = getattr(tqm._options, 'diff', False)
|
||||
self.flush_cache = getattr(tqm._options, 'flush_cache', False)
|
||||
|
||||
# the task cache is a dictionary of tuples of (host.name, task._uuid)
|
||||
# used to find the original task object of in-flight tasks and to store
|
||||
|
|
@ -956,7 +957,7 @@ class StrategyBase:
|
|||
elif meta_action == 'flush_handlers':
|
||||
self.run_handlers(iterator, play_context)
|
||||
msg = "ran handlers"
|
||||
elif meta_action == 'refresh_inventory':
|
||||
elif meta_action == 'refresh_inventory' or self.flush_cache:
|
||||
self._inventory.refresh_inventory()
|
||||
msg = "inventory successfully refreshed"
|
||||
elif meta_action == 'clear_facts':
|
||||
|
|
|
|||
Loading…
Reference in a new issue