add inventory caching & use in virtualbox inventory plugin (#34510)

* Inventory caching

* Add inventory caching for virtualbox

* Don't populate cache for virtualbox with stdout, use a dict of inventory instead

* Fix error creating the cache dir if it doesn't exist

* Keep cache default False and set to True in VariableManager __init__

* Check all groups before determining if a host is ungrouped.
This commit is contained in:
Sloane Hertel 2018-01-22 19:33:14 -05:00 committed by Ryan Brown
parent 9fdaa86c9f
commit 4a1cc661c4
5 changed files with 153 additions and 16 deletions

View file

@ -142,7 +142,7 @@ class InventoryManager(object):
self._sources = sources self._sources = sources
# get to work! # get to work!
self.parse_sources() self.parse_sources(cache=True)
@property @property
def localhost(self): def localhost(self):
@ -234,7 +234,7 @@ class InventoryManager(object):
# recursively deal with directory entries # recursively deal with directory entries
fullpath = os.path.join(b_source, i) fullpath = os.path.join(b_source, i)
parsed_this_one = self.parse_source(to_native(fullpath)) parsed_this_one = self.parse_source(to_native(fullpath), cache=cache)
display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one)) display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one))
if not parsed: if not parsed:
parsed = parsed_this_one parsed = parsed_this_one

View file

@ -79,12 +79,24 @@ class BaseFileCacheModule(BaseCacheModule):
self.plugin_name = self.__module__.split('.')[-1] self.plugin_name = self.__module__.split('.')[-1]
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT) self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
self._cache = {} self._cache = {}
self._cache_dir = None self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
self._set_inventory_cache_override(**kwargs)
self.validate_cache_connection()
if C.CACHE_PLUGIN_CONNECTION: def _get_cache_connection(self, source):
# expects a dir path if source:
self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION)) try:
return os.path.expanduser(os.path.expandvars(source))
except TypeError:
pass
def _set_inventory_cache_override(self, **kwargs):
if kwargs.get('cache_timeout'):
self._timeout = kwargs.get('cache_timeout')
if kwargs.get('cache_connection'):
self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
def validate_cache_connection(self):
if not self._cache_dir: if not self._cache_dir:
raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option " raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
"to be set (to a writeable directory path)" % self.plugin_name) "to be set (to a writeable directory path)" % self.plugin_name)
@ -141,7 +153,7 @@ class BaseFileCacheModule(BaseCacheModule):
def has_expired(self, key): def has_expired(self, key):
if self._timeout == 0: if self._timeout == 0:
return False return True
cachefile = "%s/%s" % (self._cache_dir, key) cachefile = "%s/%s" % (self._cache_dir, key)
try: try:
@ -284,3 +296,52 @@ class FactCache(MutableMapping):
host_cache = self._plugin.get(key) host_cache = self._plugin.get(key)
host_cache.update(value) host_cache.update(value)
self._plugin.set(key, host_cache) self._plugin.set(key, host_cache)
class InventoryFileCacheModule(BaseFileCacheModule):
"""
A caching module backed by file based storage.
"""
def __init__(self, plugin_name, timeout, cache_dir):
self.plugin_name = plugin_name
self._timeout = timeout
self._cache = {}
self._cache_dir = self._get_cache_connection(cache_dir)
self.validate_cache_connection()
self._plugin = self.get_plugin(plugin_name)
def validate_cache_connection(self):
try:
super(InventoryFileCacheModule, self).validate_cache_connection()
except AnsibleError as e:
cache_connection_set = False
else:
cache_connection_set = True
if not cache_connection_set:
raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
"ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
"Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
"to be set to a writeable directory path" % self.plugin_name)
def get(self, cache_key):
if not self.contains(cache_key):
# Check if cache file exists
raise KeyError
return super(InventoryFileCacheModule, self).get(cache_key)
def get_plugin(self, plugin_name):
plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
if not plugin:
raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
self._cache = {}
return plugin
def _load(self, path):
return self._plugin._load(path)
def _dump(self, value, path):
return self._plugin._dump(value, path)

View file

@ -28,6 +28,7 @@ from collections import MutableMapping
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
from ansible.plugins import AnsiblePlugin from ansible.plugins import AnsiblePlugin
from ansible.plugins.cache import InventoryFileCacheModule
from ansible.module_utils._text import to_bytes, to_native from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.parsing.convert_bool import boolean from ansible.module_utils.parsing.convert_bool import boolean
from ansible.module_utils.six import string_types from ansible.module_utils.six import string_types
@ -142,6 +143,7 @@ class BaseInventoryPlugin(AnsiblePlugin):
self._options = {} self._options = {}
self.inventory = None self.inventory = None
self.display = display self.display = display
self.cache = None
def parse(self, inventory, loader, path, cache=True): def parse(self, inventory, loader, path, cache=True):
''' Populates self.groups from the given data. Raises an error on any parse failure. ''' ''' Populates self.groups from the given data. Raises an error on any parse failure. '''
@ -185,9 +187,16 @@ class BaseInventoryPlugin(AnsiblePlugin):
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config)) raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
self.set_options(direct=config) self.set_options(direct=config)
if self._options.get('cache'):
self._set_cache_options(self._options)
return config return config
def _set_cache_options(self, options):
self.cache = InventoryFileCacheModule(plugin_name=options.get('cache_plugin'),
timeout=options.get('cache_timeout'),
cache_dir=options.get('cache_connection'))
def _consume_options(self, data): def _consume_options(self, data):
''' update existing options from file data''' ''' update existing options from file data'''
@ -213,6 +222,9 @@ class Cacheable(object):
_cache = {} _cache = {}
def get_cache_key(self, path):
return "{0}_{1}_{2}".format(self.NAME, self._get_cache_prefix(path), self._get_config_identifier(path))
def _get_cache_prefix(self, path): def _get_cache_prefix(self, path):
''' create predictable unique prefix for plugin/inventory ''' ''' create predictable unique prefix for plugin/inventory '''
@ -226,6 +238,11 @@ class Cacheable(object):
return 's_'.join([d1[:5], d2[:5]]) return 's_'.join([d1[:5], d2[:5]])
def _get_config_identifier(self, path):
''' create predictable config-specific prefix for plugin/inventory '''
return hashlib.md5(path.encode()).hexdigest()
def clear_cache(self): def clear_cache(self):
self._cache = {} self._cache = {}

View file

@ -14,6 +14,7 @@ DOCUMENTATION = '''
- The inventory_hostname is always the 'Name' of the virtualbox instance. - The inventory_hostname is always the 'Name' of the virtualbox instance.
extends_documentation_fragment: extends_documentation_fragment:
- constructed - constructed
- inventory_cache
options: options:
running_only: running_only:
description: toggles showing all vms vs only those currently running description: toggles showing all vms vs only those currently running
@ -91,7 +92,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
# constructed groups based on conditionals # constructed groups based on conditionals
self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host) self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host)
def _populate_from_source(self, source_data): def _populate_from_cache(self, source_data):
hostvars = source_data.pop('_meta', {}).get('hostvars', {})
for group in source_data:
if group == 'all':
continue
else:
self.inventory.add_group(group)
hosts = source_data[group].get('hosts', [])
for host in hosts:
self._populate_host_vars([host], hostvars.get(host, {}), group)
self.inventory.add_child('all', group)
if not source_data:
for host in hostvars:
self.inventory.add_host(host)
self._populate_host_vars([host], hostvars.get(host, {}))
def _populate_from_source(self, source_data, using_current_cache=False):
if using_current_cache:
self._populate_from_cache(source_data)
return source_data
cacheable_results = {'_meta': {'hostvars': {}}}
hostvars = {} hostvars = {}
prevkey = pref_k = '' prevkey = pref_k = ''
current_host = None current_host = None
@ -100,6 +123,9 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
netinfo = self.get_option('network_info_path') netinfo = self.get_option('network_info_path')
for line in source_data: for line in source_data:
line = to_text(line)
if ':' not in line:
continue
try: try:
k, v = line.split(':', 1) k, v = line.split(':', 1)
except: except:
@ -127,8 +153,11 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
elif k == 'Groups': elif k == 'Groups':
for group in v.split('/'): for group in v.split('/'):
if group: if group:
if group not in cacheable_results:
cacheable_results[group] = {'hosts': []}
self.inventory.add_group(group) self.inventory.add_group(group)
self.inventory.add_child(group, current_host) self.inventory.add_child(group, current_host)
cacheable_results[group]['hosts'].append(current_host)
continue continue
else: else:
@ -141,10 +170,32 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
else: else:
if v != '': if v != '':
hostvars[current_host][pref_k] = v hostvars[current_host][pref_k] = v
if self._ungrouped_host(current_host, cacheable_results):
if 'ungrouped' not in cacheable_results:
cacheable_results['ungrouped'] = {'hosts': []}
cacheable_results['ungrouped']['hosts'].append(current_host)
prevkey = pref_k prevkey = pref_k
self._set_variables(hostvars) self._set_variables(hostvars)
for host in hostvars:
h = self.inventory.get_host(host)
cacheable_results['_meta']['hostvars'][h.name] = h.vars
return cacheable_results
def _ungrouped_host(self, host, inventory):
def find_host(host, inventory):
for k, v in inventory.items():
if k == '_meta':
continue
if isinstance(v, dict):
yield self._ungrouped_host(host, v)
elif isinstance(v, list):
yield host not in v
yield True
return all([found_host for found_host in find_host(host, inventory)])
def verify_file(self, path): def verify_file(self, path):
@ -158,7 +209,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
super(InventoryModule, self).parse(inventory, loader, path) super(InventoryModule, self).parse(inventory, loader, path)
cache_key = self._get_cache_prefix(path) cache_key = self.get_cache_key(path)
config_data = self._read_config_data(path) config_data = self._read_config_data(path)
@ -166,11 +217,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
self._consume_options(config_data) self._consume_options(config_data)
source_data = None source_data = None
if cache and cache_key in self._cache: if cache:
cache = self._options.get('cache')
update_cache = False
if cache:
try: try:
source_data = self._cache[cache_key] source_data = self.cache.get(cache_key)
except KeyError: except KeyError:
pass update_cache = True
if not source_data: if not source_data:
b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict') b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict')
@ -192,7 +247,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
except Exception as e: except Exception as e:
AnsibleParserError(to_native(e)) AnsibleParserError(to_native(e))
source_data = p.stdout.read() source_data = p.stdout.read().splitlines()
self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
self._populate_from_source(source_data.splitlines()) using_current_cache = cache and not update_cache
cacheable_results = self._populate_from_source(source_data, using_current_cache)
if update_cache:
self.cache.set(cache_key, cacheable_results)

View file

@ -179,6 +179,7 @@ class StrategyBase:
self._final_q = tqm._final_q self._final_q = tqm._final_q
self._step = getattr(tqm._options, 'step', False) self._step = getattr(tqm._options, 'step', False)
self._diff = getattr(tqm._options, 'diff', False) self._diff = getattr(tqm._options, 'diff', False)
self.flush_cache = getattr(tqm._options, 'flush_cache', False)
# the task cache is a dictionary of tuples of (host.name, task._uuid) # the task cache is a dictionary of tuples of (host.name, task._uuid)
# used to find the original task object of in-flight tasks and to store # used to find the original task object of in-flight tasks and to store
@ -956,7 +957,7 @@ class StrategyBase:
elif meta_action == 'flush_handlers': elif meta_action == 'flush_handlers':
self.run_handlers(iterator, play_context) self.run_handlers(iterator, play_context)
msg = "ran handlers" msg = "ran handlers"
elif meta_action == 'refresh_inventory': elif meta_action == 'refresh_inventory' or self.flush_cache:
self._inventory.refresh_inventory() self._inventory.refresh_inventory()
msg = "inventory successfully refreshed" msg = "inventory successfully refreshed"
elif meta_action == 'clear_facts': elif meta_action == 'clear_facts':