cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'Yannig-devel_cache_for_do_template_call' into devel

Browse source
This commit is contained in:
James Cammarata 2015-11-06 22:22:23 -05:00
commit da4b4a2a33
3 changed files with 74 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
lib/ansible/inventory/__init__.py
View file

@ -37,6 +37,8 @@ from ansible.plugins import vars_loader
from ansible.utils.vars import combine_vars
from ansible.parsing.utils.addresses import parse_address
HOSTS_PATTERNS_CACHE = {}
try:
from __main__ import display
except ImportError:
@ -163,6 +165,11 @@ class Inventory(object):
or applied subsets
"""
# Check if pattern already computed
pattern_hash = str(pattern)
if pattern_hash in HOSTS_PATTERNS_CACHE:
return HOSTS_PATTERNS_CACHE[pattern_hash]
patterns = Inventory.split_host_pattern(pattern)
hosts = self._evaluate_patterns(patterns)
@ -177,6 +184,7 @@ class Inventory(object):
if self._restriction is not None:
hosts = [ h for h in hosts if h in self._restriction ]
HOSTS_PATTERNS_CACHE[pattern_hash] = hosts
return hosts
@classmethod

50
lib/ansible/template/__init__.py
View file

@ -39,6 +39,11 @@ from ansible.template.template import AnsibleJ2Template
from ansible.template.vars import AnsibleJ2Vars
from ansible.utils.debug import debug
try:
from hashlib import sha1
except ImportError:
from sha import sha as sha1
from numbers import Number
__all__ = ['Templar']
@ -122,6 +127,7 @@ class Templar:
self._filters = None
self._tests = None
self._available_variables = variables
self._cached_result = {}
if loader:
self._basedir = loader.get_basedir()
@ -254,19 +260,24 @@ class Templar:
'''
Sets the list of template variables this Templar instance will use
to template things, so we don't have to pass them around between
internal methods.
internal methods. We also clear the template cache here, as the variables
are being changed.
'''
assert isinstance(variables, dict)
self._available_variables = variables
self._cached_result = {}
def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True):
def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True, static_vars = ['']):
'''
Templates (possibly recursively) any given data as input. If convert_bare is
set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}')
before being sent through the template engine.
'''
if fail_on_undefined is None:
fail_on_undefined = self._fail_on_undefined_errors
# Don't template unsafe variables, instead drop them back down to
# their constituent type.
if hasattr(variable, '__UNSAFE__'):
@ -298,18 +309,26 @@ class Templar:
elif resolved_val is None:
return C.DEFAULT_NULL_REPRESENTATION
result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides)
# Using a cache in order to prevent template calls with already templated variables
variable_hash = sha1(text_type(variable).encode('utf-8'))
options_hash = sha1((text_type(preserve_trailing_newlines) + text_type(escape_backslashes) + text_type(fail_on_undefined) + text_type(overrides)).encode('utf-8'))
sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest()
if sha1_hash in self._cached_result:
result = self._cached_result[sha1_hash]
else:
result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides)
if convert_data:
# if this looks like a dictionary or list, convert it to such using the safe_eval method
if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
result.startswith("[") or result in ("True", "False"):
eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
if eval_results[1] is None:
result = eval_results[0]
else:
# FIXME: if the safe_eval raised an error, should we do something with it?
pass
self._cached_result[sha1_hash] = result
if convert_data:
# if this looks like a dictionary or list, convert it to such using the safe_eval method
if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
result.startswith("[") or result in ("True", "False"):
eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
if eval_results[1] is None:
result = eval_results[0]
else:
# FIXME: if the safe_eval raised an error, should we do something with it?
pass
#return self._clean_data(result)
return result
@ -321,7 +340,10 @@ class Templar:
# we don't use iteritems() here to avoid problems if the underlying dict
# changes sizes due to the templating, which can happen with hostvars
for k in variable.keys():
d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides)
if k not in static_vars:
d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides)
else:
d[k] = variable[k]
return d
else:
return variable

33
lib/ansible/vars/hostvars.py
View file

@ -28,6 +28,18 @@ from ansible import constants as C
from ansible.inventory.host import Host
from ansible.template import Templar
STATIC_VARS = [
'inventory_hostname', 'inventory_hostname_short',
'inventory_file', 'inventory_dir', 'playbook_dir',
'ansible_play_hosts', 'play_hosts', 'groups', 'ungrouped', 'group_names',
'ansible_version', 'omit', 'role_names'
]
try:
from hashlib import sha1
except ImportError:
from sha import sha as sha1
__all__ = ['HostVars']
# Note -- this is a Mapping, not a MutableMapping
@ -39,6 +51,7 @@ class HostVars(collections.Mapping):
self._loader = loader
self._play = play
self._variable_manager = variable_manager
self._cached_result = dict()
hosts = inventory.get_hosts(ignore_limits_and_restrictions=True)
@ -68,8 +81,16 @@ class HostVars(collections.Mapping):
host = self._lookup.get(host_name)
data = self._variable_manager.get_vars(loader=self._loader, host=host, play=self._play, include_hostvars=False)
templar = Templar(variables=data, loader=self._loader)
return templar.template(data, fail_on_undefined=False)
# Using cache in order to avoid template call
sha1_hash = sha1(str(data).encode('utf-8')).hexdigest()
if sha1_hash in self._cached_result:
result = self._cached_result[sha1_hash]
else:
templar = Templar(variables=data, loader=self._loader)
result = templar.template(data, fail_on_undefined=False, static_vars=STATIC_VARS)
self._cached_result[sha1_hash] = result
return result
def __contains__(self, host_name):
item = self.get(host_name)
@ -85,10 +106,16 @@ class HostVars(collections.Mapping):
return len(self._lookup)
def __getstate__(self):
return dict(loader=self._loader, lookup=self._lookup, play=self._play, var_manager=self._variable_manager)
return dict(
loader=self._loader,
lookup=self._lookup,
play=self._play,
var_manager=self._variable_manager,
)
def __setstate__(self, data):
self._play = data.get('play')
self._loader = data.get('loader')
self._lookup = data.get('lookup')
self._variable_manager = data.get('var_manager')
self._cached_result = dict()