cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Refactor vars_plugins (host/group_vars)

Browse source 
Split out parsing of vars files to per host and per group
parsing, instead of reparsing all groups for each host. This enhances
performance.

Extend vars_plugins' API with two new methods:
* get host variables: only parses host_vars
* get group variables: only parses group_vars for specific group
The initial run method is still used for backward compatibility.

Parse all vars_plugins at inventory initialisation, instead of
per host when touched first by runner. Here we can also loop through
all groups once easily, then parse them.
This also centralizes all parsing in the inventory constructor.

	modified:   bin/ansible
	modified:   bin/ansible-playbook
	modified:   lib/ansible/inventory/__init__.py
	modified:   lib/ansible/inventory/vars_plugins/group_vars.py
This commit is contained in:
Serge van Ginderachter 2014-03-17 18:05:30 +01:00
parent 3194fbd365
commit f6a55a3552
4 changed files with 70 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/ansible
View file

@ -136,7 +136,7 @@ class Cli(object):
if not options.ask_vault_pass:
vault_pass = tmp_vault_pass
inventory_manager = inventory.Inventory(options.inventory)
inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass)
if options.subset:
inventory_manager.subset(options.subset)
hosts = inventory_manager.list_hosts(pattern)

10
bin/ansible-playbook
View file

@ -97,11 +97,6 @@ def main(args):
if (options.ask_vault_pass and options.vault_password_file):
parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive")
inventory = ansible.inventory.Inventory(options.inventory)
inventory.subset(options.subset)
if len(inventory.list_hosts()) == 0:
raise errors.AnsibleError("provided hosts list is empty")
sshpass = None
sudopass = None
su_pass = None
@ -155,6 +150,11 @@ def main(args):
if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook)
inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass)
inventory.subset(options.subset)
if len(inventory.list_hosts()) == 0:
raise errors.AnsibleError("provided hosts list is empty")
# run all playbooks specified on the command line
for playbook in args:

45
lib/ansible/inventory/__init__.py
View file

@ -39,13 +39,14 @@ class Inventory(object):
__slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset',
'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
'_pattern_cache', '_vars_plugins', '_playbook_basedir']
'_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir']
def __init__(self, host_list=C.DEFAULT_HOST_LIST):
def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None):
# the host file file, or script path, or list of hosts
# if a list, inventory data will NOT be loaded
self.host_list = host_list
self._vault_password=vault_password
# caching to avoid repeated calculations, particularly with
# external inventory scripts.
@ -140,6 +141,14 @@ class Inventory(object):
self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]
# get group vars from vars plugins
for group in self.groups:
group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password))
# get host vars from vars plugins
for host in self.get_hosts():
host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password))
def _match(self, str, pattern_str):
if pattern_str.startswith('~'):
@ -370,16 +379,25 @@ class Inventory(object):
return group
return None
def get_group_variables(self, groupname):
def get_group_variables(self, groupname, vault_password=None):
if groupname not in self._vars_per_group:
self._vars_per_group[groupname] = self._get_group_variables(groupname)
self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password)
return self._vars_per_group[groupname]
def _get_group_variables(self, groupname):
def _get_group_variables(self, groupname, vault_password=None):
group = self.get_group(groupname)
if group is None:
raise Exception("group not found: %s" % groupname)
return group.get_variables()
vars = {}
vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')]
for updated in vars_results:
if updated is not None:
vars.update(updated)
vars.update(group.get_variables())
return vars
def get_variables(self, hostname, vault_password=None):
if hostname not in self._vars_per_host:
@ -393,14 +411,27 @@ class Inventory(object):
raise errors.AnsibleError("host not found: %s" % hostname)
vars = {}
vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ]
# plugin.get_host_vars retrieves just vars for specific host
vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
for updated in vars_results:
if updated is not None:
vars = utils.combine_vars(vars, updated)
# plugin.run retrieves all vars (also from groups) for host
vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')]
for updated in vars_results:
if updated is not None:
vars = utils.combine_vars(vars, updated)
vars = utils.combine_vars(vars, host.get_variables())
# still need to check InventoryParser per host vars
# which actually means InventoryScript per host,
# which is not performant
if self.parser is not None:
vars = utils.combine_vars(vars, self.parser.get_host_variables(host))
return vars
def add_group(self, group):

46
lib/ansible/inventory/vars_plugins/group_vars.py
View file

@ -143,28 +143,33 @@ class VarsModule(object):
""" constructor """
self.inventory = inventory
self.inventory_basedir = inventory.basedir()
# There's no playbook initialized yet:
self.pb_basedir = None
def run(self, host, vault_password=None):
""" main body of the plugin, does actual loading """
def get_host_vars(self, host, vault_password=None):
return self._get_vars(host=host, group=None, vault_password=vault_password)
inventory = self.inventory
basedir = inventory.playbook_basedir()
if basedir is not None:
basedir = os.path.abspath(basedir)
self.pb_basedir = basedir
# sort groups by depth so deepest groups can override the less deep ones
groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth)
groups = [ g.name for g in groupz ]
inventory_basedir = inventory.basedir()
def get_group_vars(self, group, vault_password=None):
return self._get_vars(host=None, group=group, vault_password=vault_password)
def _get_vars(self, host=None, group=None, vault_password=None):
""" main body of the plugin, does actual loading"""
if self.pb_basedir is None:
pb_basedir = self.inventory.playbook_basedir()
if pb_basedir is not None:
pb_basedir = os.path.abspath(pb_basedir)
self.pb_basedir = pb_basedir
results = {}
scan_pass = 0
# look in both the inventory base directory and the playbook base directory
for basedir in [ inventory_basedir, self.pb_basedir ]:
for basedir in [self.inventory_basedir, self.pb_basedir ]:
# this can happen from particular API usages, particularly if not run
# from /usr/bin/ansible-playbook
@ -178,17 +183,18 @@ class VarsModule(object):
continue
# save work of second scan if the directories are the same
if inventory_basedir == self.pb_basedir and scan_pass != 1:
if self.inventory_basedir == self.pb_basedir and scan_pass != 1:
continue
# load vars in dir/group_vars/name_of_group
for group in groups:
base_path = os.path.join(basedir, "group_vars/%s" % group)
if group and host is None:
# load vars in dir/group_vars/name_of_group
base_path = os.path.join(basedir, "group_vars/%s" % group.name)
results = _load_vars(base_path, results, vault_password=vault_password)
# same for hostvars in dir/host_vars/name_of_host
base_path = os.path.join(basedir, "host_vars/%s" % host.name)
results = _load_vars(base_path, results, vault_password=vault_password)
elif host and group is None:
# same for hostvars in dir/host_vars/name_of_host
base_path = os.path.join(basedir, "host_vars/%s" % host.name)
results = _load_vars(base_path, results, vault_password=vault_password)
# all done, results is a dictionary of variables for this particular host.
return results