From f6a55a35525a6ca0b56ad856f2ab11bbd226c1b4 Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Mon, 17 Mar 2014 18:05:30 +0100 Subject: [PATCH 1/5] Refactor vars_plugins (host/group_vars) Split out parsing of vars files to per host and per group parsing, instead of reparsing all groups for each host. This enhances performance. Extend vars_plugins' API with two new methods: * get host variables: only parses host_vars * get group variables: only parses group_vars for specific group The initial run method is still used for backward compatibility. Parse all vars_plugins at inventory initialisation, instead of per host when touched first by runner. Here we can also loop through all groups once easily, then parse them. This also centralizes all parsing in the inventory constructor. modified: bin/ansible modified: bin/ansible-playbook modified: lib/ansible/inventory/__init__.py modified: lib/ansible/inventory/vars_plugins/group_vars.py --- bin/ansible | 2 +- bin/ansible-playbook | 10 ++-- lib/ansible/inventory/__init__.py | 45 +++++++++++++++--- .../inventory/vars_plugins/group_vars.py | 46 +++++++++++-------- 4 files changed, 70 insertions(+), 33 deletions(-) diff --git a/bin/ansible b/bin/ansible index 1e2540fafb7..7e767b2f7db 100755 --- a/bin/ansible +++ b/bin/ansible @@ -136,7 +136,7 @@ class Cli(object): if not options.ask_vault_pass: vault_pass = tmp_vault_pass - inventory_manager = inventory.Inventory(options.inventory) + inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass) if options.subset: inventory_manager.subset(options.subset) hosts = inventory_manager.list_hosts(pattern) diff --git a/bin/ansible-playbook b/bin/ansible-playbook index f54c17d7aa7..d91e2d94847 100755 --- a/bin/ansible-playbook +++ b/bin/ansible-playbook @@ -97,11 +97,6 @@ def main(args): if (options.ask_vault_pass and options.vault_password_file): parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive") - inventory = ansible.inventory.Inventory(options.inventory) - inventory.subset(options.subset) - if len(inventory.list_hosts()) == 0: - raise errors.AnsibleError("provided hosts list is empty") - sshpass = None sudopass = None su_pass = None @@ -155,6 +150,11 @@ def main(args): if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook) + inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass) + inventory.subset(options.subset) + if len(inventory.list_hosts()) == 0: + raise errors.AnsibleError("provided hosts list is empty") + # run all playbooks specified on the command line for playbook in args: diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index a8cca8faaf2..06acf87e484 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -39,13 +39,14 @@ class Inventory(object): __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list', - '_pattern_cache', '_vars_plugins', '_playbook_basedir'] + '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir'] - def __init__(self, host_list=C.DEFAULT_HOST_LIST): + def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None): # the host file file, or script path, or list of hosts # if a list, inventory data will NOT be loaded self.host_list = host_list + self._vault_password=vault_password # caching to avoid repeated calculations, particularly with # external inventory scripts. @@ -140,6 +141,14 @@ class Inventory(object): self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ] + # get group vars from vars plugins + for group in self.groups: + group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password)) + + # get host vars from vars plugins + for host in self.get_hosts(): + host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password)) + def _match(self, str, pattern_str): if pattern_str.startswith('~'): @@ -370,16 +379,25 @@ class Inventory(object): return group return None - def get_group_variables(self, groupname): + def get_group_variables(self, groupname, vault_password=None): if groupname not in self._vars_per_group: - self._vars_per_group[groupname] = self._get_group_variables(groupname) + self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password) return self._vars_per_group[groupname] - def _get_group_variables(self, groupname): + def _get_group_variables(self, groupname, vault_password=None): + group = self.get_group(groupname) if group is None: raise Exception("group not found: %s" % groupname) - return group.get_variables() + + vars = {} + vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')] + for updated in vars_results: + if updated is not None: + vars.update(updated) + + vars.update(group.get_variables()) + return vars def get_variables(self, hostname, vault_password=None): if hostname not in self._vars_per_host: @@ -393,14 +411,27 @@ class Inventory(object): raise errors.AnsibleError("host not found: %s" % hostname) vars = {} - vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ] + + # plugin.get_host_vars retrieves just vars for specific host + vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')] + for updated in vars_results: + if updated is not None: + vars = utils.combine_vars(vars, updated) + + # plugin.run retrieves all vars (also from groups) for host + vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')] for updated in vars_results: if updated is not None: vars = utils.combine_vars(vars, updated) vars = utils.combine_vars(vars, host.get_variables()) + + # still need to check InventoryParser per host vars + # which actually means InventoryScript per host, + # which is not performant if self.parser is not None: vars = utils.combine_vars(vars, self.parser.get_host_variables(host)) + return vars def add_group(self, group): diff --git a/lib/ansible/inventory/vars_plugins/group_vars.py b/lib/ansible/inventory/vars_plugins/group_vars.py index 93edceeecb5..2e8c7511220 100644 --- a/lib/ansible/inventory/vars_plugins/group_vars.py +++ b/lib/ansible/inventory/vars_plugins/group_vars.py @@ -143,28 +143,33 @@ class VarsModule(object): """ constructor """ self.inventory = inventory + self.inventory_basedir = inventory.basedir() + # There's no playbook initialized yet: + self.pb_basedir = None - def run(self, host, vault_password=None): - """ main body of the plugin, does actual loading """ + def get_host_vars(self, host, vault_password=None): + return self._get_vars(host=host, group=None, vault_password=vault_password) - inventory = self.inventory - basedir = inventory.playbook_basedir() - if basedir is not None: - basedir = os.path.abspath(basedir) - self.pb_basedir = basedir - # sort groups by depth so deepest groups can override the less deep ones - groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth) - groups = [ g.name for g in groupz ] - inventory_basedir = inventory.basedir() + def get_group_vars(self, group, vault_password=None): + return self._get_vars(host=None, group=group, vault_password=vault_password) + + + def _get_vars(self, host=None, group=None, vault_password=None): + """ main body of the plugin, does actual loading""" + + if self.pb_basedir is None: + pb_basedir = self.inventory.playbook_basedir() + if pb_basedir is not None: + pb_basedir = os.path.abspath(pb_basedir) + self.pb_basedir = pb_basedir results = {} scan_pass = 0 # look in both the inventory base directory and the playbook base directory - for basedir in [ inventory_basedir, self.pb_basedir ]: - + for basedir in [self.inventory_basedir, self.pb_basedir ]: # this can happen from particular API usages, particularly if not run # from /usr/bin/ansible-playbook @@ -178,17 +183,18 @@ class VarsModule(object): continue # save work of second scan if the directories are the same - if inventory_basedir == self.pb_basedir and scan_pass != 1: + if self.inventory_basedir == self.pb_basedir and scan_pass != 1: continue - # load vars in dir/group_vars/name_of_group - for group in groups: - base_path = os.path.join(basedir, "group_vars/%s" % group) + if group and host is None: + # load vars in dir/group_vars/name_of_group + base_path = os.path.join(basedir, "group_vars/%s" % group.name) results = _load_vars(base_path, results, vault_password=vault_password) - # same for hostvars in dir/host_vars/name_of_host - base_path = os.path.join(basedir, "host_vars/%s" % host.name) - results = _load_vars(base_path, results, vault_password=vault_password) + elif host and group is None: + # same for hostvars in dir/host_vars/name_of_host + base_path = os.path.join(basedir, "host_vars/%s" % host.name) + results = _load_vars(base_path, results, vault_password=vault_password) # all done, results is a dictionary of variables for this particular host. return results From cc28fd891bbc2a9458c12654959b736019f5ca95 Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Mon, 17 Mar 2014 18:20:20 +0100 Subject: [PATCH 2/5] Introduce noop vars plugin. In preparation to move the group_vars plugin code into core. new file: lib/ansible/inventory/vars_plugins/noop.py --- lib/ansible/inventory/vars_plugins/noop.py | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 lib/ansible/inventory/vars_plugins/noop.py diff --git a/lib/ansible/inventory/vars_plugins/noop.py b/lib/ansible/inventory/vars_plugins/noop.py new file mode 100644 index 00000000000..5d4b4b6658c --- /dev/null +++ b/lib/ansible/inventory/vars_plugins/noop.py @@ -0,0 +1,48 @@ +# (c) 2012-2014, Michael DeHaan +# (c) 2014, Serge van Ginderachter +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +class VarsModule(object): + + """ + Loads variables for groups and/or hosts + """ + + def __init__(self, inventory): + + """ constructor """ + + self.inventory = inventory + self.inventory_basedir = inventory.basedir() + + + def run(self, host, vault_password=None): + """ For backwards compatibility, when only vars per host were retrieved + This method should return both host specific vars as well as vars + calculated from groups it is a member of """ + return {} + + + def get_host_vars(self, host, vault_password=None): + """ Get host specific variables. """ + return {} + + + def get_group_vars(self, group, vault_password=None): + """ Get group specific variables. """ + return {} + From f8ea93c732df881fe051bcc2bc832af05c364fbb Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Wed, 19 Mar 2014 11:09:38 +0100 Subject: [PATCH 3/5] Move inventory.set_playbook_basedir from ansible-playbook to playbook constructor --- bin/ansible-playbook | 3 --- lib/ansible/inventory/__init__.py | 3 ++- lib/ansible/playbook/__init__.py | 4 ++++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/ansible-playbook b/bin/ansible-playbook index d91e2d94847..4a8edad5eff 100755 --- a/bin/ansible-playbook +++ b/bin/ansible-playbook @@ -158,9 +158,6 @@ def main(args): # run all playbooks specified on the command line for playbook in args: - # let inventory know which playbooks are using so it can know the basedirs - inventory.set_playbook_basedir(os.path.dirname(playbook)) - stats = callbacks.AggregateStats() playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY) if options.step: diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index 06acf87e484..53cc0af5d88 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -57,7 +57,8 @@ class Inventory(object): self._groups_list = {} self._pattern_cache = {} - # to be set by calling set_playbook_basedir by ansible-playbook + self._inventory_basedir = inventory.basedir() + # to be set by calling set_playbook_basedir by playbook code self._playbook_basedir = None # the inventory object holds a list of groups diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py index f624be1b297..8b07a52c0ae 100644 --- a/lib/ansible/playbook/__init__.py +++ b/lib/ansible/playbook/__init__.py @@ -159,6 +159,10 @@ class PlayBook(object): self.basedir = os.path.dirname(playbook) or '.' utils.plugins.push_basedir(self.basedir) + + # let inventory know the playbook basedir so it can load more vars + self.inventory.set_playbook_basedir(self.basedir) + vars = extra_vars.copy() vars['playbook_dir'] = self.basedir if self.inventory.basedir() is not None: From d4634983f0382a5d75cff68c6c44b7b67475a2c9 Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Fri, 21 Mar 2014 19:05:18 +0100 Subject: [PATCH 4/5] Move group/host_vars parsing into core inventory modified: lib/ansible/inventory/__init__.py deleted: lib/ansible/inventory/vars_plugins/group_vars.py modified: lib/ansible/utils/__init__.py --- lib/ansible/inventory/__init__.py | 111 ++++++++-- .../inventory/vars_plugins/group_vars.py | 201 ------------------ lib/ansible/utils/__init__.py | 110 +++++++++- 3 files changed, 201 insertions(+), 221 deletions(-) delete mode 100644 lib/ansible/inventory/vars_plugins/group_vars.py diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index 53cc0af5d88..5089759081d 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -57,7 +57,6 @@ class Inventory(object): self._groups_list = {} self._pattern_cache = {} - self._inventory_basedir = inventory.basedir() # to be set by calling set_playbook_basedir by playbook code self._playbook_basedir = None @@ -142,11 +141,11 @@ class Inventory(object): self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ] - # get group vars from vars plugins + # get group vars from group_vars/ files and vars plugins for group in self.groups: group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password)) - # get host vars from vars plugins + # get host vars from host_vars/ files and vars plugins for host in self.get_hosts(): host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password)) @@ -380,8 +379,8 @@ class Inventory(object): return group return None - def get_group_variables(self, groupname, vault_password=None): - if groupname not in self._vars_per_group: + def get_group_variables(self, groupname, update_cached=False, vault_password=None): + if groupname not in self._vars_per_group or update_cached: self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password) return self._vars_per_group[groupname] @@ -392,16 +391,23 @@ class Inventory(object): raise Exception("group not found: %s" % groupname) vars = {} + + # plugin.get_group_vars retrieves just vars for specific group vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')] for updated in vars_results: if updated is not None: - vars.update(updated) + vars = utils.combine_vars(vars, updated) + + # get group variables set by Inventory Parsers + vars = utils.combine_vars(vars, group.get_variables()) + + # Read group_vars/ files + vars = utils.combine_vars(vars, self.get_group_vars(group)) - vars.update(group.get_variables()) return vars - def get_variables(self, hostname, vault_password=None): - if hostname not in self._vars_per_host: + def get_variables(self, hostname, update_cached=False, vault_password=None): + if hostname not in self._vars_per_host or update_cached: self._vars_per_host[hostname] = self._get_variables(hostname, vault_password=vault_password) return self._vars_per_host[hostname] @@ -413,18 +419,19 @@ class Inventory(object): vars = {} - # plugin.get_host_vars retrieves just vars for specific host - vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')] - for updated in vars_results: - if updated is not None: - vars = utils.combine_vars(vars, updated) - # plugin.run retrieves all vars (also from groups) for host vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')] for updated in vars_results: if updated is not None: vars = utils.combine_vars(vars, updated) + # plugin.get_host_vars retrieves just vars for specific host + vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')] + for updated in vars_results: + if updated is not None: + vars = utils.combine_vars(vars, updated) + + # get host variables set by Inventory Parsers vars = utils.combine_vars(vars, host.get_variables()) # still need to check InventoryParser per host vars @@ -433,6 +440,9 @@ class Inventory(object): if self.parser is not None: vars = utils.combine_vars(vars, self.parser.get_host_variables(host)) + # Read host_vars/ files + vars = utils.combine_vars(vars, self.get_host_vars(host)) + return vars def add_group(self, group): @@ -532,10 +542,73 @@ class Inventory(object): return self._playbook_basedir def set_playbook_basedir(self, dir): - """ - sets the base directory of the playbook so inventory plugins can use it to find - variable files and other things. """ - self._playbook_basedir = dir + sets the base directory of the playbook so inventory can use it as a + basedir for host_ and group_vars, and other things. + """ + # Only update things if dir is a different playbook basedir + if dir != self._playbook_basedir: + self._playbook_basedir = dir + # get group vars from group_vars/ files + for group in self.groups: + group.vars = utils.combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True)) + # get host vars from host_vars/ files + for host in self.get_hosts(): + host.vars = utils.combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True)) + def get_host_vars(self, host, new_pb_basedir=False): + """ Read host_vars/ files """ + return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=False) + + def get_group_vars(self, group, new_pb_basedir=False): + """ Read group_vars/ files """ + return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=False) + + def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False): + """ + Loads variables from group_vars/ and host_vars/ in directories parallel + to the inventory base directory or in the same directory as the playbook. Variables in the playbook + dir will win over the inventory dir if files are in both. + """ + + results = {} + scan_pass = 0 + _basedir = self.basedir() + + # look in both the inventory base directory and the playbook base directory + # unless we do an update for a new playbook base dir + if not new_pb_basedir: + basedirs = [_basedir, self._playbook_basedir] + else: + basedirs = [self._playbook_basedir] + + for basedir in basedirs: + + # this can happen from particular API usages, particularly if not run + # from /usr/bin/ansible-playbook + if basedir is None: + continue + + scan_pass = scan_pass + 1 + + # it's not an eror if the directory does not exist, keep moving + if not os.path.exists(basedir): + continue + + # save work of second scan if the directories are the same + if _basedir == self._playbook_basedir and scan_pass != 1: + continue + + if group and host is None: + # load vars in dir/group_vars/name_of_group + base_path = os.path.join(basedir, "group_vars/%s" % group.name) + results = utils.load_vars(base_path, results, vault_password=self._vault_password) + + elif host and group is None: + # same for hostvars in dir/host_vars/name_of_host + base_path = os.path.join(basedir, "host_vars/%s" % host.name) + results = utils.load_vars(base_path, results, vault_password=self._vault_password) + + # all done, results is a dictionary of variables for this particular host. + return results diff --git a/lib/ansible/inventory/vars_plugins/group_vars.py b/lib/ansible/inventory/vars_plugins/group_vars.py deleted file mode 100644 index 2e8c7511220..00000000000 --- a/lib/ansible/inventory/vars_plugins/group_vars.py +++ /dev/null @@ -1,201 +0,0 @@ -# (c) 2012-2014, Michael DeHaan -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -import os -import stat -import errno - -from ansible import errors -from ansible import utils -import ansible.constants as C - -def _load_vars(basepath, results, vault_password=None): - """ - Load variables from any potential yaml filename combinations of basepath, - returning result. - """ - - paths_to_check = [ "".join([basepath, ext]) - for ext in C.YAML_FILENAME_EXTENSIONS ] - - found_paths = [] - - for path in paths_to_check: - found, results = _load_vars_from_path(path, results, vault_password=vault_password) - if found: - found_paths.append(path) - - - # disallow the potentially confusing situation that there are multiple - # variable files for the same name. For example if both group_vars/all.yml - # and group_vars/all.yaml - if len(found_paths) > 1: - raise errors.AnsibleError("Multiple variable files found. " - "There should only be one. %s" % ( found_paths, )) - - return results - -def _load_vars_from_path(path, results, vault_password=None): - """ - Robustly access the file at path and load variables, carefully reporting - errors in a friendly/informative way. - - Return the tuple (found, new_results, ) - """ - - try: - # in the case of a symbolic link, we want the stat of the link itself, - # not its target - pathstat = os.lstat(path) - except os.error, err: - # most common case is that nothing exists at that path. - if err.errno == errno.ENOENT: - return False, results - # otherwise this is a condition we should report to the user - raise errors.AnsibleError( - "%s is not accessible: %s." - " Please check its permissions." % ( path, err.strerror)) - - # symbolic link - if stat.S_ISLNK(pathstat.st_mode): - try: - target = os.path.realpath(path) - except os.error, err2: - raise errors.AnsibleError("The symbolic link at %s " - "is not readable: %s. Please check its permissions." - % (path, err2.strerror, )) - # follow symbolic link chains by recursing, so we repeat the same - # permissions checks above and provide useful errors. - return _load_vars_from_path(target, results) - - # directory - if stat.S_ISDIR(pathstat.st_mode): - - # support organizing variables across multiple files in a directory - return True, _load_vars_from_folder(path, results, vault_password=vault_password) - - # regular file - elif stat.S_ISREG(pathstat.st_mode): - data = utils.parse_yaml_from_file(path, vault_password=vault_password) - if type(data) != dict: - raise errors.AnsibleError( - "%s must be stored as a dictionary/hash" % path) - - # combine vars overrides by default but can be configured to do a - # hash merge in settings - results = utils.combine_vars(results, data) - return True, results - - # something else? could be a fifo, socket, device, etc. - else: - raise errors.AnsibleError("Expected a variable file or directory " - "but found a non-file object at path %s" % (path, )) - -def _load_vars_from_folder(folder_path, results, vault_password=None): - """ - Load all variables within a folder recursively. - """ - - # this function and _load_vars_from_path are mutually recursive - - try: - names = os.listdir(folder_path) - except os.error, err: - raise errors.AnsibleError( - "This folder cannot be listed: %s: %s." - % ( folder_path, err.strerror)) - - # evaluate files in a stable order rather than whatever order the - # filesystem lists them. - names.sort() - - # do not parse hidden files or dirs, e.g. .svn/ - paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')] - for path in paths: - _found, results = _load_vars_from_path(path, results, vault_password=vault_password) - return results - - -class VarsModule(object): - - """ - Loads variables from group_vars/ and host_vars/ in directories parallel - to the inventory base directory or in the same directory as the playbook. Variables in the playbook - dir will win over the inventory dir if files are in both. - """ - - def __init__(self, inventory): - - """ constructor """ - - self.inventory = inventory - self.inventory_basedir = inventory.basedir() - # There's no playbook initialized yet: - self.pb_basedir = None - - - def get_host_vars(self, host, vault_password=None): - return self._get_vars(host=host, group=None, vault_password=vault_password) - - - def get_group_vars(self, group, vault_password=None): - return self._get_vars(host=None, group=group, vault_password=vault_password) - - - def _get_vars(self, host=None, group=None, vault_password=None): - """ main body of the plugin, does actual loading""" - - if self.pb_basedir is None: - pb_basedir = self.inventory.playbook_basedir() - if pb_basedir is not None: - pb_basedir = os.path.abspath(pb_basedir) - self.pb_basedir = pb_basedir - - results = {} - scan_pass = 0 - - # look in both the inventory base directory and the playbook base directory - for basedir in [self.inventory_basedir, self.pb_basedir ]: - - # this can happen from particular API usages, particularly if not run - # from /usr/bin/ansible-playbook - if basedir is None: - continue - - scan_pass = scan_pass + 1 - - # it's not an eror if the directory does not exist, keep moving - if not os.path.exists(basedir): - continue - - # save work of second scan if the directories are the same - if self.inventory_basedir == self.pb_basedir and scan_pass != 1: - continue - - if group and host is None: - # load vars in dir/group_vars/name_of_group - base_path = os.path.join(basedir, "group_vars/%s" % group.name) - results = _load_vars(base_path, results, vault_password=vault_password) - - elif host and group is None: - # same for hostvars in dir/host_vars/name_of_host - base_path = os.path.join(basedir, "host_vars/%s" % host.name) - results = _load_vars(base_path, results, vault_password=vault_password) - - # all done, results is a dictionary of variables for this particular host. - return results - diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index 972089d6a3c..cbd33057468 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -1,4 +1,4 @@ -# (c) 2012, Michael DeHaan +# (c) 2012-2014, Michael DeHaan # # This file is part of Ansible # @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . +import errno import sys import re import os @@ -1108,5 +1109,112 @@ def before_comment(msg): msg = msg.replace("**NOT_A_COMMENT**","#") return msg +def load_vars(basepath, results, vault_password=None): + """ + Load variables from any potential yaml filename combinations of basepath, + returning result. + """ + + paths_to_check = [ "".join([basepath, ext]) + for ext in C.YAML_FILENAME_EXTENSIONS ] + + found_paths = [] + + for path in paths_to_check: + found, results = _load_vars_from_path(path, results, vault_password=vault_password) + if found: + found_paths.append(path) + # disallow the potentially confusing situation that there are multiple + # variable files for the same name. For example if both group_vars/all.yml + # and group_vars/all.yaml + if len(found_paths) > 1: + raise errors.AnsibleError("Multiple variable files found. " + "There should only be one. %s" % ( found_paths, )) + + return results + +## load variables from yaml files/dirs +# e.g. host/group_vars +# +def _load_vars_from_path(path, results, vault_password=None): + """ + Robustly access the file at path and load variables, carefully reporting + errors in a friendly/informative way. + + Return the tuple (found, new_results, ) + """ + + try: + # in the case of a symbolic link, we want the stat of the link itself, + # not its target + pathstat = os.lstat(path) + except os.error, err: + # most common case is that nothing exists at that path. + if err.errno == errno.ENOENT: + return False, results + # otherwise this is a condition we should report to the user + raise errors.AnsibleError( + "%s is not accessible: %s." + " Please check its permissions." % ( path, err.strerror)) + + # symbolic link + if stat.S_ISLNK(pathstat.st_mode): + try: + target = os.path.realpath(path) + except os.error, err2: + raise errors.AnsibleError("The symbolic link at %s " + "is not readable: %s. Please check its permissions." + % (path, err2.strerror, )) + # follow symbolic link chains by recursing, so we repeat the same + # permissions checks above and provide useful errors. + return _load_vars_from_path(target, results) + + # directory + if stat.S_ISDIR(pathstat.st_mode): + + # support organizing variables across multiple files in a directory + return True, _load_vars_from_folder(path, results, vault_password=vault_password) + + # regular file + elif stat.S_ISREG(pathstat.st_mode): + data = parse_yaml_from_file(path, vault_password=vault_password) + if type(data) != dict: + raise errors.AnsibleError( + "%s must be stored as a dictionary/hash" % path) + + # combine vars overrides by default but can be configured to do a + # hash merge in settings + results = combine_vars(results, data) + return True, results + + # something else? could be a fifo, socket, device, etc. + else: + raise errors.AnsibleError("Expected a variable file or directory " + "but found a non-file object at path %s" % (path, )) + +def _load_vars_from_folder(folder_path, results, vault_password=None): + """ + Load all variables within a folder recursively. + """ + + # this function and _load_vars_from_path are mutually recursive + + try: + names = os.listdir(folder_path) + except os.error, err: + raise errors.AnsibleError( + "This folder cannot be listed: %s: %s." + % ( folder_path, err.strerror)) + + # evaluate files in a stable order rather than whatever order the + # filesystem lists them. + names.sort() + + # do not parse hidden files or dirs, e.g. .svn/ + paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')] + for path in paths: + _found, results = _load_vars_from_path(path, results, vault_password=vault_password) + return results + From b0ff1ea425585ec16585ec6a7d2e676ed3c45430 Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Tue, 25 Mar 2014 21:59:13 +0100 Subject: [PATCH 5/5] performance optimisation in hash merge logic rewrite deepcopy in util.merge_hash and just iterate on an inventory with 500 groups and 800 hosts this brings back the inventory initialisation from 13s to 3s (with hash_behaviour=merge) --- lib/ansible/utils/__init__.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index cbd33057468..9640ec8ed66 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -558,18 +558,19 @@ def merge_hash(a, b): ''' recursively merges hash b into a keys from b take precedence over keys from a ''' - result = copy.deepcopy(a) + result = {} - # next, iterate over b keys and values - for k, v in b.iteritems(): - # if there's already such key in a - # and that key contains dict - if k in result and isinstance(result[k], dict): - # merge those dicts recursively - result[k] = merge_hash(a[k], v) - else: - # otherwise, just copy a value from b to a - result[k] = v + for dicts in a, b: + # next, iterate over b keys and values + for k, v in dicts.iteritems(): + # if there's already such key in a + # and that key contains dict + if k in result and isinstance(result[k], dict): + # merge those dicts recursively + result[k] = merge_hash(a[k], v) + else: + # otherwise, just copy a value from b to a + result[k] = v return result