diff --git a/bin/ansible b/bin/ansible index f254eaf3ff9..844004ecc4d 100755 --- a/bin/ansible +++ b/bin/ansible @@ -98,11 +98,11 @@ class Cli(object): # ---------------------------------------------- - def get_polling_runner(self, old_runner, hosts, jid): + def get_polling_runner(self, old_runner, jid): return ansible.runner.Runner( module_name='async_status', module_path=old_runner.module_path, module_args="jid=%s" % jid, remote_user=old_runner.remote_user, - remote_pass=old_runner.remote_pass, host_list=hosts, + remote_pass=old_runner.remote_pass, inventory=old_runner.inventory, timeout=old_runner.timeout, forks=old_runner.forks, remote_port=old_runner.remote_port, pattern='*', callbacks=self.silent_callbacks, verbose=True, @@ -138,8 +138,10 @@ class Cli(object): clock = options.seconds while (clock >= 0): - polling_runner = self.get_polling_runner(runner, poll_hosts, jid) + runner.inventory.restrict_to(poll_hosts) + polling_runner = self.get_polling_runner(runner, jid) poll_results = polling_runner.run() + runner.inventory.lift_restrictions() if poll_results is None: break for (host, host_result) in poll_results['contacted'].iteritems(): diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py new file mode 100644 index 00000000000..448ef372b04 --- /dev/null +++ b/lib/ansible/inventory.py @@ -0,0 +1,189 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +############################################# + +import fnmatch +import os +import subprocess + +import constants as C +from ansible import errors +from ansible import utils + +class Inventory(object): + """ Host inventory for ansible. + + The inventory is either a simple text file with systems and [groups] of + systems, or a script that will be called with --list or --host. + """ + + def __init__(self, host_list=C.DEFAULT_HOST_LIST, extra_vars=None): + + self._restriction = None + + if type(host_list) == list: + self.host_list = host_list + self.groups = dict(ungrouped=host_list) + self._is_script = False + return + + inventory_file = os.path.expanduser(host_list) + if not os.path.exists(inventory_file): + raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list) + + self.inventory_file = os.path.abspath(inventory_file) + + if os.access(self.inventory_file, os.X_OK): + self.host_list, self.groups = self._parse_from_script(extra_vars) + self._is_script = True + else: + self.host_list, self.groups = self._parse_from_file() + self._is_script = False + + # ***************************************************** + # Public API + + def list_hosts(self, pattern="all"): + """ Return a list of hosts [matching the pattern] """ + if self._restriction is None: + host_list = self.host_list + else: + host_list = [ h for h in self.host_list if h in self._restriction ] + return [ h for h in host_list if self._matches(h, pattern) ] + + def restrict_to(self, restriction): + """ Restrict list operations to the hosts given in restriction """ + if type(restriction)!=list: + restriction = [ restriction ] + + self._restriction = restriction + + def lift_restriction(self): + """ Do not restrict list operations """ + self._restriction = None + + def get_variables(self, host, extra_vars=None): + """ Return the variables associated with this host. """ + + if not self._is_script: + return {} + + return self._get_variables_from_script(host, extra_vars) + + # ***************************************************** + + def _parse_from_file(self): + ''' parse a textual host file ''' + + results = [] + groups = dict(ungrouped=[]) + lines = file(self.inventory_file).read().split("\n") + group_name = 'ungrouped' + for item in lines: + item = item.lstrip().rstrip() + if item.startswith("#"): + # ignore commented out lines + pass + elif item.startswith("["): + # looks like a group + group_name = item.replace("[","").replace("]","").lstrip().rstrip() + groups[group_name] = [] + elif item != "": + # looks like a regular host + groups[group_name].append(item) + if not item in results: + results.append(item) + return (results, groups) + + # ***************************************************** + + def _parse_from_script(self, extra_vars=None): + ''' evaluate a script that returns list of hosts by groups ''' + + results = [] + groups = dict(ungrouped=[]) + + cmd = [self.inventory_file, '--list'] + + if extra_vars: + cmd.extend(['--extra-vars', extra_vars]) + + cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + out, err = cmd.communicate() + rc = cmd.returncode + if rc: + raise errors.AnsibleError("%s: %s" % self.inventory_file, err) + + try: + groups = utils.json_loads(out) + except: + raise errors.AnsibleError("invalid JSON response from script: %s" % self.inventory_file) + + for (groupname, hostlist) in groups.iteritems(): + for host in hostlist: + if host not in results: + results.append(host) + return (results, groups) + + # ***************************************************** + + def _get_variables_from_script(self, host, extra_vars=None): + ''' support per system variabes from external variable scripts, see web docs ''' + + cmd = [self.inventory_file, '--host', host] + + if extra_vars: + cmd.extend(['--extra-vars', extra_vars]) + + cmd = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False + ) + out, err = cmd.communicate() + + variables = {} + try: + variables = utils.json_loads(out) + except: + raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % ( + self.inventory_file, + host + )) + return variables + + def _matches(self, host_name, pattern): + ''' returns if a hostname is matched by the pattern ''' + + # a pattern is in fnmatch format but more than one pattern + # can be strung together with semicolons. ex: + # atlanta-web*.example.com;dc-web*.example.com + + if host_name == '': + return False + pattern = pattern.replace(";",":") + subpatterns = pattern.split(":") + for subpattern in subpatterns: + if subpattern == 'all': + return True + if fnmatch.fnmatch(host_name, subpattern): + return True + elif subpattern in self.groups: + if host_name in self.groups[subpattern]: + return True + return False diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py index ba5115d02d9..c3e195e0409 100755 --- a/lib/ansible/playbook.py +++ b/lib/ansible/playbook.py @@ -17,6 +17,7 @@ ############################################# +import ansible.inventory import ansible.runner import ansible.constants as C from ansible import utils @@ -68,7 +69,6 @@ class PlayBook(object): if playbook is None or callbacks is None or runner_callbacks is None or stats is None: raise Exception('missing required arguments') - self.host_list = host_list self.module_path = module_path self.forks = forks self.timeout = timeout @@ -88,9 +88,13 @@ class PlayBook(object): self.basedir = os.path.dirname(playbook) self.playbook = self._parse_playbook(playbook) - self.host_list, self.groups = ansible.runner.Runner.parse_hosts( - host_list, override_hosts=self.override_hosts, extra_vars=self.extra_vars) - + if override_hosts is not None: + if type(override_hosts) != list: + raise errors.AnsibleError("override hosts must be a list") + self.inventory = ansible.inventory.Inventory(override_hosts) + else: + self.inventory = ansible.inventory.Inventory(host_list) + # ***************************************************** def _get_vars(self, play, dirname): @@ -233,7 +237,6 @@ class PlayBook(object): def _async_poll(self, runner, hosts, async_seconds, async_poll_interval, only_if): ''' launch an async job, if poll_interval is set, wait for completion ''' - runner.host_list = hosts runner.background = async_seconds results = runner.run() self.stats.compute(results, poll=True) @@ -257,7 +260,7 @@ class PlayBook(object): return results clock = async_seconds - runner.host_list = self.hosts_to_poll(results) + host_list = self.hosts_to_poll(results) poll_results = results while (clock >= 0): @@ -267,11 +270,13 @@ class PlayBook(object): runner.module_name = 'async_status' runner.background = 0 runner.pattern = '*' + self.inventory.restrict_to(host_list) poll_results = runner.run() self.stats.compute(poll_results, poll=True) - runner.host_list = self.hosts_to_poll(poll_results) + host_list = self.hosts_to_poll(poll_results) + self.inventory.lift_restriction() - if len(runner.host_list) == 0: + if len(host_list) == 0: break if poll_results is None: break @@ -298,15 +303,16 @@ class PlayBook(object): # ***************************************************** - def _run_module(self, pattern, host_list, module, args, vars, remote_user, + def _run_module(self, pattern, module, args, vars, remote_user, async_seconds, async_poll_interval, only_if, sudo, transport): ''' run a particular module step in a playbook ''' - hosts = [ h for h in host_list if (h not in self.stats.failures) and (h not in self.stats.dark)] + hosts = [ h for h in self.inventory.list_hosts() if (h not in self.stats.failures) and (h not in self.stats.dark)] + self.inventory.restrict_to(hosts) runner = ansible.runner.Runner( - pattern=pattern, groups=self.groups, module_name=module, - module_args=args, host_list=hosts, forks=self.forks, + pattern=pattern, inventory=self.inventory, module_name=module, + module_args=args, forks=self.forks, remote_pass=self.remote_pass, module_path=self.module_path, timeout=self.timeout, remote_user=remote_user, remote_port=self.remote_port, module_vars=vars, @@ -317,13 +323,16 @@ class PlayBook(object): ) if async_seconds == 0: - return runner.run() + results = runner.run() else: - return self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if) + results = self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if) + + self.inventory.lift_restriction() + return results # ***************************************************** - def _run_task(self, pattern=None, host_list=None, task=None, + def _run_task(self, pattern=None, task=None, remote_user=None, handlers=None, conditional=False, sudo=False, transport=None): ''' run a single task in the playbook and recursively run any subtasks. ''' @@ -354,7 +363,7 @@ class PlayBook(object): # load up an appropriate ansible runner to # run the task in parallel - results = self._run_module(pattern, host_list, module_name, + results = self._run_module(pattern, module_name, module_args, module_vars, remote_user, async_seconds, async_poll_interval, only_if, sudo, transport) @@ -406,7 +415,7 @@ class PlayBook(object): # ***************************************************** - def _do_conditional_imports(self, vars_files, host_list): + def _do_conditional_imports(self, vars_files): ''' handle the vars_files section, which can contain variables ''' # FIXME: save parsed variable results in memory to avoid excessive re-reading/parsing @@ -417,7 +426,7 @@ class PlayBook(object): if type(vars_files) != list: raise errors.AnsibleError("vars_files must be a list") - for host in host_list: + for host in self.inventory.list_hosts(): cache_vars = SETUP_CACHE.get(host,{}) SETUP_CACHE[host] = cache_vars for filename in vars_files: @@ -460,16 +469,18 @@ class PlayBook(object): if vars_files is not None: self.callbacks.on_setup_secondary() - self._do_conditional_imports(vars_files, self.host_list) + self._do_conditional_imports(vars_files) else: self.callbacks.on_setup_primary() - host_list = [ h for h in self.host_list if not (h in self.stats.failures or h in self.stats.dark) ] + host_list = [ h for h in self.inventory.list_hosts(pattern) + if not (h in self.stats.failures or h in self.stats.dark) ] + self.inventory.restrict_to(host_list) # push any variables down to the system setup_results = ansible.runner.Runner( - pattern=pattern, groups=self.groups, module_name='setup', - module_args=vars, host_list=host_list, + pattern=pattern, module_name='setup', + module_args=vars, inventory=self.inventory, forks=self.forks, module_path=self.module_path, timeout=self.timeout, remote_user=user, remote_pass=self.remote_pass, remote_port=self.remote_port, @@ -479,6 +490,8 @@ class PlayBook(object): ).run() self.stats.compute(setup_results, setup=True) + self.inventory.lift_restriction() + # now for each result, load into the setup cache so we can # let runner template out future commands setup_ok = setup_results.get('contacted', {}) @@ -494,7 +507,6 @@ class PlayBook(object): SETUP_CACHE[h].update(extra_vars) except: SETUP_CACHE[h] = extra_vars - return host_list # ***************************************************** @@ -530,7 +542,6 @@ class PlayBook(object): for task in tasks: self._run_task( pattern=pattern, - host_list=self.host_list, task=task, handlers=handlers, remote_user=user, @@ -547,16 +558,17 @@ class PlayBook(object): for task in handlers: triggered_by = task.get('run', None) if type(triggered_by) == list: + self.inventory.restrict_to(triggered_by) self._run_task( pattern=pattern, task=task, handlers=[], - host_list=triggered_by, conditional=True, remote_user=user, sudo=sudo, transport=transport ) + self.inventory.lift_restriction() # end of execution for this particular pattern. Multiple patterns # can be in a single playbook file diff --git a/lib/ansible/runner.py b/lib/ansible/runner.py index 418a069ba8c..50c4029a45d 100755 --- a/lib/ansible/runner.py +++ b/lib/ansible/runner.py @@ -18,7 +18,6 @@ ################################################ -import fnmatch import multiprocessing import signal import os @@ -27,10 +26,10 @@ import Queue import random import traceback import tempfile -import subprocess import ansible.constants as C import ansible.connection +import ansible.inventory from ansible import utils from ansible import errors from ansible import callbacks as ans_callbacks @@ -67,8 +66,6 @@ def _executor_hook(job_queue, result_queue): class Runner(object): - _external_variable_script = None - def __init__(self, host_list=C.DEFAULT_HOST_LIST, module_path=C.DEFAULT_MODULE_PATH, module_name=C.DEFAULT_MODULE_NAME, module_args=C.DEFAULT_MODULE_ARGS, forks=C.DEFAULT_FORKS, timeout=C.DEFAULT_TIMEOUT, pattern=C.DEFAULT_PATTERN, @@ -76,7 +73,7 @@ class Runner(object): sudo_pass=C.DEFAULT_SUDO_PASS, remote_port=C.DEFAULT_REMOTE_PORT, background=0, basedir=None, setup_cache=None, transport=C.DEFAULT_TRANSPORT, conditional='True', groups={}, callbacks=None, verbose=False, - debug=False, sudo=False, extra_vars=None, module_vars=None): + debug=False, sudo=False, extra_vars=None, module_vars=None, inventory=None): if setup_cache is None: setup_cache = {} @@ -92,11 +89,10 @@ class Runner(object): self.transport = transport self.connector = ansible.connection.Connection(self, self.transport) - if type(host_list) == str: - self.host_list, self.groups = self.parse_hosts(host_list) + if inventory is None: + self.inventory = ansible.inventory.Inventory(host_list, extra_vars) else: - self.host_list = host_list - self.groups = groups + self.inventory = inventory self.setup_cache = setup_cache self.conditional = conditional @@ -130,106 +126,6 @@ class Runner(object): # ***************************************************** - @classmethod - def parse_hosts_from_regular_file(cls, host_list): - ''' parse a textual host file ''' - - results = [] - groups = dict(ungrouped=[]) - lines = file(host_list).read().split("\n") - group_name = 'ungrouped' - for item in lines: - item = item.lstrip().rstrip() - if item.startswith("#"): - # ignore commented out lines - pass - elif item.startswith("["): - # looks like a group - group_name = item.replace("[","").replace("]","").lstrip().rstrip() - groups[group_name] = [] - elif item != "": - # looks like a regular host - groups[group_name].append(item) - if not item in results: - results.append(item) - return (results, groups) - - # ***************************************************** - - @classmethod - def parse_hosts_from_script(cls, host_list, extra_vars): - ''' evaluate a script that returns list of hosts by groups ''' - - results = [] - groups = dict(ungrouped=[]) - host_list = os.path.abspath(host_list) - cls._external_variable_script = host_list - cmd = [host_list, '--list'] - if extra_vars: - cmd.extend(['--extra-vars', extra_vars]) - cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - out, err = cmd.communicate() - rc = cmd.returncode - if rc: - raise errors.AnsibleError("%s: %s" % (host_list, err)) - try: - groups = utils.json_loads(out) - except: - raise errors.AnsibleError("invalid JSON response from script: %s" % host_list) - for (groupname, hostlist) in groups.iteritems(): - for host in hostlist: - if host not in results: - results.append(host) - return (results, groups) - - # ***************************************************** - - @classmethod - def parse_hosts(cls, host_list, override_hosts=None, extra_vars=None): - ''' parse the host inventory file, returns (hosts, groups) ''' - - if override_hosts is not None: - if type(override_hosts) != list: - raise errors.AnsibleError("override hosts must be a list") - return (override_hosts, dict(ungrouped=override_hosts)) - - if type(host_list) == list: - raise Exception("function can only be called on inventory files") - - host_list = os.path.expanduser(host_list) - if not os.path.exists(host_list): - raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list) - - if not os.access(host_list, os.X_OK): - return Runner.parse_hosts_from_regular_file(host_list) - else: - return Runner.parse_hosts_from_script(host_list, extra_vars) - - # ***************************************************** - - def _matches(self, host_name, pattern): - ''' returns if a hostname is matched by the pattern ''' - - # a pattern is in fnmatch format but more than one pattern - # can be strung together with semicolons. ex: - # atlanta-web*.example.com;dc-web*.example.com - - if host_name == '': - return False - pattern = pattern.replace(";",":") - subpatterns = pattern.split(":") - for subpattern in subpatterns: - if subpattern == 'all': - return True - if fnmatch.fnmatch(host_name, subpattern): - return True - elif subpattern in self.groups: - if host_name in self.groups[subpattern]: - return True - return False - - # ***************************************************** - def _connect(self, host): ''' connects to a host, returns (is_successful, connection_object OR traceback_string) ''' @@ -296,34 +192,6 @@ class Runner(object): # ***************************************************** - def _add_variables_from_script(self, conn, inject): - ''' support per system variabes from external variable scripts, see web docs ''' - - host = conn.host - - cmd = [Runner._external_variable_script, '--host', host] - if self.extra_vars: - cmd.extend(['--extra-vars', self.extra_vars]) - - cmd = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False - ) - out, err = cmd.communicate() - inject2 = {} - try: - inject2 = utils.json_loads(out) - except: - raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % ( - Runner._external_variable_script, - host - )) - # store injected variables in the templates - inject.update(inject2) - - # ***************************************************** - def _add_setup_vars(self, inject, args): ''' setup module variables need special handling ''' @@ -377,8 +245,9 @@ class Runner(object): if not eval(conditional): return [ utils.smjson(dict(skipped=True)), None, 'skipped' ] - if Runner._external_variable_script is not None: - self._add_variables_from_script(conn, inject) + host_variables = self.inventory.get_variables(conn.host, self.extra_vars) + inject.update(host_variables) + if self.module_name == 'setup': args = self._add_setup_vars(inject, args) args = self._add_setup_metadata(args) @@ -692,13 +561,6 @@ class Runner(object): # ***************************************************** - def _match_hosts(self, pattern): - ''' return all matched hosts fitting a pattern ''' - - return [ h for h in self.host_list if self._matches(h, pattern) ] - - # ***************************************************** - def _parallel_exec(self, hosts): ''' handles mulitprocessing when more than 1 fork is required ''' @@ -745,7 +607,7 @@ class Runner(object): results2["dark"][host] = result # hosts which were contacted but never got a chance to return - for host in self._match_hosts(self.pattern): + for host in self.inventory.list_hosts(self.pattern): if not (host in results2['dark'] or host in results2['contacted']): results2["dark"][host] = {} @@ -757,7 +619,7 @@ class Runner(object): ''' xfer & run module on all matched hosts ''' # find hosts that match the pattern - hosts = self._match_hosts(self.pattern) + hosts = self.inventory.list_hosts(self.pattern) if len(hosts) == 0: self.callbacks.on_no_hosts() return dict(contacted={}, dark={})