From 02bc014bcd6f43c1be331e67523ae1929aca4dfe Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Thu, 15 Jan 2015 01:13:45 -0600 Subject: [PATCH] More work on getting integration tests running for v2 --- v2/ansible/executor/process/result.py | 52 +++-- v2/ansible/executor/process/worker.py | 4 +- v2/ansible/executor/task_executor.py | 30 ++- v2/ansible/module_utils/ec2.py | 10 +- v2/ansible/module_utils/facts.py | 233 ++++++++++++++-------- v2/ansible/module_utils/gce.py | 10 +- v2/ansible/module_utils/known_hosts.py | 2 +- v2/ansible/module_utils/urls.py | 24 +++ v2/ansible/parsing/mod_args.py | 48 ++++- v2/ansible/parsing/splitter.py | 15 +- v2/ansible/playbook/conditional.py | 10 +- v2/ansible/playbook/task.py | 5 - v2/ansible/plugins/action/__init__.py | 14 +- v2/ansible/plugins/action/assert.py | 6 +- v2/ansible/plugins/action/script.py | 10 +- v2/ansible/plugins/action/set_fact.py | 9 +- v2/ansible/plugins/callback/default.py | 25 ++- v2/ansible/plugins/connections/local.py | 3 +- v2/ansible/plugins/lookup/first_found.py | 33 +-- v2/ansible/plugins/lookup/items.py | 4 + v2/ansible/plugins/lookup/sequence.py | 6 +- v2/ansible/plugins/lookup/together.py | 2 +- v2/ansible/plugins/strategies/__init__.py | 15 +- v2/ansible/plugins/strategies/free.py | 5 + v2/ansible/plugins/strategies/linear.py | 21 +- v2/ansible/utils/listify.py | 1 - v2/ansible/vars/__init__.py | 4 +- 27 files changed, 414 insertions(+), 187 deletions(-) diff --git a/v2/ansible/executor/process/result.py b/v2/ansible/executor/process/result.py index 3284fe05347..52a9abadc4b 100644 --- a/v2/ansible/executor/process/result.py +++ b/v2/ansible/executor/process/result.py @@ -109,15 +109,18 @@ class ResultProcess(multiprocessing.Process): host_name = result._host.get_name() # send callbacks, execute other options based on the result status - if result.is_failed(): - self._send_result(('host_task_failed', result)) - elif result.is_unreachable(): + # FIXME: this should all be cleaned up and probably moved to a sub-function. + # the fact that this sometimes sends a TaskResult and other times + # sends a raw dictionary back may be confusing, but the result vs. + # results implementation for tasks with loops should be cleaned up + # better than this + if result.is_unreachable(): self._send_result(('host_unreachable', result)) + elif result.is_failed(): + self._send_result(('host_task_failed', result)) elif result.is_skipped(): self._send_result(('host_task_skipped', result)) else: - self._send_result(('host_task_ok', result)) - # if this task is notifying a handler, do it now if result._task.notify: # The shared dictionary for notified handlers is a proxy, which @@ -125,21 +128,32 @@ class ResultProcess(multiprocessing.Process): # So, per the docs, we reassign the list so the proxy picks up and # notifies all other threads for notify in result._task.notify: - self._send_result(('notify_handler', notify, result._host)) + self._send_result(('notify_handler', result._host, notify)) - if 'add_host' in result._result: - # this task added a new host (add_host module) - self._send_result(('add_host', result)) - elif 'add_group' in result._result: - # this task added a new group (group_by module) - self._send_result(('add_group', result)) - elif 'ansible_facts' in result._result: - # if this task is registering facts, do that now - if result._task.action in ('set_fact', 'include_vars'): - for (key, value) in result._result['ansible_facts'].iteritems(): - self._send_result(('set_host_var', result._host, key, value)) - else: - self._send_result(('set_host_facts', result._host, result._result['ansible_facts'])) + if 'results' in result._result: + # this task had a loop, and has more than one result, so + # loop over all of them instead of a single result + result_items = result._result['results'] + else: + result_items = [ result._result ] + + for result_item in result_items: + if 'add_host' in result_item: + # this task added a new host (add_host module) + self._send_result(('add_host', result_item)) + elif 'add_group' in result_item: + # this task added a new group (group_by module) + self._send_result(('add_group', result._host, result_item)) + elif 'ansible_facts' in result_item: + # if this task is registering facts, do that now + if result._task.action in ('set_fact', 'include_vars'): + for (key, value) in result_item['ansible_facts'].iteritems(): + self._send_result(('set_host_var', result._host, key, value)) + else: + self._send_result(('set_host_facts', result._host, result_item['ansible_facts'])) + + # finally, send the ok for this task + self._send_result(('host_task_ok', result)) # if this task is registering a result, do it now if result._task.register: diff --git a/v2/ansible/executor/process/worker.py b/v2/ansible/executor/process/worker.py index 7831f51ba5f..bf5ee8c93f0 100644 --- a/v2/ansible/executor/process/worker.py +++ b/v2/ansible/executor/process/worker.py @@ -97,7 +97,7 @@ class WorkerProcess(multiprocessing.Process): try: if not self._main_q.empty(): debug("there's work to be done!") - (host, task, basedir, job_vars, connection_info) = self._main_q.get(block=False) + (host, task, basedir, job_vars, connection_info, module_loader) = self._main_q.get(block=False) debug("got a task/handler to work on: %s" % task) # because the task queue manager starts workers (forks) before the @@ -118,7 +118,7 @@ class WorkerProcess(multiprocessing.Process): # execute the task and build a TaskResult from the result debug("running TaskExecutor() for %s/%s" % (host, task)) - executor_result = TaskExecutor(host, task, job_vars, new_connection_info, self._loader).run() + executor_result = TaskExecutor(host, task, job_vars, new_connection_info, self._loader, module_loader).run() debug("done running TaskExecutor() for %s/%s" % (host, task)) task_result = TaskResult(host, task, executor_result) diff --git a/v2/ansible/executor/task_executor.py b/v2/ansible/executor/task_executor.py index 71dc399a477..d82060a5107 100644 --- a/v2/ansible/executor/task_executor.py +++ b/v2/ansible/executor/task_executor.py @@ -22,8 +22,10 @@ __metaclass__ = type from ansible import constants as C from ansible.errors import AnsibleError, AnsibleParserError from ansible.executor.connection_info import ConnectionInformation +from ansible.playbook.conditional import Conditional from ansible.playbook.task import Task from ansible.plugins import lookup_loader, connection_loader, action_loader +from ansible.utils.listify import listify_lookup_plugin_terms from ansible.utils.debug import debug @@ -41,12 +43,13 @@ class TaskExecutor: class. ''' - def __init__(self, host, task, job_vars, connection_info, loader): + def __init__(self, host, task, job_vars, connection_info, loader, module_loader): self._host = host self._task = task self._job_vars = job_vars self._connection_info = connection_info self._loader = loader + self._module_loader = module_loader def run(self): ''' @@ -57,6 +60,13 @@ class TaskExecutor: debug("in run()") try: + # lookup plugins need to know if this task is executing from + # a role, so that it can properly find files/templates/etc. + roledir = None + if self._task._role: + roledir = self._task._role._role_path + self._job_vars['roledir'] = roledir + items = self._get_loop_items() if items is not None: if len(items) > 0: @@ -84,7 +94,8 @@ class TaskExecutor: items = None if self._task.loop and self._task.loop in lookup_loader: - items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=self._task.loop_args, variables=self._job_vars) + loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, variables=self._job_vars, loader=self._loader) + items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=loop_terms, variables=self._job_vars) return items @@ -184,11 +195,10 @@ class TaskExecutor: # now update them with the registered value, if it is set if self._task.register: vars_copy[self._task.register] = result - # now create a pseudo task, and assign the value of the until parameter - # to the when param, so we can use evaluate_conditional() - pseudo_task = Task() - pseudo_task.when = self._task.until - if pseudo_task.evaluate_conditional(vars_copy): + # create a conditional object to evaluate the until condition + cond = Conditional(loader=self._loader) + cond.when = self._task.until + if cond.evaluate_conditional(vars_copy): break elif 'failed' not in result and result.get('rc', 0) == 0: # if the result is not failed, stop trying @@ -223,7 +233,8 @@ class TaskExecutor: task=async_task, connection=self._connection, connection_info=self._connection_info, - loader=self._loader + loader=self._loader, + module_loader=self._module_loader, ) time_left = self._task.async @@ -283,7 +294,8 @@ class TaskExecutor: task=self._task, connection=connection, connection_info=self._connection_info, - loader=self._loader + loader=self._loader, + module_loader=self._module_loader, ) if not handler: raise AnsibleError("the handler '%s' was not found" % handler_name) diff --git a/v2/ansible/module_utils/ec2.py b/v2/ansible/module_utils/ec2.py index 417e1b9521b..0f08fead180 100644 --- a/v2/ansible/module_utils/ec2.py +++ b/v2/ansible/module_utils/ec2.py @@ -39,6 +39,7 @@ AWS_REGIONS = [ 'cn-north-1', 'eu-central-1', 'eu-west-1', + 'eu-central-1', 'sa-east-1', 'us-east-1', 'us-west-1', @@ -165,6 +166,11 @@ def boto_fix_security_token_in_profile(conn, profile_name): def connect_to_aws(aws_module, region, **params): conn = aws_module.connect_to_region(region, **params) + if not conn: + if region not in [aws_module_region.name for aws_module_region in aws_module.regions()]: + raise StandardError("Region %s does not seem to be available for aws module %s. If the region definitely exists, you may need to upgrade boto" % (region, aws_module.__name__)) + else: + raise StandardError("Unknown problem connecting to region %s for aws module %s." % (region, aws_module.__name__)) if params.get('profile_name'): conn = boto_fix_security_token_in_profile(conn, params['profile_name']) return conn @@ -180,13 +186,13 @@ def ec2_connect(module): if region: try: ec2 = connect_to_aws(boto.ec2, region, **boto_params) - except boto.exception.NoAuthHandlerFound, e: + except (boto.exception.NoAuthHandlerFound, StandardError), e: module.fail_json(msg=str(e)) # Otherwise, no region so we fallback to the old connection method elif ec2_url: try: ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params) - except boto.exception.NoAuthHandlerFound, e: + except (boto.exception.NoAuthHandlerFound, StandardError), e: module.fail_json(msg=str(e)) else: module.fail_json(msg="Either region or ec2_url must be specified") diff --git a/v2/ansible/module_utils/facts.py b/v2/ansible/module_utils/facts.py index 5ceeb405d55..be6939259a9 100644 --- a/v2/ansible/module_utils/facts.py +++ b/v2/ansible/module_utils/facts.py @@ -46,7 +46,7 @@ except ImportError: import simplejson as json # -------------------------------------------------------------- -# timeout function to make sure some fact gathering +# timeout function to make sure some fact gathering # steps do not exceed a time limit class TimeoutError(Exception): @@ -82,7 +82,8 @@ class Facts(object): subclass Facts. """ - _I386RE = re.compile(r'i[3456]86') + # i86pc is a Solaris and derivatives-ism + _I386RE = re.compile(r'i([3456]86|86pc)') # For the most part, we assume that platform.dist() will tell the truth. # This is the fallback to handle unknowns or exceptions OSDIST_LIST = ( ('/etc/redhat-release', 'RedHat'), @@ -274,84 +275,115 @@ class Facts(object): self.facts['distribution_release'] = dist[2] or 'NA' # Try to handle the exceptions now ... for (path, name) in Facts.OSDIST_LIST: - if os.path.exists(path) and os.path.getsize(path) > 0: - if self.facts['distribution'] in ('Fedora', ): - # Once we determine the value is one of these distros - # we trust the values are always correct - break - elif name == 'RedHat': - data = get_file_content(path) - if 'Red Hat' in data: + if os.path.exists(path): + if os.path.getsize(path) > 0: + if self.facts['distribution'] in ('Fedora', ): + # Once we determine the value is one of these distros + # we trust the values are always correct + break + elif name == 'RedHat': + data = get_file_content(path) + if 'Red Hat' in data: + self.facts['distribution'] = name + else: + self.facts['distribution'] = data.split()[0] + break + elif name == 'OtherLinux': + data = get_file_content(path) + if 'Amazon' in data: + self.facts['distribution'] = 'Amazon' + self.facts['distribution_version'] = data.split()[-1] + break + elif name == 'OpenWrt': + data = get_file_content(path) + if 'OpenWrt' in data: + self.facts['distribution'] = name + version = re.search('DISTRIB_RELEASE="(.*)"', data) + if version: + self.facts['distribution_version'] = version.groups()[0] + release = re.search('DISTRIB_CODENAME="(.*)"', data) + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif name == 'Alpine': + data = get_file_content(path) self.facts['distribution'] = name - else: - self.facts['distribution'] = data.split()[0] - break - elif name == 'OtherLinux': - data = get_file_content(path) - if 'Amazon' in data: - self.facts['distribution'] = 'Amazon' - self.facts['distribution_version'] = data.split()[-1] + self.facts['distribution_version'] = data break - elif name == 'OpenWrt': - data = get_file_content(path) - if 'OpenWrt' in data: - self.facts['distribution'] = name - version = re.search('DISTRIB_RELEASE="(.*)"', data) - if version: - self.facts['distribution_version'] = version.groups()[0] - release = re.search('DISTRIB_CODENAME="(.*)"', data) - if release: - self.facts['distribution_release'] = release.groups()[0] - break - elif name == 'Alpine': - data = get_file_content(path) - self.facts['distribution'] = name - self.facts['distribution_version'] = data - break - elif name == 'Solaris': - data = get_file_content(path).split('\n')[0] - if 'Solaris' in data: - ora_prefix = '' - if 'Oracle Solaris' in data: - data = data.replace('Oracle ','') - ora_prefix = 'Oracle ' - self.facts['distribution'] = data.split()[0] - self.facts['distribution_version'] = data.split()[1] - self.facts['distribution_release'] = ora_prefix + data - break - elif name == 'SuSE': - data = get_file_content(path) - if 'suse' in data.lower(): - if path == '/etc/os-release': + elif name == 'Solaris': + data = get_file_content(path).split('\n')[0] + if 'Solaris' in data: + ora_prefix = '' + if 'Oracle Solaris' in data: + data = data.replace('Oracle ','') + ora_prefix = 'Oracle ' + self.facts['distribution'] = data.split()[0] + self.facts['distribution_version'] = data.split()[1] + self.facts['distribution_release'] = ora_prefix + data + break + + uname_rc, uname_out, uname_err = module.run_command(['uname', '-v']) + distribution_version = None + if 'SmartOS' in data: + self.facts['distribution'] = 'SmartOS' + if os.path.exists('/etc/product'): + product_data = dict([l.split(': ', 1) for l in get_file_content('/etc/product').split('\n') if ': ' in l]) + if 'Image' in product_data: + distribution_version = product_data.get('Image').split()[-1] + elif 'OpenIndiana' in data: + self.facts['distribution'] = 'OpenIndiana' + elif 'OmniOS' in data: + self.facts['distribution'] = 'OmniOS' + distribution_version = data.split()[-1] + elif uname_rc == 0 and 'NexentaOS_' in uname_out: + self.facts['distribution'] = 'Nexenta' + distribution_version = data.split()[-1].lstrip('v') + + if self.facts['distribution'] in ('SmartOS', 'OpenIndiana', 'OmniOS', 'Nexenta'): + self.facts['distribution_release'] = data.strip() + if distribution_version is not None: + self.facts['distribution_version'] = distribution_version + elif uname_rc == 0: + self.facts['distribution_version'] = uname_out.split('\n')[0].strip() + break + + elif name == 'SuSE': + data = get_file_content(path) + if 'suse' in data.lower(): + if path == '/etc/os-release': + release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) + distdata = get_file_content(path).split('\n')[0] + self.facts['distribution'] = distdata.split('=')[1] + if release: + self.facts['distribution_release'] = release.groups()[0] + break + elif path == '/etc/SuSE-release': + data = data.splitlines() + distdata = get_file_content(path).split('\n')[0] + self.facts['distribution'] = distdata.split()[0] + for line in data: + release = re.search('CODENAME *= *([^\n]+)', line) + if release: + self.facts['distribution_release'] = release.groups()[0].strip() + break + elif name == 'Debian': + data = get_file_content(path) + if 'Debian' in data: release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) if release: self.facts['distribution_release'] = release.groups()[0] - break - elif path == '/etc/SuSE-release': - data = data.splitlines() - for line in data: - release = re.search('CODENAME *= *([^\n]+)', line) - if release: - self.facts['distribution_release'] = release.groups()[0].strip() - break - elif name == 'Debian': - data = get_file_content(path) - if 'Debian' in data: - release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data) - if release: - self.facts['distribution_release'] = release.groups()[0] - break - elif name == 'Mandriva': - data = get_file_content(path) - if 'Mandriva' in data: - version = re.search('DISTRIB_RELEASE="(.*)"', data) - if version: - self.facts['distribution_version'] = version.groups()[0] - release = re.search('DISTRIB_CODENAME="(.*)"', data) - if release: - self.facts['distribution_release'] = release.groups()[0] - self.facts['distribution'] = name - break + break + elif name == 'Mandriva': + data = get_file_content(path) + if 'Mandriva' in data: + version = re.search('DISTRIB_RELEASE="(.*)"', data) + if version: + self.facts['distribution_version'] = version.groups()[0] + release = re.search('DISTRIB_CODENAME="(.*)"', data) + if release: + self.facts['distribution_release'] = release.groups()[0] + self.facts['distribution'] = name + break else: self.facts['distribution'] = name @@ -598,22 +630,49 @@ class LinuxHardware(Hardware): def get_cpu_facts(self): i = 0 + vendor_id_occurrence = 0 + model_name_occurrence = 0 physid = 0 coreid = 0 sockets = {} cores = {} + + xen = False + xen_paravirt = False + try: + if os.path.exists('/proc/xen'): + xen = True + elif open('/sys/hypervisor/type').readline().strip() == 'xen': + xen = True + except IOError: + pass + if not os.access("/proc/cpuinfo", os.R_OK): return self.facts['processor'] = [] for line in open("/proc/cpuinfo").readlines(): data = line.split(":", 1) key = data[0].strip() + + if xen: + if key == 'flags': + # Check for vme cpu flag, Xen paravirt does not expose this. + # Need to detect Xen paravirt because it exposes cpuinfo + # differently than Xen HVM or KVM and causes reporting of + # only a single cpu core. + if 'vme' not in data: + xen_paravirt = True + # model name is for Intel arch, Processor (mind the uppercase P) # works for some ARM devices, like the Sheevaplug. if key == 'model name' or key == 'Processor' or key == 'vendor_id': if 'processor' not in self.facts: self.facts['processor'] = [] self.facts['processor'].append(data[1].strip()) + if key == 'vendor_id': + vendor_id_occurrence += 1 + if key == 'model name': + model_name_occurrence += 1 i += 1 elif key == 'physical id': physid = data[1].strip() @@ -629,13 +688,23 @@ class LinuxHardware(Hardware): cores[coreid] = int(data[1].strip()) elif key == '# processors': self.facts['processor_cores'] = int(data[1].strip()) + + if vendor_id_occurrence == model_name_occurrence: + i = vendor_id_occurrence + if self.facts['architecture'] != 's390x': - self.facts['processor_count'] = sockets and len(sockets) or i - self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1 - self.facts['processor_threads_per_core'] = ((cores.values() and - cores.values()[0] or 1) / self.facts['processor_cores']) - self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] * - self.facts['processor_count'] * self.facts['processor_cores']) + if xen_paravirt: + self.facts['processor_count'] = i + self.facts['processor_cores'] = i + self.facts['processor_threads_per_core'] = 1 + self.facts['processor_vcpus'] = i + else: + self.facts['processor_count'] = sockets and len(sockets) or i + self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1 + self.facts['processor_threads_per_core'] = ((cores.values() and + cores.values()[0] or 1) / self.facts['processor_cores']) + self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] * + self.facts['processor_count'] * self.facts['processor_cores']) def get_dmi_facts(self): ''' learn dmi facts from system @@ -1355,7 +1424,7 @@ class HPUX(Hardware): self.facts['memtotal_mb'] = int(data) / 1024 except AttributeError: #For systems where memory details aren't sent to syslog or the log has rotated, use parsed - #adb output. Unfortunatley /dev/kmem doesn't have world-read, so this only works as root. + #adb output. Unfortunately /dev/kmem doesn't have world-read, so this only works as root. if os.access("/dev/kmem", os.R_OK): rc, out, err = module.run_command("echo 'phys_mem_pages/D' | adb -k /stand/vmunix /dev/kmem | tail -1 | awk '{print $2}'", use_unsafe_shell=True) if not err: diff --git a/v2/ansible/module_utils/gce.py b/v2/ansible/module_utils/gce.py index 68aa66c41a9..37a4bf1deaf 100644 --- a/v2/ansible/module_utils/gce.py +++ b/v2/ansible/module_utils/gce.py @@ -32,7 +32,7 @@ import pprint USER_AGENT_PRODUCT="Ansible-gce" USER_AGENT_VERSION="v1" -def gce_connect(module): +def gce_connect(module, provider=None): """Return a Google Cloud Engine connection.""" service_account_email = module.params.get('service_account_email', None) pem_file = module.params.get('pem_file', None) @@ -71,8 +71,14 @@ def gce_connect(module): 'secrets file.') return None + # Allow for passing in libcloud Google DNS (e.g, Provider.GOOGLE) + if provider is None: + provider = Provider.GCE + try: - gce = get_driver(Provider.GCE)(service_account_email, pem_file, datacenter=module.params.get('zone'), project=project_id) + gce = get_driver(provider)(service_account_email, pem_file, + datacenter=module.params.get('zone', None), + project=project_id) gce.connection.user_agent_append("%s/%s" % ( USER_AGENT_PRODUCT, USER_AGENT_VERSION)) except (RuntimeError, ValueError), e: diff --git a/v2/ansible/module_utils/known_hosts.py b/v2/ansible/module_utils/known_hosts.py index c997596fd44..99dbf2c03ad 100644 --- a/v2/ansible/module_utils/known_hosts.py +++ b/v2/ansible/module_utils/known_hosts.py @@ -40,7 +40,7 @@ def add_git_host_key(module, url, accept_hostkey=True, create_dir=True): """ idempotently add a git url hostkey """ - fqdn = get_fqdn(module.params['repo']) + fqdn = get_fqdn(url) if fqdn: known_host = check_hostkey(module, fqdn) diff --git a/v2/ansible/module_utils/urls.py b/v2/ansible/module_utils/urls.py index c2d87c27bcf..962b868ee0d 100644 --- a/v2/ansible/module_utils/urls.py +++ b/v2/ansible/module_utils/urls.py @@ -252,9 +252,33 @@ class SSLValidationHandler(urllib2.BaseHandler): except: self.module.fail_json(msg='Connection to proxy failed') + def detect_no_proxy(self, url): + ''' + Detect if the 'no_proxy' environment variable is set and honor those locations. + ''' + env_no_proxy = os.environ.get('no_proxy') + if env_no_proxy: + env_no_proxy = env_no_proxy.split(',') + netloc = urlparse.urlparse(url).netloc + + for host in env_no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # Our requested URL matches something in no_proxy, so don't + # use the proxy for this + return False + return True + def http_request(self, req): tmp_ca_cert_path, paths_checked = self.get_ca_certs() https_proxy = os.environ.get('https_proxy') + + # Detect if 'no_proxy' environment variable is set and if our URL is included + use_proxy = self.detect_no_proxy(req.get_full_url()) + + if not use_proxy: + # ignore proxy settings for this host request + return req + try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if https_proxy: diff --git a/v2/ansible/parsing/mod_args.py b/v2/ansible/parsing/mod_args.py index 2e11b47d0a2..37ed350de54 100644 --- a/v2/ansible/parsing/mod_args.py +++ b/v2/ansible/parsing/mod_args.py @@ -55,6 +55,15 @@ class ModuleArgsParser: src: a dest: b + # extra gross, but also legal. in this case, the args specified + # will act as 'defaults' and will be overriden by any args specified + # in one of the other formats (complex args under the action, or + # parsed from the k=v string + - command: 'pwd' + args: + chdir: '/tmp' + + This class has some of the logic to canonicalize these into the form - module: @@ -104,19 +113,24 @@ class ModuleArgsParser: return (action, args) - def _normalize_parameters(self, thing, action=None): + def _normalize_parameters(self, thing, action=None, additional_args=dict()): ''' arguments can be fuzzy. Deal with all the forms. ''' - args = dict() + # final args are the ones we'll eventually return, so first update + # them with any additional args specified, which have lower priority + # than those which may be parsed/normalized next + final_args = dict() + if additional_args: + final_args.update(additional_args) # how we normalize depends if we figured out what the module name is # yet. If we have already figured it out, it's an 'old style' invocation. # otherwise, it's not if action is not None: - args = self._normalize_old_style_args(thing) + args = self._normalize_old_style_args(thing, action) else: (action, args) = self._normalize_new_style_args(thing) @@ -124,9 +138,14 @@ class ModuleArgsParser: if args and 'args' in args: args = args['args'] - return (action, args) + # finally, update the args we're going to return with the ones + # which were normalized above + if args: + final_args.update(args) - def _normalize_old_style_args(self, thing): + return (action, final_args) + + def _normalize_old_style_args(self, thing, action): ''' deals with fuzziness in old-style (action/local_action) module invocations returns tuple of (module_name, dictionary_args) @@ -144,7 +163,8 @@ class ModuleArgsParser: args = thing elif isinstance(thing, string_types): # form is like: local_action: copy src=a dest=b ... pretty common - args = parse_kv(thing) + check_raw = action in ('command', 'shell', 'script') + args = parse_kv(thing, check_raw=check_raw) elif isinstance(thing, NoneType): # this can happen with modules which take no params, like ping: args = None @@ -180,7 +200,8 @@ class ModuleArgsParser: elif isinstance(thing, string_types): # form is like: copy: src=a dest=b ... common shorthand throughout ansible (action, args) = self._split_module_string(thing) - args = parse_kv(args) + check_raw = action in ('command', 'shell', 'script') + args = parse_kv(args, check_raw=check_raw) else: # need a dict or a string, so giving up @@ -206,13 +227,20 @@ class ModuleArgsParser: # We can have one of action, local_action, or module specified # + + # this is the 'extra gross' scenario detailed above, so we grab + # the args and pass them in as additional arguments, which can/will + # be overwritten via dict updates from the other arg sources below + # FIXME: add test cases for this + additional_args = self._task_ds.get('args', dict()) + # action if 'action' in self._task_ds: # an old school 'action' statement thing = self._task_ds['action'] delegate_to = None - action, args = self._normalize_parameters(thing) + action, args = self._normalize_parameters(thing, additional_args=additional_args) # local_action if 'local_action' in self._task_ds: @@ -222,7 +250,7 @@ class ModuleArgsParser: raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds) thing = self._task_ds.get('local_action', '') delegate_to = 'localhost' - action, args = self._normalize_parameters(thing) + action, args = self._normalize_parameters(thing, additional_args=additional_args) # module: is the more new-style invocation @@ -234,7 +262,7 @@ class ModuleArgsParser: raise AnsibleParserError("conflicting action statements", obj=self._task_ds) action = item thing = value - action, args = self._normalize_parameters(value, action=action) + action, args = self._normalize_parameters(value, action=action, additional_args=additional_args) # if we didn't see any module in the task at all, it's not a task really if action is None: diff --git a/v2/ansible/parsing/splitter.py b/v2/ansible/parsing/splitter.py index 470ab90c3cd..9705baf169d 100644 --- a/v2/ansible/parsing/splitter.py +++ b/v2/ansible/parsing/splitter.py @@ -40,7 +40,20 @@ def parse_kv(args, check_raw=False): raw_params = [] for x in vargs: if "=" in x: - k, v = x.split("=", 1) + pos = 0 + try: + while True: + pos = x.index('=', pos + 1) + if pos > 0 and x[pos - 1] != '\\': + break + except ValueError: + # ran out of string, but we must have some escaped equals, + # so replace those and append this to the list of raw params + raw_params.append(x.replace('\\=', '=')) + continue + + k = x[:pos] + v = x[pos + 1:] # only internal variables can start with an underscore, so # we don't allow users to set them directy in arguments diff --git a/v2/ansible/playbook/conditional.py b/v2/ansible/playbook/conditional.py index 82a09a30656..65ed7c4b540 100644 --- a/v2/ansible/playbook/conditional.py +++ b/v2/ansible/playbook/conditional.py @@ -32,7 +32,15 @@ class Conditional: _when = FieldAttribute(isa='list', default=[]) - def __init__(self): + def __init__(self, loader=None): + # when used directly, this class needs a loader, but we want to + # make sure we don't trample on the existing one if this class + # is used as a mix-in with a playbook base class + if not hasattr(self, '_loader'): + if loader is None: + raise AnsibleError("a loader must be specified when using Conditional() directly") + else: + self._loader = loader super(Conditional, self).__init__() def _validate_when(self, attr, name, value): diff --git a/v2/ansible/playbook/task.py b/v2/ansible/playbook/task.py index f9c52281586..db5c017d673 100644 --- a/v2/ansible/playbook/task.py +++ b/v2/ansible/playbook/task.py @@ -34,8 +34,6 @@ from ansible.playbook.conditional import Conditional from ansible.playbook.role import Role from ansible.playbook.taggable import Taggable -from ansible.utils.listify import listify_lookup_plugin_terms - class Task(Base, Conditional, Taggable): """ @@ -199,9 +197,6 @@ class Task(Base, Conditional, Taggable): super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) - def _post_validate_loop_args(self, attr, value, all_vars, fail_on_undefined): - return listify_lookup_plugin_terms(value, all_vars, loader=self._loader) - def get_vars(self): all_vars = self.serialize() if 'tags' in all_vars: diff --git a/v2/ansible/plugins/action/__init__.py b/v2/ansible/plugins/action/__init__.py index aa3ab5657f3..e8f3507b399 100644 --- a/v2/ansible/plugins/action/__init__.py +++ b/v2/ansible/plugins/action/__init__.py @@ -31,7 +31,7 @@ from ansible import constants as C from ansible.errors import AnsibleError from ansible.executor.module_common import ModuleReplacer from ansible.parsing.utils.jsonify import jsonify -from ansible.plugins import module_loader, shell_loader +from ansible.plugins import shell_loader from ansible.utils.debug import debug @@ -44,11 +44,12 @@ class ActionBase: action in use. ''' - def __init__(self, task, connection, connection_info, loader): + def __init__(self, task, connection, connection_info, loader, module_loader): self._task = task self._connection = connection self._connection_info = connection_info self._loader = loader + self._module_loader = module_loader self._shell = self.get_shell() def get_shell(self): @@ -80,9 +81,9 @@ class ActionBase: # Search module path(s) for named module. module_suffixes = getattr(self._connection, 'default_suffixes', None) - module_path = module_loader.find_plugin(module_name, module_suffixes, transport=self._connection.get_transport()) + module_path = self._module_loader.find_plugin(module_name, module_suffixes, transport=self._connection.get_transport()) if module_path is None: - module_path2 = module_loader.find_plugin('ping', module_suffixes) + module_path2 = self._module_loader.find_plugin('ping', module_suffixes) if module_path2 is not None: raise AnsibleError("The module %s was not found in configured module paths" % (module_name)) else: @@ -391,6 +392,10 @@ class ActionBase: data = json.loads(self._filter_leading_non_json_lines(res['stdout'])) if 'parsed' in data and data['parsed'] == False: data['msg'] += res['stderr'] + # pre-split stdout into lines, if stdout is in the data and there + # isn't already a stdout_lines value there + if 'stdout' in data and 'stdout_lines' not in data: + data['stdout_lines'] = data.get('stdout', '').splitlines() else: data = dict() @@ -424,7 +429,6 @@ class ActionBase: cmd, prompt, success_key = self._connection_info.make_sudo_cmd('/usr/bin/sudo', executable, cmd) debug("executing the command through the connection") - #rc, stdin, stdout, stderr = self._connection.exec_command(cmd, tmp, executable=executable, in_data=in_data, sudoable=sudoable) rc, stdin, stdout, stderr = self._connection.exec_command(cmd, tmp, executable=executable, in_data=in_data) debug("command execution done") diff --git a/v2/ansible/plugins/action/assert.py b/v2/ansible/plugins/action/assert.py index e1b2ba19737..7204d93875e 100644 --- a/v2/ansible/plugins/action/assert.py +++ b/v2/ansible/plugins/action/assert.py @@ -16,6 +16,7 @@ # along with Ansible. If not, see . from ansible.errors import AnsibleError +from ansible.playbook.conditional import Conditional from ansible.plugins.action import ActionBase class ActionModule(ActionBase): @@ -42,9 +43,10 @@ class ActionModule(ActionBase): # the built in evaluate function. The when has already been evaluated # by this point, and is not used again, so we don't care about mangling # that value now + cond = Conditional(loader=self._loader) for that in thats: - self._task.when = [ that ] - test_result = self._task.evaluate_conditional(all_vars=task_vars) + cond.when = [ that ] + test_result = cond.evaluate_conditional(all_vars=task_vars) if not test_result: result = dict( failed = True, diff --git a/v2/ansible/plugins/action/script.py b/v2/ansible/plugins/action/script.py index ceccd71e7d4..6e8c1e1b9a4 100644 --- a/v2/ansible/plugins/action/script.py +++ b/v2/ansible/plugins/action/script.py @@ -63,12 +63,10 @@ class ActionModule(ActionBase): source = parts[0] args = ' '.join(parts[1:]) - # FIXME: need to sort out all the _original_file stuff still - #if '_original_file' in task_vars: - # source = self._loader.path_dwim_relative(inject['_original_file'], 'files', source, self.runner.basedir) - #else: - # source = self._loader.path_dwim(self.runner.basedir, source) - source = self._loader.path_dwim(source) + if self._task._role is not None: + source = self._loader.path_dwim_relative(self._task._role._role_path, 'files', source) + else: + source = self._loader.path_dwim(source) # transfer the file to a remote tmp location tmp_src = self._shell.join_path(tmp, os.path.basename(source)) diff --git a/v2/ansible/plugins/action/set_fact.py b/v2/ansible/plugins/action/set_fact.py index 8c54ac18f8e..c380cadca91 100644 --- a/v2/ansible/plugins/action/set_fact.py +++ b/v2/ansible/plugins/action/set_fact.py @@ -17,10 +17,17 @@ from ansible.errors import AnsibleError from ansible.plugins.action import ActionBase +from ansible.template import Templar class ActionModule(ActionBase): TRANSFERS_FILES = False def run(self, tmp=None, task_vars=dict()): - return dict(changed=True, ansible_facts=self._task.args) + templar = Templar(loader=self._loader, variables=task_vars) + facts = dict() + if self._task.args: + for (k, v) in self._task.args.iteritems(): + k = templar.template(k) + facts[k] = v + return dict(changed=True, ansible_facts=facts) diff --git a/v2/ansible/plugins/callback/default.py b/v2/ansible/plugins/callback/default.py index d498e817fda..9ae156931b0 100644 --- a/v2/ansible/plugins/callback/default.py +++ b/v2/ansible/plugins/callback/default.py @@ -46,23 +46,34 @@ class CallbackModule(CallbackBase): pass def runner_on_failed(self, task, result, ignore_errors=False): - self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), result._result), color='red') + self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), json.dumps(result._result, ensure_ascii=False)), color='red') def runner_on_ok(self, task, result): - msg = "ok: [%s]" % result._host.get_name() + + if result._result.get('changed', False): + msg = "changed: [%s]" % result._host.get_name() + color = 'yellow' + else: + msg = "ok: [%s]" % result._host.get_name() + color = 'green' + if self._display._verbosity > 0 or 'verbose_always' in result._result: + indent = None if 'verbose_always' in result._result: + indent = 4 del result._result['verbose_always'] - msg += " => %s" % result._result - self._display.display(msg, color='green') + msg += " => %s" % json.dumps(result._result, indent=indent, ensure_ascii=False) + self._display.display(msg, color=color) def runner_on_skipped(self, task, result): - msg = "SKIPPED: [%s]" % result._host.get_name() + msg = "skipping: [%s]" % result._host.get_name() if self._display._verbosity > 0 or 'verbose_always' in result._result: + indent = None if 'verbose_always' in result._result: + indent = 4 del result._result['verbose_always'] - msg += " => %s" % result._result - self._display.display(msg) + msg += " => %s" % json.dumps(result._result, indent=indent, ensure_ascii=False) + self._display.display(msg, color='cyan') def runner_on_unreachable(self, task, result): self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), result._result), color='red') diff --git a/v2/ansible/plugins/connections/local.py b/v2/ansible/plugins/connections/local.py index f327c3f9de7..58e8a20a2ee 100644 --- a/v2/ansible/plugins/connections/local.py +++ b/v2/ansible/plugins/connections/local.py @@ -117,7 +117,8 @@ class Connection(ConnectionBase): #vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) self._display.vvv("%s PUT %s TO %s" % (self._host, in_path, out_path)) if not os.path.exists(in_path): - raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) + #raise AnsibleFileNotFound("file or module does not exist: %s" % in_path) + raise AnsibleError("file or module does not exist: %s" % in_path) try: shutil.copyfile(in_path, out_path) except shutil.Error: diff --git a/v2/ansible/plugins/lookup/first_found.py b/v2/ansible/plugins/lookup/first_found.py index 439b48ee440..3b59a191cbf 100644 --- a/v2/ansible/plugins/lookup/first_found.py +++ b/v2/ansible/plugins/lookup/first_found.py @@ -120,6 +120,7 @@ import os from ansible.plugins.lookup import LookupBase +from ansible.template import Templar class LookupModule(LookupBase): @@ -167,22 +168,26 @@ class LookupModule(LookupBase): else: total_search = terms + templar = Templar(loader=self._loader, variables=variables) + roledir = variables.get('roledir') for fn in total_search: - # FIXME: the original file stuff needs to be fixed/implemented - #if variables and '_original_file' in variables: - # # check the templates and vars directories too, - # # if they exist - # for roledir in ('templates', 'vars'): - # path = self._loader.path_dwim(os.path.join(self.basedir, '..', roledir), fn) - # if os.path.exists(path): - # return [path] + fn = templar.template(fn) + if os.path.isabs(fn) and os.path.exists(fn): + return [fn] + else: + if roledir is not None: + # check the templates and vars directories too,if they exist + for subdir in ('templates', 'vars'): + path = self._loader.path_dwim_relative(roledir, subdir, fn) + if os.path.exists(path): + return [path] - # if none of the above were found, just check the - # current filename against the basedir (this will already - # have ../files from runner, if it's a role task - path = self._loader.path_dwim(fn) - if os.path.exists(path): - return [path] + # if none of the above were found, just check the + # current filename against the basedir (this will already + # have ../files from runner, if it's a role task + path = self._loader.path_dwim(fn) + if os.path.exists(path): + return [path] else: if skip: return [] diff --git a/v2/ansible/plugins/lookup/items.py b/v2/ansible/plugins/lookup/items.py index 9dceb22a8f1..46925d2a8ba 100644 --- a/v2/ansible/plugins/lookup/items.py +++ b/v2/ansible/plugins/lookup/items.py @@ -20,5 +20,9 @@ from ansible.plugins.lookup import LookupBase class LookupModule(LookupBase): def run(self, terms, **kwargs): + + if not isinstance(terms, list): + terms = [ terms ] + return self._flatten(terms) diff --git a/v2/ansible/plugins/lookup/sequence.py b/v2/ansible/plugins/lookup/sequence.py index 5347b90f6a0..99783cf566b 100644 --- a/v2/ansible/plugins/lookup/sequence.py +++ b/v2/ansible/plugins/lookup/sequence.py @@ -20,6 +20,7 @@ from re import compile as re_compile, IGNORECASE from ansible.errors import * from ansible.parsing.splitter import parse_kv from ansible.plugins.lookup import LookupBase +from ansible.template import Templar # shortcut format NUM = "(0?x?[0-9a-f]+)" @@ -174,15 +175,18 @@ class LookupModule(LookupBase): if isinstance(terms, basestring): terms = [ terms ] + templar = Templar(loader=self._loader, variables=variables) + for term in terms: try: self.reset() # clear out things for this iteration + term = templar.template(term) try: if not self.parse_simple_args(term): self.parse_kv_args(parse_kv(term)) except Exception, e: - raise AnsibleError("unknown error parsing with_sequence arguments: %r" % term) + raise AnsibleError("unknown error parsing with_sequence arguments: %r. Error was: %s" % (term, e)) self.sanity_check() diff --git a/v2/ansible/plugins/lookup/together.py b/v2/ansible/plugins/lookup/together.py index aa13cd7e60d..8b5ff5c8919 100644 --- a/v2/ansible/plugins/lookup/together.py +++ b/v2/ansible/plugins/lookup/together.py @@ -32,7 +32,7 @@ class LookupModule(LookupBase): def __lookup_variabless(self, terms, variables): results = [] for x in terms: - intermediate = listify_lookup_plugin_terms(x, variables) + intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader) results.append(intermediate) return results diff --git a/v2/ansible/plugins/strategies/__init__.py b/v2/ansible/plugins/strategies/__init__.py index be623289405..9a64216cef1 100644 --- a/v2/ansible/plugins/strategies/__init__.py +++ b/v2/ansible/plugins/strategies/__init__.py @@ -29,6 +29,7 @@ from ansible.inventory.group import Group from ansible.playbook.helpers import compile_block_list from ansible.playbook.role import ROLE_CACHE +from ansible.plugins import module_loader from ansible.utils.debug import debug @@ -103,7 +104,7 @@ class StrategyBase: self._cur_worker = 0 self._pending_results += 1 - main_q.put((host, task, self._loader.get_basedir(), task_vars, connection_info), block=False) + main_q.put((host, task, self._loader.get_basedir(), task_vars, connection_info, module_loader), block=False) except (EOFError, IOError, AssertionError), e: # most likely an abort debug("got an error while queuing: %s" % e) @@ -154,20 +155,20 @@ class StrategyBase: elif result[0] == 'add_host': task_result = result[1] - new_host_info = task_result._result.get('add_host', dict()) + new_host_info = task_result.get('add_host', dict()) self._add_host(new_host_info) elif result[0] == 'add_group': - task_result = result[1] - host = task_result._host - group_name = task_result._result.get('add_group') + host = result[1] + task_result = result[2] + group_name = task_result.get('add_group') self._add_group(host, group_name) elif result[0] == 'notify_handler': - handler_name = result[1] - host = result[2] + host = result[1] + handler_name = result[2] if host not in self._notified_handlers[handler_name]: self._notified_handlers[handler_name].append(host) diff --git a/v2/ansible/plugins/strategies/free.py b/v2/ansible/plugins/strategies/free.py index 2828bff1bdc..6aab495fec3 100644 --- a/v2/ansible/plugins/strategies/free.py +++ b/v2/ansible/plugins/strategies/free.py @@ -67,6 +67,11 @@ class StrategyModule(StrategyBase): # anything to do do for this host if host_name not in self._tqm._failed_hosts and host_name not in self._tqm._unreachable_hosts and iterator.get_next_task_for_host(host, peek=True): + # FIXME: check task tags, etc. here as we do in linear + # FIXME: handle meta tasks here, which will require a tweak + # to run_handlers so that only the handlers on this host + # are flushed and not all + # set the flag so the outer loop knows we've still found # some work which needs to be done work_to_do = True diff --git a/v2/ansible/plugins/strategies/linear.py b/v2/ansible/plugins/strategies/linear.py index f5625ec89b9..93c2c5a4b3b 100644 --- a/v2/ansible/plugins/strategies/linear.py +++ b/v2/ansible/plugins/strategies/linear.py @@ -19,8 +19,8 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible.errors import AnsibleError from ansible.plugins.strategies import StrategyBase - from ansible.utils.debug import debug class StrategyModule(StrategyBase): @@ -80,12 +80,21 @@ class StrategyModule(StrategyBase): continue work_to_do = True - if not callback_sent: - self._callback.playbook_on_task_start(task.get_name(), False) - callback_sent = True + if task.action == 'meta': + # meta tasks store their args in the _raw_params field of args, + # since they do not use k=v pairs, so get that + meta_action = task.args.get('_raw_params') + if meta_action == 'flush_handlers': + self.run_handlers(iterator, connection_info) + else: + raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds) + else: + if not callback_sent: + self._callback.playbook_on_task_start(task.get_name(), False) + callback_sent = True - self._blocked_hosts[host.get_name()] = True - self._queue_task(host, task, task_vars, connection_info) + self._blocked_hosts[host.get_name()] = True + self._queue_task(host, task, task_vars, connection_info) self._process_pending_results() diff --git a/v2/ansible/utils/listify.py b/v2/ansible/utils/listify.py index bec7326b359..800b99b8ecc 100644 --- a/v2/ansible/utils/listify.py +++ b/v2/ansible/utils/listify.py @@ -58,7 +58,6 @@ def listify_lookup_plugin_terms(terms, variables, loader): if '{' in terms or '[' in terms: # Jinja2 already evaluated a variable to a list. # Jinja2-ified list needs to be converted back to a real type - # TODO: something a bit less heavy than eval return safe_eval(terms) if isinstance(terms, basestring): diff --git a/v2/ansible/vars/__init__.py b/v2/ansible/vars/__init__.py index 074271c715b..16a3e32e612 100644 --- a/v2/ansible/vars/__init__.py +++ b/v2/ansible/vars/__init__.py @@ -150,7 +150,6 @@ class VariableManager: # next comes the facts cache and the vars cache, respectively all_vars = self._merge_dicts(all_vars, self._fact_cache.get(host.get_name(), dict())) - all_vars = self._merge_dicts(all_vars, self._vars_cache.get(host.get_name(), dict())) if play: all_vars = self._merge_dicts(all_vars, play.get_vars()) @@ -168,6 +167,9 @@ class VariableManager: for role in play.get_roles(): all_vars = self._merge_dicts(all_vars, role.get_vars()) + if host: + all_vars = self._merge_dicts(all_vars, self._vars_cache.get(host.get_name(), dict())) + if task: if task._role: all_vars = self._merge_dicts(all_vars, task._role.get_vars())