More work on getting integration tests running for v2

This commit is contained in:
James Cammarata 2015-01-15 01:13:45 -06:00
parent 6326daa34e
commit 02bc014bcd
27 changed files with 414 additions and 187 deletions

View file

@ -109,15 +109,18 @@ class ResultProcess(multiprocessing.Process):
host_name = result._host.get_name()
# send callbacks, execute other options based on the result status
if result.is_failed():
self._send_result(('host_task_failed', result))
elif result.is_unreachable():
# FIXME: this should all be cleaned up and probably moved to a sub-function.
# the fact that this sometimes sends a TaskResult and other times
# sends a raw dictionary back may be confusing, but the result vs.
# results implementation for tasks with loops should be cleaned up
# better than this
if result.is_unreachable():
self._send_result(('host_unreachable', result))
elif result.is_failed():
self._send_result(('host_task_failed', result))
elif result.is_skipped():
self._send_result(('host_task_skipped', result))
else:
self._send_result(('host_task_ok', result))
# if this task is notifying a handler, do it now
if result._task.notify:
# The shared dictionary for notified handlers is a proxy, which
@ -125,21 +128,32 @@ class ResultProcess(multiprocessing.Process):
# So, per the docs, we reassign the list so the proxy picks up and
# notifies all other threads
for notify in result._task.notify:
self._send_result(('notify_handler', notify, result._host))
self._send_result(('notify_handler', result._host, notify))
if 'add_host' in result._result:
# this task added a new host (add_host module)
self._send_result(('add_host', result))
elif 'add_group' in result._result:
# this task added a new group (group_by module)
self._send_result(('add_group', result))
elif 'ansible_facts' in result._result:
# if this task is registering facts, do that now
if result._task.action in ('set_fact', 'include_vars'):
for (key, value) in result._result['ansible_facts'].iteritems():
self._send_result(('set_host_var', result._host, key, value))
else:
self._send_result(('set_host_facts', result._host, result._result['ansible_facts']))
if 'results' in result._result:
# this task had a loop, and has more than one result, so
# loop over all of them instead of a single result
result_items = result._result['results']
else:
result_items = [ result._result ]
for result_item in result_items:
if 'add_host' in result_item:
# this task added a new host (add_host module)
self._send_result(('add_host', result_item))
elif 'add_group' in result_item:
# this task added a new group (group_by module)
self._send_result(('add_group', result._host, result_item))
elif 'ansible_facts' in result_item:
# if this task is registering facts, do that now
if result._task.action in ('set_fact', 'include_vars'):
for (key, value) in result_item['ansible_facts'].iteritems():
self._send_result(('set_host_var', result._host, key, value))
else:
self._send_result(('set_host_facts', result._host, result_item['ansible_facts']))
# finally, send the ok for this task
self._send_result(('host_task_ok', result))
# if this task is registering a result, do it now
if result._task.register:

View file

@ -97,7 +97,7 @@ class WorkerProcess(multiprocessing.Process):
try:
if not self._main_q.empty():
debug("there's work to be done!")
(host, task, basedir, job_vars, connection_info) = self._main_q.get(block=False)
(host, task, basedir, job_vars, connection_info, module_loader) = self._main_q.get(block=False)
debug("got a task/handler to work on: %s" % task)
# because the task queue manager starts workers (forks) before the
@ -118,7 +118,7 @@ class WorkerProcess(multiprocessing.Process):
# execute the task and build a TaskResult from the result
debug("running TaskExecutor() for %s/%s" % (host, task))
executor_result = TaskExecutor(host, task, job_vars, new_connection_info, self._loader).run()
executor_result = TaskExecutor(host, task, job_vars, new_connection_info, self._loader, module_loader).run()
debug("done running TaskExecutor() for %s/%s" % (host, task))
task_result = TaskResult(host, task, executor_result)

View file

@ -22,8 +22,10 @@ __metaclass__ = type
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleParserError
from ansible.executor.connection_info import ConnectionInformation
from ansible.playbook.conditional import Conditional
from ansible.playbook.task import Task
from ansible.plugins import lookup_loader, connection_loader, action_loader
from ansible.utils.listify import listify_lookup_plugin_terms
from ansible.utils.debug import debug
@ -41,12 +43,13 @@ class TaskExecutor:
class.
'''
def __init__(self, host, task, job_vars, connection_info, loader):
def __init__(self, host, task, job_vars, connection_info, loader, module_loader):
self._host = host
self._task = task
self._job_vars = job_vars
self._connection_info = connection_info
self._loader = loader
self._module_loader = module_loader
def run(self):
'''
@ -57,6 +60,13 @@ class TaskExecutor:
debug("in run()")
try:
# lookup plugins need to know if this task is executing from
# a role, so that it can properly find files/templates/etc.
roledir = None
if self._task._role:
roledir = self._task._role._role_path
self._job_vars['roledir'] = roledir
items = self._get_loop_items()
if items is not None:
if len(items) > 0:
@ -84,7 +94,8 @@ class TaskExecutor:
items = None
if self._task.loop and self._task.loop in lookup_loader:
items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=self._task.loop_args, variables=self._job_vars)
loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, variables=self._job_vars, loader=self._loader)
items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=loop_terms, variables=self._job_vars)
return items
@ -184,11 +195,10 @@ class TaskExecutor:
# now update them with the registered value, if it is set
if self._task.register:
vars_copy[self._task.register] = result
# now create a pseudo task, and assign the value of the until parameter
# to the when param, so we can use evaluate_conditional()
pseudo_task = Task()
pseudo_task.when = self._task.until
if pseudo_task.evaluate_conditional(vars_copy):
# create a conditional object to evaluate the until condition
cond = Conditional(loader=self._loader)
cond.when = self._task.until
if cond.evaluate_conditional(vars_copy):
break
elif 'failed' not in result and result.get('rc', 0) == 0:
# if the result is not failed, stop trying
@ -223,7 +233,8 @@ class TaskExecutor:
task=async_task,
connection=self._connection,
connection_info=self._connection_info,
loader=self._loader
loader=self._loader,
module_loader=self._module_loader,
)
time_left = self._task.async
@ -283,7 +294,8 @@ class TaskExecutor:
task=self._task,
connection=connection,
connection_info=self._connection_info,
loader=self._loader
loader=self._loader,
module_loader=self._module_loader,
)
if not handler:
raise AnsibleError("the handler '%s' was not found" % handler_name)

View file

@ -39,6 +39,7 @@ AWS_REGIONS = [
'cn-north-1',
'eu-central-1',
'eu-west-1',
'eu-central-1',
'sa-east-1',
'us-east-1',
'us-west-1',
@ -165,6 +166,11 @@ def boto_fix_security_token_in_profile(conn, profile_name):
def connect_to_aws(aws_module, region, **params):
conn = aws_module.connect_to_region(region, **params)
if not conn:
if region not in [aws_module_region.name for aws_module_region in aws_module.regions()]:
raise StandardError("Region %s does not seem to be available for aws module %s. If the region definitely exists, you may need to upgrade boto" % (region, aws_module.__name__))
else:
raise StandardError("Unknown problem connecting to region %s for aws module %s." % (region, aws_module.__name__))
if params.get('profile_name'):
conn = boto_fix_security_token_in_profile(conn, params['profile_name'])
return conn
@ -180,13 +186,13 @@ def ec2_connect(module):
if region:
try:
ec2 = connect_to_aws(boto.ec2, region, **boto_params)
except boto.exception.NoAuthHandlerFound, e:
except (boto.exception.NoAuthHandlerFound, StandardError), e:
module.fail_json(msg=str(e))
# Otherwise, no region so we fallback to the old connection method
elif ec2_url:
try:
ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
except boto.exception.NoAuthHandlerFound, e:
except (boto.exception.NoAuthHandlerFound, StandardError), e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="Either region or ec2_url must be specified")

View file

@ -46,7 +46,7 @@ except ImportError:
import simplejson as json
# --------------------------------------------------------------
# timeout function to make sure some fact gathering
# timeout function to make sure some fact gathering
# steps do not exceed a time limit
class TimeoutError(Exception):
@ -82,7 +82,8 @@ class Facts(object):
subclass Facts.
"""
_I386RE = re.compile(r'i[3456]86')
# i86pc is a Solaris and derivatives-ism
_I386RE = re.compile(r'i([3456]86|86pc)')
# For the most part, we assume that platform.dist() will tell the truth.
# This is the fallback to handle unknowns or exceptions
OSDIST_LIST = ( ('/etc/redhat-release', 'RedHat'),
@ -274,84 +275,115 @@ class Facts(object):
self.facts['distribution_release'] = dist[2] or 'NA'
# Try to handle the exceptions now ...
for (path, name) in Facts.OSDIST_LIST:
if os.path.exists(path) and os.path.getsize(path) > 0:
if self.facts['distribution'] in ('Fedora', ):
# Once we determine the value is one of these distros
# we trust the values are always correct
break
elif name == 'RedHat':
data = get_file_content(path)
if 'Red Hat' in data:
if os.path.exists(path):
if os.path.getsize(path) > 0:
if self.facts['distribution'] in ('Fedora', ):
# Once we determine the value is one of these distros
# we trust the values are always correct
break
elif name == 'RedHat':
data = get_file_content(path)
if 'Red Hat' in data:
self.facts['distribution'] = name
else:
self.facts['distribution'] = data.split()[0]
break
elif name == 'OtherLinux':
data = get_file_content(path)
if 'Amazon' in data:
self.facts['distribution'] = 'Amazon'
self.facts['distribution_version'] = data.split()[-1]
break
elif name == 'OpenWrt':
data = get_file_content(path)
if 'OpenWrt' in data:
self.facts['distribution'] = name
version = re.search('DISTRIB_RELEASE="(.*)"', data)
if version:
self.facts['distribution_version'] = version.groups()[0]
release = re.search('DISTRIB_CODENAME="(.*)"', data)
if release:
self.facts['distribution_release'] = release.groups()[0]
break
elif name == 'Alpine':
data = get_file_content(path)
self.facts['distribution'] = name
else:
self.facts['distribution'] = data.split()[0]
break
elif name == 'OtherLinux':
data = get_file_content(path)
if 'Amazon' in data:
self.facts['distribution'] = 'Amazon'
self.facts['distribution_version'] = data.split()[-1]
self.facts['distribution_version'] = data
break
elif name == 'OpenWrt':
data = get_file_content(path)
if 'OpenWrt' in data:
self.facts['distribution'] = name
version = re.search('DISTRIB_RELEASE="(.*)"', data)
if version:
self.facts['distribution_version'] = version.groups()[0]
release = re.search('DISTRIB_CODENAME="(.*)"', data)
if release:
self.facts['distribution_release'] = release.groups()[0]
break
elif name == 'Alpine':
data = get_file_content(path)
self.facts['distribution'] = name
self.facts['distribution_version'] = data
break
elif name == 'Solaris':
data = get_file_content(path).split('\n')[0]
if 'Solaris' in data:
ora_prefix = ''
if 'Oracle Solaris' in data:
data = data.replace('Oracle ','')
ora_prefix = 'Oracle '
self.facts['distribution'] = data.split()[0]
self.facts['distribution_version'] = data.split()[1]
self.facts['distribution_release'] = ora_prefix + data
break
elif name == 'SuSE':
data = get_file_content(path)
if 'suse' in data.lower():
if path == '/etc/os-release':
elif name == 'Solaris':
data = get_file_content(path).split('\n')[0]
if 'Solaris' in data:
ora_prefix = ''
if 'Oracle Solaris' in data:
data = data.replace('Oracle ','')
ora_prefix = 'Oracle '
self.facts['distribution'] = data.split()[0]
self.facts['distribution_version'] = data.split()[1]
self.facts['distribution_release'] = ora_prefix + data
break
uname_rc, uname_out, uname_err = module.run_command(['uname', '-v'])
distribution_version = None
if 'SmartOS' in data:
self.facts['distribution'] = 'SmartOS'
if os.path.exists('/etc/product'):
product_data = dict([l.split(': ', 1) for l in get_file_content('/etc/product').split('\n') if ': ' in l])
if 'Image' in product_data:
distribution_version = product_data.get('Image').split()[-1]
elif 'OpenIndiana' in data:
self.facts['distribution'] = 'OpenIndiana'
elif 'OmniOS' in data:
self.facts['distribution'] = 'OmniOS'
distribution_version = data.split()[-1]
elif uname_rc == 0 and 'NexentaOS_' in uname_out:
self.facts['distribution'] = 'Nexenta'
distribution_version = data.split()[-1].lstrip('v')
if self.facts['distribution'] in ('SmartOS', 'OpenIndiana', 'OmniOS', 'Nexenta'):
self.facts['distribution_release'] = data.strip()
if distribution_version is not None:
self.facts['distribution_version'] = distribution_version
elif uname_rc == 0:
self.facts['distribution_version'] = uname_out.split('\n')[0].strip()
break
elif name == 'SuSE':
data = get_file_content(path)
if 'suse' in data.lower():
if path == '/etc/os-release':
release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data)
distdata = get_file_content(path).split('\n')[0]
self.facts['distribution'] = distdata.split('=')[1]
if release:
self.facts['distribution_release'] = release.groups()[0]
break
elif path == '/etc/SuSE-release':
data = data.splitlines()
distdata = get_file_content(path).split('\n')[0]
self.facts['distribution'] = distdata.split()[0]
for line in data:
release = re.search('CODENAME *= *([^\n]+)', line)
if release:
self.facts['distribution_release'] = release.groups()[0].strip()
break
elif name == 'Debian':
data = get_file_content(path)
if 'Debian' in data:
release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data)
if release:
self.facts['distribution_release'] = release.groups()[0]
break
elif path == '/etc/SuSE-release':
data = data.splitlines()
for line in data:
release = re.search('CODENAME *= *([^\n]+)', line)
if release:
self.facts['distribution_release'] = release.groups()[0].strip()
break
elif name == 'Debian':
data = get_file_content(path)
if 'Debian' in data:
release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data)
if release:
self.facts['distribution_release'] = release.groups()[0]
break
elif name == 'Mandriva':
data = get_file_content(path)
if 'Mandriva' in data:
version = re.search('DISTRIB_RELEASE="(.*)"', data)
if version:
self.facts['distribution_version'] = version.groups()[0]
release = re.search('DISTRIB_CODENAME="(.*)"', data)
if release:
self.facts['distribution_release'] = release.groups()[0]
self.facts['distribution'] = name
break
break
elif name == 'Mandriva':
data = get_file_content(path)
if 'Mandriva' in data:
version = re.search('DISTRIB_RELEASE="(.*)"', data)
if version:
self.facts['distribution_version'] = version.groups()[0]
release = re.search('DISTRIB_CODENAME="(.*)"', data)
if release:
self.facts['distribution_release'] = release.groups()[0]
self.facts['distribution'] = name
break
else:
self.facts['distribution'] = name
@ -598,22 +630,49 @@ class LinuxHardware(Hardware):
def get_cpu_facts(self):
i = 0
vendor_id_occurrence = 0
model_name_occurrence = 0
physid = 0
coreid = 0
sockets = {}
cores = {}
xen = False
xen_paravirt = False
try:
if os.path.exists('/proc/xen'):
xen = True
elif open('/sys/hypervisor/type').readline().strip() == 'xen':
xen = True
except IOError:
pass
if not os.access("/proc/cpuinfo", os.R_OK):
return
self.facts['processor'] = []
for line in open("/proc/cpuinfo").readlines():
data = line.split(":", 1)
key = data[0].strip()
if xen:
if key == 'flags':
# Check for vme cpu flag, Xen paravirt does not expose this.
# Need to detect Xen paravirt because it exposes cpuinfo
# differently than Xen HVM or KVM and causes reporting of
# only a single cpu core.
if 'vme' not in data:
xen_paravirt = True
# model name is for Intel arch, Processor (mind the uppercase P)
# works for some ARM devices, like the Sheevaplug.
if key == 'model name' or key == 'Processor' or key == 'vendor_id':
if 'processor' not in self.facts:
self.facts['processor'] = []
self.facts['processor'].append(data[1].strip())
if key == 'vendor_id':
vendor_id_occurrence += 1
if key == 'model name':
model_name_occurrence += 1
i += 1
elif key == 'physical id':
physid = data[1].strip()
@ -629,13 +688,23 @@ class LinuxHardware(Hardware):
cores[coreid] = int(data[1].strip())
elif key == '# processors':
self.facts['processor_cores'] = int(data[1].strip())
if vendor_id_occurrence == model_name_occurrence:
i = vendor_id_occurrence
if self.facts['architecture'] != 's390x':
self.facts['processor_count'] = sockets and len(sockets) or i
self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1
self.facts['processor_threads_per_core'] = ((cores.values() and
cores.values()[0] or 1) / self.facts['processor_cores'])
self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] *
self.facts['processor_count'] * self.facts['processor_cores'])
if xen_paravirt:
self.facts['processor_count'] = i
self.facts['processor_cores'] = i
self.facts['processor_threads_per_core'] = 1
self.facts['processor_vcpus'] = i
else:
self.facts['processor_count'] = sockets and len(sockets) or i
self.facts['processor_cores'] = sockets.values() and sockets.values()[0] or 1
self.facts['processor_threads_per_core'] = ((cores.values() and
cores.values()[0] or 1) / self.facts['processor_cores'])
self.facts['processor_vcpus'] = (self.facts['processor_threads_per_core'] *
self.facts['processor_count'] * self.facts['processor_cores'])
def get_dmi_facts(self):
''' learn dmi facts from system
@ -1355,7 +1424,7 @@ class HPUX(Hardware):
self.facts['memtotal_mb'] = int(data) / 1024
except AttributeError:
#For systems where memory details aren't sent to syslog or the log has rotated, use parsed
#adb output. Unfortunatley /dev/kmem doesn't have world-read, so this only works as root.
#adb output. Unfortunately /dev/kmem doesn't have world-read, so this only works as root.
if os.access("/dev/kmem", os.R_OK):
rc, out, err = module.run_command("echo 'phys_mem_pages/D' | adb -k /stand/vmunix /dev/kmem | tail -1 | awk '{print $2}'", use_unsafe_shell=True)
if not err:

View file

@ -32,7 +32,7 @@ import pprint
USER_AGENT_PRODUCT="Ansible-gce"
USER_AGENT_VERSION="v1"
def gce_connect(module):
def gce_connect(module, provider=None):
"""Return a Google Cloud Engine connection."""
service_account_email = module.params.get('service_account_email', None)
pem_file = module.params.get('pem_file', None)
@ -71,8 +71,14 @@ def gce_connect(module):
'secrets file.')
return None
# Allow for passing in libcloud Google DNS (e.g, Provider.GOOGLE)
if provider is None:
provider = Provider.GCE
try:
gce = get_driver(Provider.GCE)(service_account_email, pem_file, datacenter=module.params.get('zone'), project=project_id)
gce = get_driver(provider)(service_account_email, pem_file,
datacenter=module.params.get('zone', None),
project=project_id)
gce.connection.user_agent_append("%s/%s" % (
USER_AGENT_PRODUCT, USER_AGENT_VERSION))
except (RuntimeError, ValueError), e:

View file

@ -40,7 +40,7 @@ def add_git_host_key(module, url, accept_hostkey=True, create_dir=True):
""" idempotently add a git url hostkey """
fqdn = get_fqdn(module.params['repo'])
fqdn = get_fqdn(url)
if fqdn:
known_host = check_hostkey(module, fqdn)

View file

@ -252,9 +252,33 @@ class SSLValidationHandler(urllib2.BaseHandler):
except:
self.module.fail_json(msg='Connection to proxy failed')
def detect_no_proxy(self, url):
'''
Detect if the 'no_proxy' environment variable is set and honor those locations.
'''
env_no_proxy = os.environ.get('no_proxy')
if env_no_proxy:
env_no_proxy = env_no_proxy.split(',')
netloc = urlparse.urlparse(url).netloc
for host in env_no_proxy:
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
# Our requested URL matches something in no_proxy, so don't
# use the proxy for this
return False
return True
def http_request(self, req):
tmp_ca_cert_path, paths_checked = self.get_ca_certs()
https_proxy = os.environ.get('https_proxy')
# Detect if 'no_proxy' environment variable is set and if our URL is included
use_proxy = self.detect_no_proxy(req.get_full_url())
if not use_proxy:
# ignore proxy settings for this host request
return req
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if https_proxy:

View file

@ -55,6 +55,15 @@ class ModuleArgsParser:
src: a
dest: b
# extra gross, but also legal. in this case, the args specified
# will act as 'defaults' and will be overriden by any args specified
# in one of the other formats (complex args under the action, or
# parsed from the k=v string
- command: 'pwd'
args:
chdir: '/tmp'
This class has some of the logic to canonicalize these into the form
- module: <module_name>
@ -104,19 +113,24 @@ class ModuleArgsParser:
return (action, args)
def _normalize_parameters(self, thing, action=None):
def _normalize_parameters(self, thing, action=None, additional_args=dict()):
'''
arguments can be fuzzy. Deal with all the forms.
'''
args = dict()
# final args are the ones we'll eventually return, so first update
# them with any additional args specified, which have lower priority
# than those which may be parsed/normalized next
final_args = dict()
if additional_args:
final_args.update(additional_args)
# how we normalize depends if we figured out what the module name is
# yet. If we have already figured it out, it's an 'old style' invocation.
# otherwise, it's not
if action is not None:
args = self._normalize_old_style_args(thing)
args = self._normalize_old_style_args(thing, action)
else:
(action, args) = self._normalize_new_style_args(thing)
@ -124,9 +138,14 @@ class ModuleArgsParser:
if args and 'args' in args:
args = args['args']
return (action, args)
# finally, update the args we're going to return with the ones
# which were normalized above
if args:
final_args.update(args)
def _normalize_old_style_args(self, thing):
return (action, final_args)
def _normalize_old_style_args(self, thing, action):
'''
deals with fuzziness in old-style (action/local_action) module invocations
returns tuple of (module_name, dictionary_args)
@ -144,7 +163,8 @@ class ModuleArgsParser:
args = thing
elif isinstance(thing, string_types):
# form is like: local_action: copy src=a dest=b ... pretty common
args = parse_kv(thing)
check_raw = action in ('command', 'shell', 'script')
args = parse_kv(thing, check_raw=check_raw)
elif isinstance(thing, NoneType):
# this can happen with modules which take no params, like ping:
args = None
@ -180,7 +200,8 @@ class ModuleArgsParser:
elif isinstance(thing, string_types):
# form is like: copy: src=a dest=b ... common shorthand throughout ansible
(action, args) = self._split_module_string(thing)
args = parse_kv(args)
check_raw = action in ('command', 'shell', 'script')
args = parse_kv(args, check_raw=check_raw)
else:
# need a dict or a string, so giving up
@ -206,13 +227,20 @@ class ModuleArgsParser:
# We can have one of action, local_action, or module specified
#
# this is the 'extra gross' scenario detailed above, so we grab
# the args and pass them in as additional arguments, which can/will
# be overwritten via dict updates from the other arg sources below
# FIXME: add test cases for this
additional_args = self._task_ds.get('args', dict())
# action
if 'action' in self._task_ds:
# an old school 'action' statement
thing = self._task_ds['action']
delegate_to = None
action, args = self._normalize_parameters(thing)
action, args = self._normalize_parameters(thing, additional_args=additional_args)
# local_action
if 'local_action' in self._task_ds:
@ -222,7 +250,7 @@ class ModuleArgsParser:
raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds)
thing = self._task_ds.get('local_action', '')
delegate_to = 'localhost'
action, args = self._normalize_parameters(thing)
action, args = self._normalize_parameters(thing, additional_args=additional_args)
# module: <stuff> is the more new-style invocation
@ -234,7 +262,7 @@ class ModuleArgsParser:
raise AnsibleParserError("conflicting action statements", obj=self._task_ds)
action = item
thing = value
action, args = self._normalize_parameters(value, action=action)
action, args = self._normalize_parameters(value, action=action, additional_args=additional_args)
# if we didn't see any module in the task at all, it's not a task really
if action is None:

View file

@ -40,7 +40,20 @@ def parse_kv(args, check_raw=False):
raw_params = []
for x in vargs:
if "=" in x:
k, v = x.split("=", 1)
pos = 0
try:
while True:
pos = x.index('=', pos + 1)
if pos > 0 and x[pos - 1] != '\\':
break
except ValueError:
# ran out of string, but we must have some escaped equals,
# so replace those and append this to the list of raw params
raw_params.append(x.replace('\\=', '='))
continue
k = x[:pos]
v = x[pos + 1:]
# only internal variables can start with an underscore, so
# we don't allow users to set them directy in arguments

View file

@ -32,7 +32,15 @@ class Conditional:
_when = FieldAttribute(isa='list', default=[])
def __init__(self):
def __init__(self, loader=None):
# when used directly, this class needs a loader, but we want to
# make sure we don't trample on the existing one if this class
# is used as a mix-in with a playbook base class
if not hasattr(self, '_loader'):
if loader is None:
raise AnsibleError("a loader must be specified when using Conditional() directly")
else:
self._loader = loader
super(Conditional, self).__init__()
def _validate_when(self, attr, name, value):

View file

@ -34,8 +34,6 @@ from ansible.playbook.conditional import Conditional
from ansible.playbook.role import Role
from ansible.playbook.taggable import Taggable
from ansible.utils.listify import listify_lookup_plugin_terms
class Task(Base, Conditional, Taggable):
"""
@ -199,9 +197,6 @@ class Task(Base, Conditional, Taggable):
super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined)
def _post_validate_loop_args(self, attr, value, all_vars, fail_on_undefined):
return listify_lookup_plugin_terms(value, all_vars, loader=self._loader)
def get_vars(self):
all_vars = self.serialize()
if 'tags' in all_vars:

View file

@ -31,7 +31,7 @@ from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.executor.module_common import ModuleReplacer
from ansible.parsing.utils.jsonify import jsonify
from ansible.plugins import module_loader, shell_loader
from ansible.plugins import shell_loader
from ansible.utils.debug import debug
@ -44,11 +44,12 @@ class ActionBase:
action in use.
'''
def __init__(self, task, connection, connection_info, loader):
def __init__(self, task, connection, connection_info, loader, module_loader):
self._task = task
self._connection = connection
self._connection_info = connection_info
self._loader = loader
self._module_loader = module_loader
self._shell = self.get_shell()
def get_shell(self):
@ -80,9 +81,9 @@ class ActionBase:
# Search module path(s) for named module.
module_suffixes = getattr(self._connection, 'default_suffixes', None)
module_path = module_loader.find_plugin(module_name, module_suffixes, transport=self._connection.get_transport())
module_path = self._module_loader.find_plugin(module_name, module_suffixes, transport=self._connection.get_transport())
if module_path is None:
module_path2 = module_loader.find_plugin('ping', module_suffixes)
module_path2 = self._module_loader.find_plugin('ping', module_suffixes)
if module_path2 is not None:
raise AnsibleError("The module %s was not found in configured module paths" % (module_name))
else:
@ -391,6 +392,10 @@ class ActionBase:
data = json.loads(self._filter_leading_non_json_lines(res['stdout']))
if 'parsed' in data and data['parsed'] == False:
data['msg'] += res['stderr']
# pre-split stdout into lines, if stdout is in the data and there
# isn't already a stdout_lines value there
if 'stdout' in data and 'stdout_lines' not in data:
data['stdout_lines'] = data.get('stdout', '').splitlines()
else:
data = dict()
@ -424,7 +429,6 @@ class ActionBase:
cmd, prompt, success_key = self._connection_info.make_sudo_cmd('/usr/bin/sudo', executable, cmd)
debug("executing the command through the connection")
#rc, stdin, stdout, stderr = self._connection.exec_command(cmd, tmp, executable=executable, in_data=in_data, sudoable=sudoable)
rc, stdin, stdout, stderr = self._connection.exec_command(cmd, tmp, executable=executable, in_data=in_data)
debug("command execution done")

View file

@ -16,6 +16,7 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
from ansible.errors import AnsibleError
from ansible.playbook.conditional import Conditional
from ansible.plugins.action import ActionBase
class ActionModule(ActionBase):
@ -42,9 +43,10 @@ class ActionModule(ActionBase):
# the built in evaluate function. The when has already been evaluated
# by this point, and is not used again, so we don't care about mangling
# that value now
cond = Conditional(loader=self._loader)
for that in thats:
self._task.when = [ that ]
test_result = self._task.evaluate_conditional(all_vars=task_vars)
cond.when = [ that ]
test_result = cond.evaluate_conditional(all_vars=task_vars)
if not test_result:
result = dict(
failed = True,

View file

@ -63,12 +63,10 @@ class ActionModule(ActionBase):
source = parts[0]
args = ' '.join(parts[1:])
# FIXME: need to sort out all the _original_file stuff still
#if '_original_file' in task_vars:
# source = self._loader.path_dwim_relative(inject['_original_file'], 'files', source, self.runner.basedir)
#else:
# source = self._loader.path_dwim(self.runner.basedir, source)
source = self._loader.path_dwim(source)
if self._task._role is not None:
source = self._loader.path_dwim_relative(self._task._role._role_path, 'files', source)
else:
source = self._loader.path_dwim(source)
# transfer the file to a remote tmp location
tmp_src = self._shell.join_path(tmp, os.path.basename(source))

View file

@ -17,10 +17,17 @@
from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.template import Templar
class ActionModule(ActionBase):
TRANSFERS_FILES = False
def run(self, tmp=None, task_vars=dict()):
return dict(changed=True, ansible_facts=self._task.args)
templar = Templar(loader=self._loader, variables=task_vars)
facts = dict()
if self._task.args:
for (k, v) in self._task.args.iteritems():
k = templar.template(k)
facts[k] = v
return dict(changed=True, ansible_facts=facts)

View file

@ -46,23 +46,34 @@ class CallbackModule(CallbackBase):
pass
def runner_on_failed(self, task, result, ignore_errors=False):
self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), result._result), color='red')
self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), json.dumps(result._result, ensure_ascii=False)), color='red')
def runner_on_ok(self, task, result):
msg = "ok: [%s]" % result._host.get_name()
if result._result.get('changed', False):
msg = "changed: [%s]" % result._host.get_name()
color = 'yellow'
else:
msg = "ok: [%s]" % result._host.get_name()
color = 'green'
if self._display._verbosity > 0 or 'verbose_always' in result._result:
indent = None
if 'verbose_always' in result._result:
indent = 4
del result._result['verbose_always']
msg += " => %s" % result._result
self._display.display(msg, color='green')
msg += " => %s" % json.dumps(result._result, indent=indent, ensure_ascii=False)
self._display.display(msg, color=color)
def runner_on_skipped(self, task, result):
msg = "SKIPPED: [%s]" % result._host.get_name()
msg = "skipping: [%s]" % result._host.get_name()
if self._display._verbosity > 0 or 'verbose_always' in result._result:
indent = None
if 'verbose_always' in result._result:
indent = 4
del result._result['verbose_always']
msg += " => %s" % result._result
self._display.display(msg)
msg += " => %s" % json.dumps(result._result, indent=indent, ensure_ascii=False)
self._display.display(msg, color='cyan')
def runner_on_unreachable(self, task, result):
self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), result._result), color='red')

View file

@ -117,7 +117,8 @@ class Connection(ConnectionBase):
#vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
self._display.vvv("%s PUT %s TO %s" % (self._host, in_path, out_path))
if not os.path.exists(in_path):
raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
#raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
raise AnsibleError("file or module does not exist: %s" % in_path)
try:
shutil.copyfile(in_path, out_path)
except shutil.Error:

View file

@ -120,6 +120,7 @@
import os
from ansible.plugins.lookup import LookupBase
from ansible.template import Templar
class LookupModule(LookupBase):
@ -167,22 +168,26 @@ class LookupModule(LookupBase):
else:
total_search = terms
templar = Templar(loader=self._loader, variables=variables)
roledir = variables.get('roledir')
for fn in total_search:
# FIXME: the original file stuff needs to be fixed/implemented
#if variables and '_original_file' in variables:
# # check the templates and vars directories too,
# # if they exist
# for roledir in ('templates', 'vars'):
# path = self._loader.path_dwim(os.path.join(self.basedir, '..', roledir), fn)
# if os.path.exists(path):
# return [path]
fn = templar.template(fn)
if os.path.isabs(fn) and os.path.exists(fn):
return [fn]
else:
if roledir is not None:
# check the templates and vars directories too,if they exist
for subdir in ('templates', 'vars'):
path = self._loader.path_dwim_relative(roledir, subdir, fn)
if os.path.exists(path):
return [path]
# if none of the above were found, just check the
# current filename against the basedir (this will already
# have ../files from runner, if it's a role task
path = self._loader.path_dwim(fn)
if os.path.exists(path):
return [path]
# if none of the above were found, just check the
# current filename against the basedir (this will already
# have ../files from runner, if it's a role task
path = self._loader.path_dwim(fn)
if os.path.exists(path):
return [path]
else:
if skip:
return []

View file

@ -20,5 +20,9 @@ from ansible.plugins.lookup import LookupBase
class LookupModule(LookupBase):
def run(self, terms, **kwargs):
if not isinstance(terms, list):
terms = [ terms ]
return self._flatten(terms)

View file

@ -20,6 +20,7 @@ from re import compile as re_compile, IGNORECASE
from ansible.errors import *
from ansible.parsing.splitter import parse_kv
from ansible.plugins.lookup import LookupBase
from ansible.template import Templar
# shortcut format
NUM = "(0?x?[0-9a-f]+)"
@ -174,15 +175,18 @@ class LookupModule(LookupBase):
if isinstance(terms, basestring):
terms = [ terms ]
templar = Templar(loader=self._loader, variables=variables)
for term in terms:
try:
self.reset() # clear out things for this iteration
term = templar.template(term)
try:
if not self.parse_simple_args(term):
self.parse_kv_args(parse_kv(term))
except Exception, e:
raise AnsibleError("unknown error parsing with_sequence arguments: %r" % term)
raise AnsibleError("unknown error parsing with_sequence arguments: %r. Error was: %s" % (term, e))
self.sanity_check()

View file

@ -32,7 +32,7 @@ class LookupModule(LookupBase):
def __lookup_variabless(self, terms, variables):
results = []
for x in terms:
intermediate = listify_lookup_plugin_terms(x, variables)
intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader)
results.append(intermediate)
return results

View file

@ -29,6 +29,7 @@ from ansible.inventory.group import Group
from ansible.playbook.helpers import compile_block_list
from ansible.playbook.role import ROLE_CACHE
from ansible.plugins import module_loader
from ansible.utils.debug import debug
@ -103,7 +104,7 @@ class StrategyBase:
self._cur_worker = 0
self._pending_results += 1
main_q.put((host, task, self._loader.get_basedir(), task_vars, connection_info), block=False)
main_q.put((host, task, self._loader.get_basedir(), task_vars, connection_info, module_loader), block=False)
except (EOFError, IOError, AssertionError), e:
# most likely an abort
debug("got an error while queuing: %s" % e)
@ -154,20 +155,20 @@ class StrategyBase:
elif result[0] == 'add_host':
task_result = result[1]
new_host_info = task_result._result.get('add_host', dict())
new_host_info = task_result.get('add_host', dict())
self._add_host(new_host_info)
elif result[0] == 'add_group':
task_result = result[1]
host = task_result._host
group_name = task_result._result.get('add_group')
host = result[1]
task_result = result[2]
group_name = task_result.get('add_group')
self._add_group(host, group_name)
elif result[0] == 'notify_handler':
handler_name = result[1]
host = result[2]
host = result[1]
handler_name = result[2]
if host not in self._notified_handlers[handler_name]:
self._notified_handlers[handler_name].append(host)

View file

@ -67,6 +67,11 @@ class StrategyModule(StrategyBase):
# anything to do do for this host
if host_name not in self._tqm._failed_hosts and host_name not in self._tqm._unreachable_hosts and iterator.get_next_task_for_host(host, peek=True):
# FIXME: check task tags, etc. here as we do in linear
# FIXME: handle meta tasks here, which will require a tweak
# to run_handlers so that only the handlers on this host
# are flushed and not all
# set the flag so the outer loop knows we've still found
# some work which needs to be done
work_to_do = True

View file

@ -19,8 +19,8 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.errors import AnsibleError
from ansible.plugins.strategies import StrategyBase
from ansible.utils.debug import debug
class StrategyModule(StrategyBase):
@ -80,12 +80,21 @@ class StrategyModule(StrategyBase):
continue
work_to_do = True
if not callback_sent:
self._callback.playbook_on_task_start(task.get_name(), False)
callback_sent = True
if task.action == 'meta':
# meta tasks store their args in the _raw_params field of args,
# since they do not use k=v pairs, so get that
meta_action = task.args.get('_raw_params')
if meta_action == 'flush_handlers':
self.run_handlers(iterator, connection_info)
else:
raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)
else:
if not callback_sent:
self._callback.playbook_on_task_start(task.get_name(), False)
callback_sent = True
self._blocked_hosts[host.get_name()] = True
self._queue_task(host, task, task_vars, connection_info)
self._blocked_hosts[host.get_name()] = True
self._queue_task(host, task, task_vars, connection_info)
self._process_pending_results()

View file

@ -58,7 +58,6 @@ def listify_lookup_plugin_terms(terms, variables, loader):
if '{' in terms or '[' in terms:
# Jinja2 already evaluated a variable to a list.
# Jinja2-ified list needs to be converted back to a real type
# TODO: something a bit less heavy than eval
return safe_eval(terms)
if isinstance(terms, basestring):

View file

@ -150,7 +150,6 @@ class VariableManager:
# next comes the facts cache and the vars cache, respectively
all_vars = self._merge_dicts(all_vars, self._fact_cache.get(host.get_name(), dict()))
all_vars = self._merge_dicts(all_vars, self._vars_cache.get(host.get_name(), dict()))
if play:
all_vars = self._merge_dicts(all_vars, play.get_vars())
@ -168,6 +167,9 @@ class VariableManager:
for role in play.get_roles():
all_vars = self._merge_dicts(all_vars, role.get_vars())
if host:
all_vars = self._merge_dicts(all_vars, self._vars_cache.get(host.get_name(), dict()))
if task:
if task._role:
all_vars = self._merge_dicts(all_vars, task._role.get_vars())