From cc948f339c1befc95820b40ac8f5fb574a3cae2e Mon Sep 17 00:00:00 2001
From: Daniel Hokka Zakrisson <daniel@hozac.com>
Date: Mon, 17 Sep 2012 14:02:30 +0200
Subject: [PATCH] Allow including files through variables
$FILE{file} will be replaced with the contents of "file"
$PIPE{cat file} will be replaced with the output of "cat file"
---
lib/ansible/playbook/__init__.py | 2 +-
lib/ansible/playbook/play.py | 16 +++----
lib/ansible/playbook/task.py | 4 +-
lib/ansible/runner/__init__.py | 10 ++--
lib/ansible/runner/action_plugins/copy.py | 4 +-
lib/ansible/runner/action_plugins/fetch.py | 4 +-
lib/ansible/runner/action_plugins/template.py | 4 +-
lib/ansible/utils.py | 46 +++++++++++++++++--
test/TestUtils.py | 18 +++++++-
9 files changed, 79 insertions(+), 29 deletions(-)
diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py
index 53d72001105..29bd2743a08 100644
--- a/lib/ansible/playbook/__init__.py
+++ b/lib/ansible/playbook/__init__.py
@@ -267,7 +267,7 @@ class PlayBook(object):
for host, results in results.get('contacted',{}).iteritems():
if results.get('changed', False):
for handler_name in task.notify:
- self._flag_handler(play.handlers(), utils.template(handler_name, task.module_vars), host)
+ self._flag_handler(play.handlers(), utils.template(play.basedir, handler_name, task.module_vars), host)
# *****************************************************
diff --git a/lib/ansible/playbook/play.py b/lib/ansible/playbook/play.py
index 1b507df0ecf..e2b57064e7b 100644
--- a/lib/ansible/playbook/play.py
+++ b/lib/ansible/playbook/play.py
@@ -57,7 +57,7 @@ class Play(object):
raise errors.AnsibleError('hosts declaration is required')
elif isinstance(hosts, list):
hosts = ';'.join(hosts)
- hosts = utils.template(hosts, playbook.extra_vars)
+ hosts = utils.template(basedir, hosts, playbook.extra_vars)
self._ds = ds
self.playbook = playbook
self.basedir = basedir
@@ -69,7 +69,7 @@ class Play(object):
self.vars = self._get_vars()
self._tasks = ds.get('tasks', [])
self._handlers = ds.get('handlers', [])
- self.remote_user = utils.template(ds.get('user', self.playbook.remote_user), playbook.extra_vars)
+ self.remote_user = utils.template(basedir, ds.get('user', self.playbook.remote_user), playbook.extra_vars)
self.remote_port = ds.get('port', self.playbook.remote_port)
self.sudo = ds.get('sudo', self.playbook.sudo)
self.sudo_user = ds.get('sudo_user', self.playbook.sudo_user)
@@ -106,8 +106,8 @@ class Play(object):
tokens = shlex.split(x['include'])
for t in tokens[1:]:
(k,v) = t.split("=", 1)
- task_vars[k] = utils.template(v, task_vars)
- include_file = utils.template(tokens[0], task_vars)
+ task_vars[k] = utils.template(self.basedir, v, task_vars)
+ include_file = utils.template(self.basedir, tokens[0], task_vars)
data = utils.parse_yaml_from_file(utils.path_dwim(self.basedir, include_file))
elif type(x) == dict:
data = [x]
@@ -261,10 +261,10 @@ class Play(object):
found = False
sequence = []
for real_filename in filename:
- filename2 = utils.template(real_filename, self.vars)
+ filename2 = utils.template(self.basedir, real_filename, self.vars)
filename3 = filename2
if host is not None:
- filename3 = utils.template(filename2, self.playbook.SETUP_CACHE[host])
+ filename3 = utils.template(self.basedir, filename2, self.playbook.SETUP_CACHE[host])
filename4 = utils.path_dwim(self.basedir, filename3)
sequence.append(filename4)
if os.path.exists(filename4):
@@ -294,10 +294,10 @@ class Play(object):
else:
# just one filename supplied, load it!
- filename2 = utils.template(filename, self.vars)
+ filename2 = utils.template(self.basedir, filename, self.vars)
filename3 = filename2
if host is not None:
- filename3 = utils.template(filename2, self.playbook.SETUP_CACHE[host])
+ filename3 = utils.template(self.basedir, filename2, self.playbook.SETUP_CACHE[host])
filename4 = utils.path_dwim(self.basedir, filename3)
if self._has_vars_in(filename4):
return
diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py
index e5b660f2c95..69f62f4796f 100644
--- a/lib/ansible/playbook/task.py
+++ b/lib/ansible/playbook/task.py
@@ -103,8 +103,8 @@ class Task(object):
# allow the user to list comma delimited tags
import_tags = import_tags.split(",")
- self.name = utils.template(self.name, self.module_vars)
- self.action = utils.template(self.action, self.module_vars)
+ self.name = utils.template(None, self.name, self.module_vars)
+ self.action = utils.template(None, self.action, self.module_vars)
# handle mutually incompatible options
if self.with_items is not None and self.first_available_file is not None:
diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py
index 8f983b87aa2..266d80cbe1d 100644
--- a/lib/ansible/runner/__init__.py
+++ b/lib/ansible/runner/__init__.py
@@ -200,7 +200,7 @@ class Runner(object):
cmd = ""
if not is_new_style:
- args = utils.template(args, inject)
+ args = utils.template(self.basedir, args, inject)
argsfile = self._transfer_str(conn, tmp, 'arguments', args)
if async_jid is None:
cmd = "%s %s" % (remote_module_path, argsfile)
@@ -327,13 +327,13 @@ class Runner(object):
for (k,v) in self.module_args.iteritems():
new_args = new_args + "%s='%s' " % (k,v)
self.module_args = new_args
- self.module_args = utils.template(self.module_args, inject)
+ self.module_args = utils.template(self.basedir, self.module_args, inject)
def _check_conditional(conditional):
def is_set(var):
return not var.startswith("$")
return eval(conditional)
- conditional = utils.template(self.conditional, inject)
+ conditional = utils.template(self.basedir, self.conditional, inject)
if not _check_conditional(conditional):
result = utils.jsonify(dict(skipped=True))
self.callbacks.on_skipped(host, inject.get('item',None))
@@ -352,7 +352,7 @@ class Runner(object):
result = dict(failed=True, msg="FAILED: %s" % str(e))
return ReturnData(host=host, comm_ok=False, result=result)
- module_name = utils.template(self.module_name, inject)
+ module_name = utils.template(self.basedir, self.module_name, inject)
tmp = ''
if self.module_name != 'raw':
@@ -499,7 +499,7 @@ class Runner(object):
if module_common.REPLACER in module_data:
is_new_style=True
module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
- encoded_args = "\"\"\"%s\"\"\"" % utils.template(self.module_args, inject).replace("\"","\\\"")
+ encoded_args = "\"\"\"%s\"\"\"" % utils.template(self.basedir, self.module_args, inject).replace("\"","\\\"")
module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
# use the correct python interpreter for the host
diff --git a/lib/ansible/runner/action_plugins/copy.py b/lib/ansible/runner/action_plugins/copy.py
index 2747ad959be..812a2484c86 100644
--- a/lib/ansible/runner/action_plugins/copy.py
+++ b/lib/ansible/runner/action_plugins/copy.py
@@ -48,7 +48,7 @@ class ActionModule(object):
if 'first_available_file' in inject:
found = False
for fn in inject.get('first_available_file'):
- fn = utils.template(fn, inject)
+ fn = utils.template(self.runner.basedir, fn, inject)
if os.path.exists(fn):
source = fn
found = True
@@ -57,7 +57,7 @@ class ActionModule(object):
results=dict(failed=True, msg="could not find src in first_available_file list")
return ReturnData(conn=conn, results=results)
- source = utils.template(source, inject)
+ source = utils.template(self.runner.basedir, source, inject)
source = utils.path_dwim(self.runner.basedir, source)
local_md5 = utils.md5(source)
diff --git a/lib/ansible/runner/action_plugins/fetch.py b/lib/ansible/runner/action_plugins/fetch.py
index a487094bbc6..b0c81614515 100644
--- a/lib/ansible/runner/action_plugins/fetch.py
+++ b/lib/ansible/runner/action_plugins/fetch.py
@@ -44,9 +44,9 @@ class ActionModule(object):
return ReturnData(conn=conn, result=results)
# apply templating to source argument
- source = utils.template(source, inject)
+ source = utils.template(self.runner.basedir, source, inject)
# apply templating to dest argument
- dest = utils.template(dest, inject)
+ dest = utils.template(self.runner.basedir, dest, inject)
# files are saved in dest dir, with a subdir for each host, then the filename
dest = "%s/%s/%s" % (utils.path_dwim(self.runner.basedir, dest), conn.host, source)
diff --git a/lib/ansible/runner/action_plugins/template.py b/lib/ansible/runner/action_plugins/template.py
index 61f2da14718..91a50bdede5 100644
--- a/lib/ansible/runner/action_plugins/template.py
+++ b/lib/ansible/runner/action_plugins/template.py
@@ -51,7 +51,7 @@ class ActionModule(object):
if 'first_available_file' in inject:
found = False
for fn in self.runner.module_vars.get('first_available_file'):
- fn = utils.template(fn, inject)
+ fn = utils.template(self.runner.basedir, fn, inject)
if os.path.exists(fn):
source = fn
found = True
@@ -60,7 +60,7 @@ class ActionModule(object):
result = dict(failed=True, msg="could not find src in first_available_file list")
return ReturnData(conn=conn, comm_ok=False, result=result)
- source = utils.template(source, inject)
+ source = utils.template(self.runner.basedir, source, inject)
# template the source data locally & transfer
try:
diff --git a/lib/ansible/utils.py b/lib/ansible/utils.py
index 107d7061f92..3ebf5b38464 100644
--- a/lib/ansible/utils.py
+++ b/lib/ansible/utils.py
@@ -31,6 +31,7 @@ import time
import StringIO
import imp
import glob
+import subprocess
VERBOSITY=0
@@ -182,7 +183,7 @@ def _varLookup(name, vars):
_KEYCRE = re.compile(r"\$(?P<complex>\{){0,1}((?(complex)[\w\.\[\]]+|\w+))(?(complex)\})")
def varLookup(varname, vars):
- ''' helper function used by varReplace '''
+ ''' helper function used by with_items '''
m = _KEYCRE.search(varname)
if not m:
@@ -206,10 +207,9 @@ def varReplace(raw, vars):
# Determine replacement value (if unknown variable then preserve
# original)
- varname = m.group(2)
try:
- replacement = unicode(_varLookup(varname, vars))
+ replacement = unicode(_varLookup(m.group(2), vars))
except VarNotFoundException:
replacement = m.group()
@@ -220,7 +220,42 @@ def varReplace(raw, vars):
return ''.join(done)
-def template(text, vars):
+_FILEPIPECRE = re.compile(r"\$(?P<special>FILE|PIPE)\{([^\}]+)\}")
+def varReplaceFilesAndPipes(basedir, raw):
+ done = [] # Completed chunks to return
+
+ while raw:
+ m = _FILEPIPECRE.search(raw)
+ if not m:
+ done.append(raw)
+ break
+
+ # Determine replacement value (if unknown variable then preserve
+ # original)
+
+ if m.group(1) == "FILE":
+ try:
+ f = open(path_dwim(basedir, m.group(2)), "r")
+ except IOError:
+ raise VarNotFoundException()
+ replacement = f.read()
+ f.close()
+ elif m.group(1) == "PIPE":
+ p = subprocess.Popen(m.group(2), shell=True, stdout=subprocess.PIPE)
+ (stdout, stderr) = p.communicate()
+ if p.returncode != 0:
+ raise VarNotFoundException()
+ replacement = stdout
+
+ start, end = m.span()
+ done.append(raw[:start]) # Keep stuff leading up to token
+ done.append(replacement) # Append replacement value
+ raw = raw[end:] # Continue with remainder of string
+
+ return ''.join(done)
+
+
+def template(basedir, text, vars):
''' run a text buffer through the templating engine until it no longer changes '''
prev_text = ''
@@ -235,6 +270,7 @@ def template(text, vars):
raise errors.AnsibleError("template recursion depth exceeded")
prev_text = text
text = varReplace(unicode(text), vars)
+ text = varReplaceFilesAndPipes(basedir, text)
return text
def template_from_file(basedir, path, vars):
@@ -251,7 +287,7 @@ def template_from_file(basedir, path, vars):
res = t.render(vars)
if data.endswith('\n') and not res.endswith('\n'):
res = res + '\n'
- return template(res, vars)
+ return template(basedir, res, vars)
def parse_yaml(data):
''' convert a yaml string to a data structure '''
diff --git a/test/TestUtils.py b/test/TestUtils.py
index 00eee68fcfc..1f5c9b6e74c 100644
--- a/test/TestUtils.py
+++ b/test/TestUtils.py
@@ -16,7 +16,7 @@ class TestUtils(unittest.TestCase):
}
}
- res = ansible.utils._varLookup('data.who', vars)
+ res = ansible.utils.varLookup('${data.who}', vars)
assert sorted(res) == sorted(vars['data']['who'])
@@ -209,10 +209,24 @@ class TestUtils(unittest.TestCase):
'person': 'one',
}
- res = ansible.utils.template(template, vars)
+ res = ansible.utils.template(None, template, vars)
assert res == u'hello oh great one'
+ def test_varReplace_include(self):
+ template = 'hello $FILE{world}'
+
+ res = ansible.utils.template("test", template, {})
+
+ assert res == u'hello world\n'
+
+ def test_varReplace_include_script(self):
+ template = 'hello $PIPE{echo world}'
+
+ res = ansible.utils.template("test", template, {})
+
+ assert res == u'hello world\n'
+
#####################################
### Template function tests