Merge pull request #1053 from dhozac/varreplace-include

Allow including files through variables

lib/ansible/playbook/__init__.py

@@ -265,7 +265,7 @@ class PlayBook(object):
             for host, results in results.get('contacted',{}).iteritems():
                 if results.get('changed', False):
                     for handler_name in task.notify:
-                        self._flag_handler(play.handlers(), utils.template(handler_name, task.module_vars), host)
+                        self._flag_handler(play.handlers(), utils.template(play.basedir, handler_name, task.module_vars), host)
 
     # *****************************************************
 

lib/ansible/playbook/play.py

@@ -57,7 +57,7 @@ class Play(object):
             raise errors.AnsibleError('hosts declaration is required')
         elif isinstance(hosts, list):
             hosts = ';'.join(hosts)
-        hosts = utils.template(hosts, playbook.extra_vars)
+        hosts = utils.template(basedir, hosts, playbook.extra_vars)
         self._ds          = ds
         self.playbook     = playbook
         self.basedir      = basedir
@@ -69,7 +69,7 @@ class Play(object):
         self.vars         = self._get_vars()
         self._tasks       = ds.get('tasks', [])
         self._handlers    = ds.get('handlers', [])
-        self.remote_user  = utils.template(ds.get('user', self.playbook.remote_user), playbook.extra_vars)
+        self.remote_user  = utils.template(basedir, ds.get('user', self.playbook.remote_user), playbook.extra_vars)
         self.remote_port  = ds.get('port', self.playbook.remote_port)
         self.sudo         = ds.get('sudo', self.playbook.sudo)
         self.sudo_user    = ds.get('sudo_user', self.playbook.sudo_user)
@@ -106,8 +106,8 @@ class Play(object):
                 tokens = shlex.split(x['include'])
                 for t in tokens[1:]:
                     (k,v) = t.split("=", 1)
-                    task_vars[k] = utils.template(v, task_vars)
+                    task_vars[k] = utils.template(self.basedir, v, task_vars)
-                include_file = utils.template(tokens[0], task_vars)
+                include_file = utils.template(self.basedir, tokens[0], task_vars)
                 data = utils.parse_yaml_from_file(utils.path_dwim(self.basedir, include_file))
             elif type(x) == dict:
                 data = [x]
@@ -261,10 +261,10 @@ class Play(object):
                 found = False
                 sequence = []
                 for real_filename in filename:
-                    filename2 = utils.template(real_filename, self.vars)
+                    filename2 = utils.template(self.basedir, real_filename, self.vars)
                     filename3 = filename2
                     if host is not None:
-                        filename3 = utils.template(filename2, self.playbook.SETUP_CACHE[host])
+                        filename3 = utils.template(self.basedir, filename2, self.playbook.SETUP_CACHE[host])
                     filename4 = utils.path_dwim(self.basedir, filename3)
                     sequence.append(filename4)
                     if os.path.exists(filename4):
@@ -294,10 +294,10 @@ class Play(object):
             else:
                 # just one filename supplied, load it!
 
-                filename2 = utils.template(filename, self.vars)
+                filename2 = utils.template(self.basedir, filename, self.vars)
                 filename3 = filename2
                 if host is not None:
-                    filename3 = utils.template(filename2, self.playbook.SETUP_CACHE[host])
+                    filename3 = utils.template(self.basedir, filename2, self.playbook.SETUP_CACHE[host])
                 filename4 = utils.path_dwim(self.basedir, filename3)
                 if self._has_vars_in(filename4):
                     return

lib/ansible/playbook/task.py

@@ -103,8 +103,8 @@ class Task(object):
             # allow the user to list comma delimited tags
             import_tags = import_tags.split(",")
 
-        self.name = utils.template(self.name, self.module_vars)
+        self.name = utils.template(None, self.name, self.module_vars)
-        self.action = utils.template(self.action, self.module_vars)
+        self.action = utils.template(None, self.action, self.module_vars)
 
         # handle mutually incompatible options
         if self.with_items is not None and self.first_available_file is not None:

lib/ansible/runner/__init__.py

@@ -200,7 +200,7 @@ class Runner(object):
 
         cmd = ""
         if not is_new_style:
-            args = utils.template(args, inject)
+            args = utils.template(self.basedir, args, inject)
             argsfile = self._transfer_str(conn, tmp, 'arguments', args)
             if async_jid is None:
                 cmd = "%s %s" % (remote_module_path, argsfile)
@@ -258,7 +258,7 @@ class Runner(object):
 
         items = self.module_vars.get('items', [])
         if isinstance(items, basestring) and items.startswith("$"):
-            items = utils.varLookup(items, inject)
+            items = utils.varLookup(self.basedir, items, inject)
         if type(items) != list:
             raise errors.AnsibleError("with_items only takes a list: %s" % items)
 
@@ -327,13 +327,13 @@ class Runner(object):
             for (k,v) in self.module_args.iteritems():
                 new_args = new_args + "%s='%s' " % (k,v)
             self.module_args = new_args
-        self.module_args = utils.template(self.module_args, inject)
+        self.module_args = utils.template(self.basedir, self.module_args, inject)
 
         def _check_conditional(conditional):
             def is_set(var):
                 return not var.startswith("$")
             return eval(conditional)
-        conditional = utils.template(self.conditional, inject)
+        conditional = utils.template(self.basedir, self.conditional, inject)
         if not _check_conditional(conditional):
             result = utils.jsonify(dict(skipped=True))
             self.callbacks.on_skipped(host, inject.get('item',None))
@@ -352,7 +352,7 @@ class Runner(object):
             result = dict(failed=True, msg="FAILED: %s" % str(e))
             return ReturnData(host=host, comm_ok=False, result=result)
 
-        module_name = utils.template(self.module_name, inject)
+        module_name = utils.template(self.basedir, self.module_name, inject)
 
         tmp = ''
         if self.module_name != 'raw':
@@ -499,7 +499,7 @@ class Runner(object):
             if module_common.REPLACER in module_data:
                 is_new_style=True
             module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON)
-            encoded_args = "\"\"\"%s\"\"\"" % utils.template(self.module_args, inject).replace("\"","\\\"")
+            encoded_args = "\"\"\"%s\"\"\"" % utils.template(self.basedir, self.module_args, inject).replace("\"","\\\"")
             module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args)
 
         # use the correct python interpreter for the host

lib/ansible/runner/action_plugins/copy.py

@@ -48,7 +48,7 @@ class ActionModule(object):
         if 'first_available_file' in inject:
             found = False
             for fn in inject.get('first_available_file'):
-                fn = utils.template(fn, inject)
+                fn = utils.template(self.runner.basedir, fn, inject)
                 if os.path.exists(fn):
                     source = fn
                     found = True
@@ -57,7 +57,7 @@ class ActionModule(object):
                 results=dict(failed=True, msg="could not find src in first_available_file list")
                 return ReturnData(conn=conn, results=results)
 
-        source = utils.template(source, inject)
+        source = utils.template(self.runner.basedir, source, inject)
         source = utils.path_dwim(self.runner.basedir, source)
 
         local_md5 = utils.md5(source)

lib/ansible/runner/action_plugins/fetch.py

@@ -44,9 +44,9 @@ class ActionModule(object):
             return ReturnData(conn=conn, result=results)
 
         # apply templating to source argument
-        source = utils.template(source, inject)
+        source = utils.template(self.runner.basedir, source, inject)
         # apply templating to dest argument
-        dest = utils.template(dest, inject)
+        dest = utils.template(self.runner.basedir, dest, inject)
 
         # files are saved in dest dir, with a subdir for each host, then the filename
         dest   = "%s/%s/%s" % (utils.path_dwim(self.runner.basedir, dest), conn.host, source)

lib/ansible/runner/action_plugins/template.py

@@ -51,7 +51,7 @@ class ActionModule(object):
         if 'first_available_file' in inject:
             found = False
             for fn in self.runner.module_vars.get('first_available_file'):
-                fn = utils.template(fn, inject)
+                fn = utils.template(self.runner.basedir, fn, inject)
                 if os.path.exists(fn):
                     source = fn
                     found = True
@@ -60,7 +60,7 @@ class ActionModule(object):
                 result = dict(failed=True, msg="could not find src in first_available_file list")
                 return ReturnData(conn=conn, comm_ok=False, result=result)
 
-        source = utils.template(source, inject)
+        source = utils.template(self.runner.basedir, source, inject)
 
         # template the source data locally & transfer
         try:

lib/ansible/utils.py

@@ -31,6 +31,7 @@ import time
 import StringIO
 import imp
 import glob
+import subprocess
 
 VERBOSITY=0
 
@@ -182,7 +183,7 @@ def _varLookup(name, vars):
 _KEYCRE = re.compile(r"\$(?P<complex>\{){0,1}((?(complex)[\w\.\[\]]+|\w+))(?(complex)\})")
 
 def varLookup(varname, vars):
-    ''' helper function used by varReplace '''
+    ''' helper function used by with_items '''
 
     m = _KEYCRE.search(varname)
     if not m:
@@ -206,10 +207,9 @@ def varReplace(raw, vars):
 
         # Determine replacement value (if unknown variable then preserve
         # original)
-        varname = m.group(2)
 
         try:
-            replacement = unicode(_varLookup(varname, vars))
+            replacement = unicode(_varLookup(m.group(2), vars))
         except VarNotFoundException:
             replacement = m.group()
 
@@ -220,7 +220,42 @@ def varReplace(raw, vars):
 
     return ''.join(done)
 
-def template(text, vars):
+_FILEPIPECRE = re.compile(r"\$(?P<special>FILE|PIPE)\{([^\}]+)\}")
+def varReplaceFilesAndPipes(basedir, raw):
+    done = [] # Completed chunks to return
+
+    while raw:
+        m = _FILEPIPECRE.search(raw)
+        if not m:
+            done.append(raw)
+            break
+
+        # Determine replacement value (if unknown variable then preserve
+        # original)
+
+        if m.group(1) == "FILE":
+            try:
+                f = open(path_dwim(basedir, m.group(2)), "r")
+            except IOError:
+                raise VarNotFoundException()
+            replacement = f.read()
+            f.close()
+        elif m.group(1) == "PIPE":
+            p = subprocess.Popen(m.group(2), shell=True, stdout=subprocess.PIPE)
+            (stdout, stderr) = p.communicate()
+            if p.returncode != 0:
+                raise VarNotFoundException()
+            replacement = stdout
+
+        start, end = m.span()
+        done.append(raw[:start])    # Keep stuff leading up to token
+        done.append(replacement)    # Append replacement value
+        raw = raw[end:]             # Continue with remainder of string
+
+    return ''.join(done)
+
+
+def template(basedir, text, vars):
     ''' run a text buffer through the templating engine until it no longer changes '''
 
     prev_text = ''
@@ -235,6 +270,7 @@ def template(text, vars):
             raise errors.AnsibleError("template recursion depth exceeded")
         prev_text = text
         text = varReplace(unicode(text), vars)
+    text = varReplaceFilesAndPipes(basedir, text)
     return text
 
 def template_from_file(basedir, path, vars):
@@ -251,7 +287,7 @@ def template_from_file(basedir, path, vars):
     res = t.render(vars)
     if data.endswith('\n') and not res.endswith('\n'):
         res = res + '\n'
-    return template(res, vars)
+    return template(basedir, res, vars)
 
 def parse_yaml(data):
     ''' convert a yaml string to a data structure '''

test/TestUtils.py

@@ -16,7 +16,7 @@ class TestUtils(unittest.TestCase):
             }
         }
 
-        res = ansible.utils._varLookup('data.who', vars)
+        res = ansible.utils.varLookup('${data.who}', vars)
 
         assert sorted(res) == sorted(vars['data']['who'])
 
@@ -209,10 +209,24 @@ class TestUtils(unittest.TestCase):
             'person': 'one',
         }
 
-        res = ansible.utils.template(template, vars)
+        res = ansible.utils.template(None, template, vars)
 
         assert res == u'hello oh great one'
 
+    def test_varReplace_include(self):
+        template = 'hello $FILE{world}'
+
+        res = ansible.utils.template("test", template, {})
+
+        assert res == u'hello world\n'
+
+    def test_varReplace_include_script(self):
+        template = 'hello $PIPE{echo world}'
+
+        res = ansible.utils.template("test", template, {})
+
+        assert res == u'hello world\n'
+
     #####################################
     ### Template function tests
 

test/world

@@ -0,0 +1 @@
+world