Allow config to enable native jinja types (#32738)

Co-authored-by: Martin Krizek <martin.krizek@gmail.com>

lib/ansible/config/base.yml

@@ -327,7 +327,7 @@ LOCALHOST_WARNING:
   description:
     - By default Ansible will issue a warning when there are no hosts in the
       inventory.
-    - These warnings can be silenced by adjusting this setting to False. 
+    - These warnings can be silenced by adjusting this setting to False.
   env: [{name: ANSIBLE_LOCALHOST_WARNING}]
   ini:
   - {key: localhost_warning, section: defaults}
@@ -508,7 +508,7 @@ DEFAULT_DEBUG:
   description:
     - "Toggles debug output in Ansible. This is *very* verbose and can hinder
       multiprocessing.  Debug output can also include secret information
-      despite no_log settings being enabled, which means debug mode should not be used in 
+      despite no_log settings being enabled, which means debug mode should not be used in
       production."
   env: [{name: ANSIBLE_DEBUG}]
   ini:
@@ -694,6 +694,16 @@ DEFAULT_JINJA2_EXTENSIONS:
   env: [{name: ANSIBLE_JINJA2_EXTENSIONS}]
   ini:
   - {key: jinja2_extensions, section: defaults}
+DEFAULT_JINJA2_NATIVE:
+  name: Use Jinja2's NativeEnvironment for templating
+  default: False
+  description: This option preserves variable types during template operations. This requires Jinja2 >= 2.10.
+  env: [{name: ANSIBLE_JINJA2_NATIVE}]
+  ini:
+  - {key: jinja2_native, section: defaults}
+  type: boolean
+  yaml: {key: jinja2_native}
+  version_added: 2.7
 DEFAULT_KEEP_REMOTE_FILES:
   name: Keep remote files
   default: False

lib/ansible/template/__init__.py

@@ -37,11 +37,9 @@ try:
 except ImportError:
     from sha import sha as sha1
 
-from jinja2 import Environment
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError
 from jinja2.loaders import FileSystemLoader
 from jinja2.runtime import Context, StrictUndefined
-from jinja2.utils import concat as j2_concat
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError
@@ -70,6 +68,19 @@ NON_TEMPLATED_TYPES = (bool, Number)
 
 JINJA2_OVERRIDE = '#jinja2:'
 
+USE_JINJA2_NATIVE = False
+if C.DEFAULT_JINJA2_NATIVE:
+    try:
+        from jinja2.nativetypes import NativeEnvironment as Environment
+        from ansible.template.native_helpers import ansible_native_concat as j2_concat
+        USE_JINJA2_NATIVE = True
+    except ImportError:
+        from jinja2 import Environment
+        from jinja2.utils import concat as j2_concat
+else:
+    from jinja2 import Environment
+    from jinja2.utils import concat as j2_concat
+
 
 def generate_ansible_template_vars(path):
     b_path = to_bytes(path)
@@ -479,19 +490,20 @@ class Templar:
                             disable_lookups=disable_lookups,
                         )
 
-                        unsafe = hasattr(result, '__UNSAFE__')
-                        if convert_data and not self._no_type_regex.match(variable):
-                            # if this looks like a dictionary or list, convert it to such using the safe_eval method
-                            if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
-                                    result.startswith("[") or result in ("True", "False"):
-                                eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
-                                if eval_results[1] is None:
-                                    result = eval_results[0]
-                                    if unsafe:
-                                        result = wrap_var(result)
-                                else:
-                                    # FIXME: if the safe_eval raised an error, should we do something with it?
-                                    pass
+                        if not USE_JINJA2_NATIVE:
+                            unsafe = hasattr(result, '__UNSAFE__')
+                            if convert_data and not self._no_type_regex.match(variable):
+                                # if this looks like a dictionary or list, convert it to such using the safe_eval method
+                                if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
+                                        result.startswith("[") or result in ("True", "False"):
+                                    eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
+                                    if eval_results[1] is None:
+                                        result = eval_results[0]
+                                        if unsafe:
+                                            result = wrap_var(result)
+                                    else:
+                                        # FIXME: if the safe_eval raised an error, should we do something with it?
+                                        pass
 
                         # we only cache in the case where we have a single variable
                         # name, to make sure we're not putting things which may otherwise
@@ -663,9 +675,15 @@ class Templar:
             raise AnsibleError("lookup plugin (%s) not found" % name)
 
     def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False):
+        if USE_JINJA2_NATIVE and not isinstance(data, string_types):
+            return data
+
         # For preserving the number of input newlines in the output (used
         # later in this method)
-        data_newlines = _count_newlines_from_end(data)
+        if not USE_JINJA2_NATIVE:
+            data_newlines = _count_newlines_from_end(data)
+        else:
+            data_newlines = None
 
         if fail_on_undefined is None:
             fail_on_undefined = self._fail_on_undefined_errors
@@ -678,7 +696,7 @@ class Templar:
                 myenv = self.environment.overlay(overrides)
 
             # Get jinja env overrides from template
-            if data.startswith(JINJA2_OVERRIDE):
+            if hasattr(data, 'startswith') and data.startswith(JINJA2_OVERRIDE):
                 eol = data.find('\n')
                 line = data[len(JINJA2_OVERRIDE):eol]
                 data = data[eol + 1:]
@@ -720,7 +738,7 @@ class Templar:
 
             try:
                 res = j2_concat(rf)
-                if new_context.unsafe:
+                if getattr(new_context, 'unsafe', False):
                     res = wrap_var(res)
             except TypeError as te:
                 if 'StrictUndefined' in to_native(te):
@@ -731,6 +749,9 @@ class Templar:
                     display.debug("failing because of a type error, template data is: %s" % to_native(data))
                     raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
 
+            if USE_JINJA2_NATIVE:
+                return res
+
             if preserve_trailing_newlines:
                 # The low level calls above do not preserve the newline
                 # characters at the end of the input data, so we use the

lib/ansible/template/native_helpers.py

@@ -0,0 +1,44 @@
+# Copyright: (c) 2018, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from ast import literal_eval
+from itertools import islice, chain
+import types
+
+from jinja2._compat import text_type
+
+
+def ansible_native_concat(nodes):
+    """Return a native Python type from the list of compiled nodes. If the
+    result is a single node, its value is returned. Otherwise, the nodes are
+    concatenated as strings. If the result can be parsed with
+    :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
+    string is returned.
+    """
+
+    # https://github.com/pallets/jinja/blob/master/jinja2/nativetypes.py
+
+    head = list(islice(nodes, 2))
+
+    if not head:
+        return None
+
+    if len(head) == 1:
+        out = head[0]
+        # short circuit literal_eval when possible
+        if not isinstance(out, list):  # FIXME is this needed?
+            return out
+    else:
+        if isinstance(nodes, types.GeneratorType):
+            nodes = chain(head, nodes)
+        out = u''.join([text_type(v) for v in nodes])
+
+    try:
+        return literal_eval(out)
+    except (ValueError, SyntaxError, MemoryError):
+        return out

test/integration/targets/jinja2_native_types/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py

@@ -0,0 +1,8 @@
+from ansible.module_utils._text import to_text
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'to_text': to_text,
+        }

test/integration/targets/jinja2_native_types/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types runtests.yml -v "$@"

test/integration/targets/jinja2_native_types/runtests.yml

@@ -0,0 +1,47 @@
+- name: Test jinja2 native types
+  hosts: localhost
+  gather_facts: no
+  vars:
+      i_one: 1
+      i_two: 2
+      i_three: 3
+      s_one: "1"
+      s_two: "2"
+      s_three: "3"
+      dict_one:
+          foo: bar
+          baz: bang
+      dict_two:
+          bar: foo
+          foobar: barfoo
+      list_one:
+          - one
+          - two
+      list_two:
+          - three
+          - four
+      list_ints:
+          - 4
+          - 2
+      list_one_int:
+          - 1
+      b_true: True
+      b_false: False
+      s_true: "True"
+      s_false: "False"
+  tasks:
+      - name: check jinja version
+        shell: python -c 'import jinja2; print(jinja2.__version__)'
+        register: jinja2_version
+
+      - name: make sure jinja is the right version
+        set_fact:
+            is_native: "{{ jinja2_version.stdout is version('2.10', '>=') }}"
+
+      - block:
+            - import_tasks: test_casting.yml
+            - import_tasks: test_concatentation.yml
+            - import_tasks: test_bool.yml
+            - import_tasks: test_dunder.yml
+            - import_tasks: test_types.yml
+        when: is_native

test/integration/targets/jinja2_native_types/test_bool.yml

@@ -0,0 +1,53 @@
+- name: test bool True
+  set_fact:
+      bool_var_true: "{{ b_true }}"
+
+- assert:
+    that:
+        - 'bool_var_true is sameas true'
+        - 'bool_var_true|type_debug == "bool"'
+
+- name: test bool False
+  set_fact:
+      bool_var_false: "{{ b_false }}"
+
+- assert:
+    that:
+        - 'bool_var_false is sameas false'
+        - 'bool_var_false|type_debug == "bool"'
+
+- name: test bool expr True
+  set_fact:
+      bool_var_expr_true: "{{ 1 == 1 }}"
+
+- assert:
+    that:
+        - 'bool_var_expr_true is sameas true'
+        - 'bool_var_expr_true|type_debug == "bool"'
+
+- name: test bool expr False
+  set_fact:
+      bool_var_expr_false: "{{ 2 + 2 == 5 }}"
+
+- assert:
+    that:
+        - 'bool_var_expr_false is sameas false'
+        - 'bool_var_expr_false|type_debug == "bool"'
+
+- name: test bool expr with None, True
+  set_fact:
+      bool_var_none_expr_true: "{{ None == None }}"
+
+- assert:
+    that:
+        - 'bool_var_none_expr_true is sameas true'
+        - 'bool_var_none_expr_true|type_debug == "bool"'
+
+- name: test bool expr with None, False
+  set_fact:
+      bool_var_none_expr_false: "{{ '' == None }}"
+
+- assert:
+    that:
+        - 'bool_var_none_expr_false is sameas false'
+        - 'bool_var_none_expr_false|type_debug == "bool"'

test/integration/targets/jinja2_native_types/test_casting.yml

@@ -0,0 +1,24 @@
+- name: cast things to other things
+  set_fact:
+      int_to_str: "{{ i_two|to_text }}"
+      str_to_int: "{{ s_two|int }}"
+      dict_to_str: "{{ dict_one|to_text }}"
+      list_to_str: "{{ list_one|to_text }}"
+      int_to_bool: "{{ i_one|bool }}"
+      str_true_to_bool: "{{ s_true|bool }}"
+      str_false_to_bool: "{{ s_false|bool }}"
+
+- assert:
+    that:
+        - 'int_to_str == "2"'
+        - 'int_to_str|type_debug in ["string", "unicode"]'
+        - 'str_to_int == 2'
+        - 'str_to_int|type_debug == "int"'
+        - 'dict_to_str|type_debug in ["string", "unicode"]'
+        - 'list_to_str|type_debug in ["string", "unicode"]'
+        - 'int_to_bool is sameas true'
+        - 'int_to_bool|type_debug == "bool"'
+        - 'str_true_to_bool is sameas true'
+        - 'str_true_to_bool|type_debug == "bool"'
+        - 'str_false_to_bool is sameas false'
+        - 'str_false_to_bool|type_debug == "bool"'

test/integration/targets/jinja2_native_types/test_concatentation.yml

@@ -0,0 +1,88 @@
+- name: add two ints
+  set_fact:
+      integer_sum: "{{ i_one + i_two }}"
+
+- assert:
+    that:
+        - 'integer_sum == 3'
+        - 'integer_sum|type_debug == "int"'
+
+- name: add casted string and int
+  set_fact:
+      integer_sum2: "{{ s_one|int + i_two }}"
+
+- assert:
+    that:
+        - 'integer_sum2 == 3'
+        - 'integer_sum2|type_debug == "int"'
+
+- name: concatenate int and string
+  set_fact:
+      string_sum: "{{ [(i_one|to_text), s_two]|join('') }}"
+
+- assert:
+    that:
+        - 'string_sum == "12"'
+        - 'string_sum|type_debug in ["string", "unicode"]'
+
+- name: add two lists
+  set_fact:
+      list_sum: "{{ list_one + list_two }}"
+
+- assert:
+    that:
+        - 'list_sum == ["one", "two", "three", "four"]'
+        - 'list_sum|type_debug == "list"'
+
+- name: add two lists, multi expression
+  set_fact:
+      list_sum_multi: "{{ list_one }} + {{ list_two }}"
+
+- assert:
+    that:
+        - 'list_sum_multi|type_debug in ["string", "unicode"]'
+
+- name: add two dicts
+  set_fact:
+      dict_sum: "{{ dict_one + dict_two }}"
+  ignore_errors: yes
+
+- assert:
+    that:
+        - 'dict_sum is undefined'
+
+- name: loop through list with strings
+  set_fact:
+      list_for_strings: "{% for x in list_one %}{{ x }}{% endfor %}"
+
+- assert:
+    that:
+        - 'list_for_strings == "onetwo"'
+        - 'list_for_strings|type_debug in ["string", "unicode"]'
+
+- name: loop through list with int
+  set_fact:
+      list_for_int: "{% for x in list_one_int %}{{ x }}{% endfor %}"
+
+- assert:
+    that:
+        - 'list_for_int == 1'
+        - 'list_for_int|type_debug == "int"'
+
+- name: loop through list with ints
+  set_fact:
+      list_for_ints: "{% for x in list_ints %}{{ x }}{% endfor %}"
+
+- assert:
+    that:
+        - 'list_for_ints == 42'
+        - 'list_for_ints|type_debug == "int"'
+
+- name: loop through list to create a new list
+  set_fact:
+      list_from_list: "[{% for x in list_ints %}{{ x }},{% endfor %}]"
+
+- assert:
+    that:
+        - 'list_from_list == [4, 2]'
+        - 'list_from_list|type_debug == "list"'

test/integration/targets/jinja2_native_types/test_dunder.yml

@@ -0,0 +1,23 @@
+- name: test variable dunder
+  set_fact:
+      var_dunder: "{{ b_true.__class__ }}"
+
+- assert:
+    that:
+        - 'var_dunder|type_debug == "type"'
+
+- name: test constant dunder
+  set_fact:
+      const_dunder: "{{ true.__class__ }}"
+
+- assert:
+    that:
+        - 'const_dunder|type_debug == "type"'
+
+- name: test constant dunder to string
+  set_fact:
+      const_dunder: "{{ true.__class__|string }}"
+
+- assert:
+    that:
+        - 'const_dunder|type_debug in ["string", "unicode"]'

test/integration/targets/jinja2_native_types/test_types.yml

@@ -0,0 +1,20 @@
+- assert:
+    that:
+        - 'i_one|type_debug == "int"'
+        - 's_one|type_debug == "AnsibleUnicode"'
+        - 'dict_one|type_debug == "dict"'
+        - 'dict_one is mapping'
+        - 'list_one|type_debug == "list"'
+        - 'b_true|type_debug == "bool"'
+        - 's_true|type_debug == "AnsibleUnicode"'
+
+- set_fact:
+    a_list: "{{[i_one, s_two]}}"
+
+- assert:
+    that:
+        - 'a_list|type_debug == "list"'
+        - 'a_list[0] == 1'
+        - 'a_list[0]|type_debug == "int"'
+        - 'a_list[1] == "2"'
+        - 'a_list[1]|type_debug == "AnsibleUnicode"'