Skip literal_eval for string filters results in native jinja. (#70988)

Fixes #70831

changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831)

lib/ansible/config/base.yml

@@ -1837,7 +1837,7 @@ SHOW_CUSTOM_STATS:
   type: bool
 STRING_TYPE_FILTERS:
   name: Filters to preserve strings
-  default: [string, to_json, to_nice_json, to_yaml, ppretty, json]
+  default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json]
   description:
     - "This list of filters avoids 'type conversion' when templating variables"
     - Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example.

lib/ansible/template/__init__.py

@@ -78,6 +78,7 @@ if C.DEFAULT_JINJA2_NATIVE:
     try:
         from jinja2.nativetypes import NativeEnvironment as Environment
         from ansible.template.native_helpers import ansible_native_concat as j2_concat
+        from ansible.template.native_helpers import NativeJinjaText
         USE_JINJA2_NATIVE = True
     except ImportError:
         from jinja2 import Environment
@@ -256,6 +257,10 @@ def _unroll_iterator(func):
             return list(ret)
         return ret
 
+    return _update_wrapper(wrapper, func)
+
+
+def _update_wrapper(wrapper, func):
     # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
     # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
     for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
@@ -271,6 +276,19 @@ def _unroll_iterator(func):
     return wrapper
 
 
+def _wrap_native_text(func):
+    """Wrapper function, that intercepts the result of a filter
+    and wraps it into NativeJinjaText which is then used
+    in ``ansible_native_concat`` to indicate that it is a text
+    which should not be passed into ``literal_eval``.
+    """
+    def wrapper(*args, **kwargs):
+        ret = func(*args, **kwargs)
+        return NativeJinjaText(ret)
+
+    return _update_wrapper(wrapper, func)
+
+
 class AnsibleUndefined(StrictUndefined):
     '''
     A custom Undefined class, which returns further Undefined objects on access,
@@ -488,10 +506,13 @@ class JinjaPluginIntercept(MutableMapping):
 
                 method_map = getattr(plugin_impl, self._method_map_name)
 
-                for f in iteritems(method_map()):
-                    fq_name = '.'.join((parent_prefix, f[0]))
+                for func_name, func in iteritems(method_map()):
+                    fq_name = '.'.join((parent_prefix, func_name))
                     # FIXME: detect/warn on intra-collection function name collisions
-                    self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1])
+                    if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS:
+                        self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
+                    else:
+                        self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
 
             function_impl = self._collection_jinja_func_cache[key]
             return function_impl
@@ -612,6 +633,17 @@ class Templar:
         for fp in self._filter_loader.all():
             self._filters.update(fp.filters())
 
+        if USE_JINJA2_NATIVE:
+            for string_filter in C.STRING_TYPE_FILTERS:
+                try:
+                    orig_filter = self._filters[string_filter]
+                except KeyError:
+                    try:
+                        orig_filter = self.environment.filters[string_filter]
+                    except KeyError:
+                        continue
+                self._filters[string_filter] = _wrap_native_text(orig_filter)
+
         return self._filters.copy()
 
     def _get_tests(self):

lib/ansible/template/native_helpers.py

@@ -15,10 +15,14 @@ from jinja2.runtime import StrictUndefined
 from ansible.module_utils._text import to_text
 from ansible.module_utils.common.collections import is_sequence, Mapping
 from ansible.module_utils.common.text.converters import container_to_text
-from ansible.module_utils.six import PY2
+from ansible.module_utils.six import PY2, text_type
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 
 
+class NativeJinjaText(text_type):
+    pass
+
+
 def _fail_on_undefined(data):
     """Recursively find an undefined value in a nested data structure
     and properly raise the undefined exception.
@@ -62,6 +66,16 @@ def ansible_native_concat(nodes):
         # TODO send unvaulted data to literal_eval?
         if isinstance(out, AnsibleVaultEncryptedUnicode):
             return out.data
+
+        if isinstance(out, NativeJinjaText):
+            # Sometimes (e.g. ``| string``) we need to mark variables
+            # in a special way so that they remain strings and are not
+            # passed into literal_eval.
+            # See:
+            # https://github.com/ansible/ansible/issues/70831
+            # https://github.com/pallets/jinja/issues/1200
+            # https://github.com/ansible/ansible/issues/70831#issuecomment-664190894
+            return out
     else:
         if isinstance(nodes, types.GeneratorType):
             nodes = chain(head, nodes)

test/integration/targets/jinja2_native_types/test_casting.yml

@@ -1,17 +1,22 @@
 - name: cast things to other things
   set_fact:
       int_to_str: "'{{ i_two }}'"
+      int_to_str2: "{{ i_two | string }}"
       str_to_int: "{{ s_two|int }}"
       dict_to_str: "'{{ dict_one }}'"
       list_to_str: "'{{ list_one }}'"
       int_to_bool: "{{ i_one|bool }}"
       str_true_to_bool: "{{ s_true|bool }}"
       str_false_to_bool: "{{ s_false|bool }}"
+      list_to_json_str: "{{ list_one | to_json }}"
+      list_to_yaml_str: "{{ list_one | to_yaml }}"
 
 - assert:
     that:
         - 'int_to_str == "2"'
         - 'int_to_str|type_debug in ["str", "unicode"]'
+        - 'int_to_str2 == "2"'
+        - 'int_to_str2|type_debug in ["NativeJinjaText"]'
         - 'str_to_int == 2'
         - 'str_to_int|type_debug == "int"'
         - 'dict_to_str|type_debug in ["str", "unicode"]'
@@ -22,3 +27,5 @@
         - 'str_true_to_bool|type_debug == "bool"'
         - 'str_false_to_bool is sameas false'
         - 'str_false_to_bool|type_debug == "bool"'
+        - 'list_to_json_str|type_debug in ["NativeJinjaText"]'
+        - 'list_to_yaml_str|type_debug in ["NativeJinjaText"]'

test/integration/targets/jinja2_native_types/test_dunder.yml

@@ -20,4 +20,4 @@
 
 - assert:
     that:
-        - 'const_dunder|type_debug in ["str", "unicode"]'
+        - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'