Skip literal_eval for string filters results in native jinja. (#70988) (#71313)

Fixes #70831

(cherry picked from commit b66d66027e)
This commit is contained in:
Martin Krizek 2020-08-27 19:47:02 +02:00 committed by GitHub
parent 90a8d07f31
commit 02f4fc1a14
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 61 additions and 6 deletions

View file

@ -0,0 +1,2 @@
bugfixes:
- Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831)

View file

@ -1858,7 +1858,7 @@ SHOW_CUSTOM_STATS:
type: bool type: bool
STRING_TYPE_FILTERS: STRING_TYPE_FILTERS:
name: Filters to preserve strings name: Filters to preserve strings
default: [string, to_json, to_nice_json, to_yaml, ppretty, json] default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json]
description: description:
- "This list of filters avoids 'type conversion' when templating variables" - "This list of filters avoids 'type conversion' when templating variables"
- Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example. - Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example.

View file

@ -78,6 +78,7 @@ if C.DEFAULT_JINJA2_NATIVE:
try: try:
from jinja2.nativetypes import NativeEnvironment as Environment from jinja2.nativetypes import NativeEnvironment as Environment
from ansible.template.native_helpers import ansible_native_concat as j2_concat from ansible.template.native_helpers import ansible_native_concat as j2_concat
from ansible.template.native_helpers import NativeJinjaText
USE_JINJA2_NATIVE = True USE_JINJA2_NATIVE = True
except ImportError: except ImportError:
from jinja2 import Environment from jinja2 import Environment
@ -256,6 +257,10 @@ def _unroll_iterator(func):
return list(ret) return list(ret)
return ret return ret
return _update_wrapper(wrapper, func)
def _update_wrapper(wrapper, func):
# This code is duplicated from ``functools.update_wrapper`` from Py3.7. # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
# ``functools.update_wrapper`` was failing when the func was ``functools.partial`` # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
@ -271,6 +276,19 @@ def _unroll_iterator(func):
return wrapper return wrapper
def _wrap_native_text(func):
"""Wrapper function, that intercepts the result of a filter
and wraps it into NativeJinjaText which is then used
in ``ansible_native_concat`` to indicate that it is a text
which should not be passed into ``literal_eval``.
"""
def wrapper(*args, **kwargs):
ret = func(*args, **kwargs)
return NativeJinjaText(ret)
return _update_wrapper(wrapper, func)
class AnsibleUndefined(StrictUndefined): class AnsibleUndefined(StrictUndefined):
''' '''
A custom Undefined class, which returns further Undefined objects on access, A custom Undefined class, which returns further Undefined objects on access,
@ -484,10 +502,13 @@ class JinjaPluginIntercept(MutableMapping):
method_map = getattr(plugin_impl, self._method_map_name) method_map = getattr(plugin_impl, self._method_map_name)
for f in iteritems(method_map()): for func_name, func in iteritems(method_map()):
fq_name = '.'.join((parent_prefix, f[0])) fq_name = '.'.join((parent_prefix, func_name))
# FIXME: detect/warn on intra-collection function name collisions # FIXME: detect/warn on intra-collection function name collisions
self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1]) if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS:
self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
else:
self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
function_impl = self._collection_jinja_func_cache[key] function_impl = self._collection_jinja_func_cache[key]
return function_impl return function_impl
@ -608,6 +629,17 @@ class Templar:
for fp in self._filter_loader.all(): for fp in self._filter_loader.all():
self._filters.update(fp.filters()) self._filters.update(fp.filters())
if USE_JINJA2_NATIVE:
for string_filter in C.STRING_TYPE_FILTERS:
try:
orig_filter = self._filters[string_filter]
except KeyError:
try:
orig_filter = self.environment.filters[string_filter]
except KeyError:
continue
self._filters[string_filter] = _wrap_native_text(orig_filter)
return self._filters.copy() return self._filters.copy()
def _get_tests(self): def _get_tests(self):

View file

@ -14,10 +14,14 @@ from jinja2.runtime import StrictUndefined
from ansible.module_utils._text import to_text from ansible.module_utils._text import to_text
from ansible.module_utils.common.text.converters import container_to_text from ansible.module_utils.common.text.converters import container_to_text
from ansible.module_utils.six import PY2 from ansible.module_utils.six import PY2, text_type
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
class NativeJinjaText(text_type):
pass
def ansible_native_concat(nodes): def ansible_native_concat(nodes):
"""Return a native Python type from the list of compiled nodes. If the """Return a native Python type from the list of compiled nodes. If the
result is a single node, its value is returned. Otherwise, the nodes are result is a single node, its value is returned. Otherwise, the nodes are
@ -50,6 +54,16 @@ def ansible_native_concat(nodes):
# See https://github.com/ansible/ansible/issues/52158 # See https://github.com/ansible/ansible/issues/52158
# We do that only here because it is taken care of by to_text() in the else block below already. # We do that only here because it is taken care of by to_text() in the else block below already.
str(out) str(out)
if isinstance(out, NativeJinjaText):
# Sometimes (e.g. ``| string``) we need to mark variables
# in a special way so that they remain strings and are not
# passed into literal_eval.
# See:
# https://github.com/ansible/ansible/issues/70831
# https://github.com/pallets/jinja/issues/1200
# https://github.com/ansible/ansible/issues/70831#issuecomment-664190894
return out
else: else:
if isinstance(nodes, types.GeneratorType): if isinstance(nodes, types.GeneratorType):
nodes = chain(head, nodes) nodes = chain(head, nodes)

View file

@ -1,17 +1,22 @@
- name: cast things to other things - name: cast things to other things
set_fact: set_fact:
int_to_str: "'{{ i_two }}'" int_to_str: "'{{ i_two }}'"
int_to_str2: "{{ i_two | string }}"
str_to_int: "{{ s_two|int }}" str_to_int: "{{ s_two|int }}"
dict_to_str: "'{{ dict_one }}'" dict_to_str: "'{{ dict_one }}'"
list_to_str: "'{{ list_one }}'" list_to_str: "'{{ list_one }}'"
int_to_bool: "{{ i_one|bool }}" int_to_bool: "{{ i_one|bool }}"
str_true_to_bool: "{{ s_true|bool }}" str_true_to_bool: "{{ s_true|bool }}"
str_false_to_bool: "{{ s_false|bool }}" str_false_to_bool: "{{ s_false|bool }}"
list_to_json_str: "{{ list_one | to_json }}"
list_to_yaml_str: "{{ list_one | to_yaml }}"
- assert: - assert:
that: that:
- 'int_to_str == "2"' - 'int_to_str == "2"'
- 'int_to_str|type_debug in ["str", "unicode"]' - 'int_to_str|type_debug in ["str", "unicode"]'
- 'int_to_str2 == "2"'
- 'int_to_str2|type_debug in ["NativeJinjaText"]'
- 'str_to_int == 2' - 'str_to_int == 2'
- 'str_to_int|type_debug == "int"' - 'str_to_int|type_debug == "int"'
- 'dict_to_str|type_debug in ["str", "unicode"]' - 'dict_to_str|type_debug in ["str", "unicode"]'
@ -22,3 +27,5 @@
- 'str_true_to_bool|type_debug == "bool"' - 'str_true_to_bool|type_debug == "bool"'
- 'str_false_to_bool is sameas false' - 'str_false_to_bool is sameas false'
- 'str_false_to_bool|type_debug == "bool"' - 'str_false_to_bool|type_debug == "bool"'
- 'list_to_json_str|type_debug in ["NativeJinjaText"]'
- 'list_to_yaml_str|type_debug in ["NativeJinjaText"]'

View file

@ -20,4 +20,4 @@
- assert: - assert:
that: that:
- 'const_dunder|type_debug in ["str", "unicode"]' - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'