Skip literal_eval for string filters results in native jinja. (#70988)

Fixes #70831
This commit is contained in:
Martin Krizek 2020-08-11 10:19:49 +02:00 committed by GitHub
parent 7195788ffe
commit b66d66027e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 61 additions and 6 deletions

View file

@ -0,0 +1,2 @@
bugfixes:
- Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831)

View file

@ -1837,7 +1837,7 @@ SHOW_CUSTOM_STATS:
type: bool
STRING_TYPE_FILTERS:
name: Filters to preserve strings
default: [string, to_json, to_nice_json, to_yaml, ppretty, json]
default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json]
description:
- "This list of filters avoids 'type conversion' when templating variables"
- Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example.

View file

@ -78,6 +78,7 @@ if C.DEFAULT_JINJA2_NATIVE:
try:
from jinja2.nativetypes import NativeEnvironment as Environment
from ansible.template.native_helpers import ansible_native_concat as j2_concat
from ansible.template.native_helpers import NativeJinjaText
USE_JINJA2_NATIVE = True
except ImportError:
from jinja2 import Environment
@ -256,6 +257,10 @@ def _unroll_iterator(func):
return list(ret)
return ret
return _update_wrapper(wrapper, func)
def _update_wrapper(wrapper, func):
# This code is duplicated from ``functools.update_wrapper`` from Py3.7.
# ``functools.update_wrapper`` was failing when the func was ``functools.partial``
for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
@ -271,6 +276,19 @@ def _unroll_iterator(func):
return wrapper
def _wrap_native_text(func):
"""Wrapper function, that intercepts the result of a filter
and wraps it into NativeJinjaText which is then used
in ``ansible_native_concat`` to indicate that it is a text
which should not be passed into ``literal_eval``.
"""
def wrapper(*args, **kwargs):
ret = func(*args, **kwargs)
return NativeJinjaText(ret)
return _update_wrapper(wrapper, func)
class AnsibleUndefined(StrictUndefined):
'''
A custom Undefined class, which returns further Undefined objects on access,
@ -488,10 +506,13 @@ class JinjaPluginIntercept(MutableMapping):
method_map = getattr(plugin_impl, self._method_map_name)
for f in iteritems(method_map()):
fq_name = '.'.join((parent_prefix, f[0]))
for func_name, func in iteritems(method_map()):
fq_name = '.'.join((parent_prefix, func_name))
# FIXME: detect/warn on intra-collection function name collisions
self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1])
if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS:
self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
else:
self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
function_impl = self._collection_jinja_func_cache[key]
return function_impl
@ -612,6 +633,17 @@ class Templar:
for fp in self._filter_loader.all():
self._filters.update(fp.filters())
if USE_JINJA2_NATIVE:
for string_filter in C.STRING_TYPE_FILTERS:
try:
orig_filter = self._filters[string_filter]
except KeyError:
try:
orig_filter = self.environment.filters[string_filter]
except KeyError:
continue
self._filters[string_filter] = _wrap_native_text(orig_filter)
return self._filters.copy()
def _get_tests(self):

View file

@ -15,10 +15,14 @@ from jinja2.runtime import StrictUndefined
from ansible.module_utils._text import to_text
from ansible.module_utils.common.collections import is_sequence, Mapping
from ansible.module_utils.common.text.converters import container_to_text
from ansible.module_utils.six import PY2
from ansible.module_utils.six import PY2, text_type
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
class NativeJinjaText(text_type):
pass
def _fail_on_undefined(data):
"""Recursively find an undefined value in a nested data structure
and properly raise the undefined exception.
@ -62,6 +66,16 @@ def ansible_native_concat(nodes):
# TODO send unvaulted data to literal_eval?
if isinstance(out, AnsibleVaultEncryptedUnicode):
return out.data
if isinstance(out, NativeJinjaText):
# Sometimes (e.g. ``| string``) we need to mark variables
# in a special way so that they remain strings and are not
# passed into literal_eval.
# See:
# https://github.com/ansible/ansible/issues/70831
# https://github.com/pallets/jinja/issues/1200
# https://github.com/ansible/ansible/issues/70831#issuecomment-664190894
return out
else:
if isinstance(nodes, types.GeneratorType):
nodes = chain(head, nodes)

View file

@ -1,17 +1,22 @@
- name: cast things to other things
set_fact:
int_to_str: "'{{ i_two }}'"
int_to_str2: "{{ i_two | string }}"
str_to_int: "{{ s_two|int }}"
dict_to_str: "'{{ dict_one }}'"
list_to_str: "'{{ list_one }}'"
int_to_bool: "{{ i_one|bool }}"
str_true_to_bool: "{{ s_true|bool }}"
str_false_to_bool: "{{ s_false|bool }}"
list_to_json_str: "{{ list_one | to_json }}"
list_to_yaml_str: "{{ list_one | to_yaml }}"
- assert:
that:
- 'int_to_str == "2"'
- 'int_to_str|type_debug in ["str", "unicode"]'
- 'int_to_str2 == "2"'
- 'int_to_str2|type_debug in ["NativeJinjaText"]'
- 'str_to_int == 2'
- 'str_to_int|type_debug == "int"'
- 'dict_to_str|type_debug in ["str", "unicode"]'
@ -22,3 +27,5 @@
- 'str_true_to_bool|type_debug == "bool"'
- 'str_false_to_bool is sameas false'
- 'str_false_to_bool|type_debug == "bool"'
- 'list_to_json_str|type_debug in ["NativeJinjaText"]'
- 'list_to_yaml_str|type_debug in ["NativeJinjaText"]'

View file

@ -20,4 +20,4 @@
- assert:
that:
- 'const_dunder|type_debug in ["str", "unicode"]'
- 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'