From 02f4fc1a145f71d2368307739749e35d18a447e4 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 27 Aug 2020 19:47:02 +0200 Subject: [PATCH] Skip literal_eval for string filters results in native jinja. (#70988) (#71313) Fixes #70831 (cherry picked from commit b66d66027ece03f3f0a3fdb5fd6b8213965a2f1d) --- ...iteral_eval-string-filter-native-jinja.yml | 2 + lib/ansible/config/base.yml | 2 +- lib/ansible/template/__init__.py | 38 +++++++++++++++++-- lib/ansible/template/native_helpers.py | 16 +++++++- .../jinja2_native_types/test_casting.yml | 7 ++++ .../jinja2_native_types/test_dunder.yml | 2 +- 6 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml diff --git a/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml new file mode 100644 index 00000000000..40b426e50bf --- /dev/null +++ b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml @@ -0,0 +1,2 @@ +bugfixes: + - Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831) diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 4901ba15f5a..789749a626f 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1858,7 +1858,7 @@ SHOW_CUSTOM_STATS: type: bool STRING_TYPE_FILTERS: name: Filters to preserve strings - default: [string, to_json, to_nice_json, to_yaml, ppretty, json] + default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json] description: - "This list of filters avoids 'type conversion' when templating variables" - Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example. diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 603e8df3a32..2184050aaab 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -78,6 +78,7 @@ if C.DEFAULT_JINJA2_NATIVE: try: from jinja2.nativetypes import NativeEnvironment as Environment from ansible.template.native_helpers import ansible_native_concat as j2_concat + from ansible.template.native_helpers import NativeJinjaText USE_JINJA2_NATIVE = True except ImportError: from jinja2 import Environment @@ -256,6 +257,10 @@ def _unroll_iterator(func): return list(ret) return ret + return _update_wrapper(wrapper, func) + + +def _update_wrapper(wrapper, func): # This code is duplicated from ``functools.update_wrapper`` from Py3.7. # ``functools.update_wrapper`` was failing when the func was ``functools.partial`` for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): @@ -271,6 +276,19 @@ def _unroll_iterator(func): return wrapper +def _wrap_native_text(func): + """Wrapper function, that intercepts the result of a filter + and wraps it into NativeJinjaText which is then used + in ``ansible_native_concat`` to indicate that it is a text + which should not be passed into ``literal_eval``. + """ + def wrapper(*args, **kwargs): + ret = func(*args, **kwargs) + return NativeJinjaText(ret) + + return _update_wrapper(wrapper, func) + + class AnsibleUndefined(StrictUndefined): ''' A custom Undefined class, which returns further Undefined objects on access, @@ -484,10 +502,13 @@ class JinjaPluginIntercept(MutableMapping): method_map = getattr(plugin_impl, self._method_map_name) - for f in iteritems(method_map()): - fq_name = '.'.join((parent_prefix, f[0])) + for func_name, func in iteritems(method_map()): + fq_name = '.'.join((parent_prefix, func_name)) # FIXME: detect/warn on intra-collection function name collisions - self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1]) + if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS: + self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) + else: + self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) function_impl = self._collection_jinja_func_cache[key] return function_impl @@ -608,6 +629,17 @@ class Templar: for fp in self._filter_loader.all(): self._filters.update(fp.filters()) + if USE_JINJA2_NATIVE: + for string_filter in C.STRING_TYPE_FILTERS: + try: + orig_filter = self._filters[string_filter] + except KeyError: + try: + orig_filter = self.environment.filters[string_filter] + except KeyError: + continue + self._filters[string_filter] = _wrap_native_text(orig_filter) + return self._filters.copy() def _get_tests(self): diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py index 1ff92092bd5..21766d3479d 100644 --- a/lib/ansible/template/native_helpers.py +++ b/lib/ansible/template/native_helpers.py @@ -14,10 +14,14 @@ from jinja2.runtime import StrictUndefined from ansible.module_utils._text import to_text from ansible.module_utils.common.text.converters import container_to_text -from ansible.module_utils.six import PY2 +from ansible.module_utils.six import PY2, text_type from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +class NativeJinjaText(text_type): + pass + + def ansible_native_concat(nodes): """Return a native Python type from the list of compiled nodes. If the result is a single node, its value is returned. Otherwise, the nodes are @@ -50,6 +54,16 @@ def ansible_native_concat(nodes): # See https://github.com/ansible/ansible/issues/52158 # We do that only here because it is taken care of by to_text() in the else block below already. str(out) + + if isinstance(out, NativeJinjaText): + # Sometimes (e.g. ``| string``) we need to mark variables + # in a special way so that they remain strings and are not + # passed into literal_eval. + # See: + # https://github.com/ansible/ansible/issues/70831 + # https://github.com/pallets/jinja/issues/1200 + # https://github.com/ansible/ansible/issues/70831#issuecomment-664190894 + return out else: if isinstance(nodes, types.GeneratorType): nodes = chain(head, nodes) diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml index e66489ff5b1..8627a0563c9 100644 --- a/test/integration/targets/jinja2_native_types/test_casting.yml +++ b/test/integration/targets/jinja2_native_types/test_casting.yml @@ -1,17 +1,22 @@ - name: cast things to other things set_fact: int_to_str: "'{{ i_two }}'" + int_to_str2: "{{ i_two | string }}" str_to_int: "{{ s_two|int }}" dict_to_str: "'{{ dict_one }}'" list_to_str: "'{{ list_one }}'" int_to_bool: "{{ i_one|bool }}" str_true_to_bool: "{{ s_true|bool }}" str_false_to_bool: "{{ s_false|bool }}" + list_to_json_str: "{{ list_one | to_json }}" + list_to_yaml_str: "{{ list_one | to_yaml }}" - assert: that: - 'int_to_str == "2"' - 'int_to_str|type_debug in ["str", "unicode"]' + - 'int_to_str2 == "2"' + - 'int_to_str2|type_debug in ["NativeJinjaText"]' - 'str_to_int == 2' - 'str_to_int|type_debug == "int"' - 'dict_to_str|type_debug in ["str", "unicode"]' @@ -22,3 +27,5 @@ - 'str_true_to_bool|type_debug == "bool"' - 'str_false_to_bool is sameas false' - 'str_false_to_bool|type_debug == "bool"' + - 'list_to_json_str|type_debug in ["NativeJinjaText"]' + - 'list_to_yaml_str|type_debug in ["NativeJinjaText"]' diff --git a/test/integration/targets/jinja2_native_types/test_dunder.yml b/test/integration/targets/jinja2_native_types/test_dunder.yml index 46fd4d0a90e..df5ea9276b7 100644 --- a/test/integration/targets/jinja2_native_types/test_dunder.yml +++ b/test/integration/targets/jinja2_native_types/test_dunder.yml @@ -20,4 +20,4 @@ - assert: that: - - 'const_dunder|type_debug in ["str", "unicode"]' + - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'