From b66d66027ece03f3f0a3fdb5fd6b8213965a2f1d Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 11 Aug 2020 10:19:49 +0200 Subject: [PATCH] Skip literal_eval for string filters results in native jinja. (#70988) Fixes #70831 --- ...iteral_eval-string-filter-native-jinja.yml | 2 + lib/ansible/config/base.yml | 2 +- lib/ansible/template/__init__.py | 38 +++++++++++++++++-- lib/ansible/template/native_helpers.py | 16 +++++++- .../jinja2_native_types/test_casting.yml | 7 ++++ .../jinja2_native_types/test_dunder.yml | 2 +- 6 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml diff --git a/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml new file mode 100644 index 00000000000..40b426e50bf --- /dev/null +++ b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml @@ -0,0 +1,2 @@ +bugfixes: + - Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831) diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index f739905bde6..0b9e504e2f9 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1837,7 +1837,7 @@ SHOW_CUSTOM_STATS: type: bool STRING_TYPE_FILTERS: name: Filters to preserve strings - default: [string, to_json, to_nice_json, to_yaml, ppretty, json] + default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json] description: - "This list of filters avoids 'type conversion' when templating variables" - Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example. diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 2f9e9004bf4..e19c91ff6ab 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -78,6 +78,7 @@ if C.DEFAULT_JINJA2_NATIVE: try: from jinja2.nativetypes import NativeEnvironment as Environment from ansible.template.native_helpers import ansible_native_concat as j2_concat + from ansible.template.native_helpers import NativeJinjaText USE_JINJA2_NATIVE = True except ImportError: from jinja2 import Environment @@ -256,6 +257,10 @@ def _unroll_iterator(func): return list(ret) return ret + return _update_wrapper(wrapper, func) + + +def _update_wrapper(wrapper, func): # This code is duplicated from ``functools.update_wrapper`` from Py3.7. # ``functools.update_wrapper`` was failing when the func was ``functools.partial`` for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'): @@ -271,6 +276,19 @@ def _unroll_iterator(func): return wrapper +def _wrap_native_text(func): + """Wrapper function, that intercepts the result of a filter + and wraps it into NativeJinjaText which is then used + in ``ansible_native_concat`` to indicate that it is a text + which should not be passed into ``literal_eval``. + """ + def wrapper(*args, **kwargs): + ret = func(*args, **kwargs) + return NativeJinjaText(ret) + + return _update_wrapper(wrapper, func) + + class AnsibleUndefined(StrictUndefined): ''' A custom Undefined class, which returns further Undefined objects on access, @@ -488,10 +506,13 @@ class JinjaPluginIntercept(MutableMapping): method_map = getattr(plugin_impl, self._method_map_name) - for f in iteritems(method_map()): - fq_name = '.'.join((parent_prefix, f[0])) + for func_name, func in iteritems(method_map()): + fq_name = '.'.join((parent_prefix, func_name)) # FIXME: detect/warn on intra-collection function name collisions - self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1]) + if USE_JINJA2_NATIVE and func_name in C.STRING_TYPE_FILTERS: + self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) + else: + self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) function_impl = self._collection_jinja_func_cache[key] return function_impl @@ -612,6 +633,17 @@ class Templar: for fp in self._filter_loader.all(): self._filters.update(fp.filters()) + if USE_JINJA2_NATIVE: + for string_filter in C.STRING_TYPE_FILTERS: + try: + orig_filter = self._filters[string_filter] + except KeyError: + try: + orig_filter = self.environment.filters[string_filter] + except KeyError: + continue + self._filters[string_filter] = _wrap_native_text(orig_filter) + return self._filters.copy() def _get_tests(self): diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py index 5ed44599c1b..81bef436d96 100644 --- a/lib/ansible/template/native_helpers.py +++ b/lib/ansible/template/native_helpers.py @@ -15,10 +15,14 @@ from jinja2.runtime import StrictUndefined from ansible.module_utils._text import to_text from ansible.module_utils.common.collections import is_sequence, Mapping from ansible.module_utils.common.text.converters import container_to_text -from ansible.module_utils.six import PY2 +from ansible.module_utils.six import PY2, text_type from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +class NativeJinjaText(text_type): + pass + + def _fail_on_undefined(data): """Recursively find an undefined value in a nested data structure and properly raise the undefined exception. @@ -62,6 +66,16 @@ def ansible_native_concat(nodes): # TODO send unvaulted data to literal_eval? if isinstance(out, AnsibleVaultEncryptedUnicode): return out.data + + if isinstance(out, NativeJinjaText): + # Sometimes (e.g. ``| string``) we need to mark variables + # in a special way so that they remain strings and are not + # passed into literal_eval. + # See: + # https://github.com/ansible/ansible/issues/70831 + # https://github.com/pallets/jinja/issues/1200 + # https://github.com/ansible/ansible/issues/70831#issuecomment-664190894 + return out else: if isinstance(nodes, types.GeneratorType): nodes = chain(head, nodes) diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml index e66489ff5b1..8627a0563c9 100644 --- a/test/integration/targets/jinja2_native_types/test_casting.yml +++ b/test/integration/targets/jinja2_native_types/test_casting.yml @@ -1,17 +1,22 @@ - name: cast things to other things set_fact: int_to_str: "'{{ i_two }}'" + int_to_str2: "{{ i_two | string }}" str_to_int: "{{ s_two|int }}" dict_to_str: "'{{ dict_one }}'" list_to_str: "'{{ list_one }}'" int_to_bool: "{{ i_one|bool }}" str_true_to_bool: "{{ s_true|bool }}" str_false_to_bool: "{{ s_false|bool }}" + list_to_json_str: "{{ list_one | to_json }}" + list_to_yaml_str: "{{ list_one | to_yaml }}" - assert: that: - 'int_to_str == "2"' - 'int_to_str|type_debug in ["str", "unicode"]' + - 'int_to_str2 == "2"' + - 'int_to_str2|type_debug in ["NativeJinjaText"]' - 'str_to_int == 2' - 'str_to_int|type_debug == "int"' - 'dict_to_str|type_debug in ["str", "unicode"]' @@ -22,3 +27,5 @@ - 'str_true_to_bool|type_debug == "bool"' - 'str_false_to_bool is sameas false' - 'str_false_to_bool|type_debug == "bool"' + - 'list_to_json_str|type_debug in ["NativeJinjaText"]' + - 'list_to_yaml_str|type_debug in ["NativeJinjaText"]' diff --git a/test/integration/targets/jinja2_native_types/test_dunder.yml b/test/integration/targets/jinja2_native_types/test_dunder.yml index 46fd4d0a90e..df5ea9276b7 100644 --- a/test/integration/targets/jinja2_native_types/test_dunder.yml +++ b/test/integration/targets/jinja2_native_types/test_dunder.yml @@ -20,4 +20,4 @@ - assert: that: - - 'const_dunder|type_debug in ["str", "unicode"]' + - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'