From ce7b95499fb8f7789fd233d1c2c334657aa0c3c4 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Fri, 28 Aug 2020 14:57:17 +0200 Subject: [PATCH] native types: properly handle Undefined in nested data (#68432) (#71105) (cherry picked from commit 5ca3aec3c4bd03d8619325034ba12a34293276ef) --- ...types-properly-handle-nested-undefined.yml | 2 + lib/ansible/template/native_helpers.py | 46 +++++++++++-------- .../inventory.jinja2_native_types | 1 - .../jinja2_native_types/nested_undefined.yml | 24 ++++++++++ .../targets/jinja2_native_types/runme.sh | 9 ++-- .../jinja2_native_types/test_hostvars.yml | 2 +- 6 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 changelogs/fragments/native-jinja2-types-properly-handle-nested-undefined.yml delete mode 100644 test/integration/targets/jinja2_native_types/inventory.jinja2_native_types create mode 100644 test/integration/targets/jinja2_native_types/nested_undefined.yml diff --git a/changelogs/fragments/native-jinja2-types-properly-handle-nested-undefined.yml b/changelogs/fragments/native-jinja2-types-properly-handle-nested-undefined.yml new file mode 100644 index 00000000000..1e24fb98c8d --- /dev/null +++ b/changelogs/fragments/native-jinja2-types-properly-handle-nested-undefined.yml @@ -0,0 +1,2 @@ +bugfixes: + - native jinja2 types - properly handle Undefined in nested data. diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py index 21766d3479d..81bef436d96 100644 --- a/lib/ansible/template/native_helpers.py +++ b/lib/ansible/template/native_helpers.py @@ -13,6 +13,7 @@ import types from jinja2.runtime import StrictUndefined from ansible.module_utils._text import to_text +from ansible.module_utils.common.collections import is_sequence, Mapping from ansible.module_utils.common.text.converters import container_to_text from ansible.module_utils.six import PY2, text_type from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode @@ -22,39 +23,50 @@ class NativeJinjaText(text_type): pass +def _fail_on_undefined(data): + """Recursively find an undefined value in a nested data structure + and properly raise the undefined exception. + """ + if isinstance(data, Mapping): + for value in data.values(): + _fail_on_undefined(value) + elif is_sequence(data): + for item in data: + _fail_on_undefined(item) + else: + if isinstance(data, StrictUndefined): + # To actually raise the undefined exception we need to + # access the undefined object otherwise the exception would + # be raised on the next access which might not be properly + # handled. + # See https://github.com/ansible/ansible/issues/52158 + # and StrictUndefined implementation in upstream Jinja2. + str(data) + + return data + + def ansible_native_concat(nodes): """Return a native Python type from the list of compiled nodes. If the result is a single node, its value is returned. Otherwise, the nodes are concatenated as strings. If the result can be parsed with :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the string is returned. + + https://github.com/pallets/jinja/blob/master/src/jinja2/nativetypes.py """ - - # https://github.com/pallets/jinja/blob/master/jinja2/nativetypes.py - head = list(islice(nodes, 2)) if not head: return None if len(head) == 1: - out = head[0] + out = _fail_on_undefined(head[0]) # TODO send unvaulted data to literal_eval? if isinstance(out, AnsibleVaultEncryptedUnicode): return out.data - if isinstance(out, StrictUndefined): - # A hack to raise proper UndefinedError/AnsibleUndefinedVariable exception. - # We need to access the AnsibleUndefined(StrictUndefined) object by either of the following: - # __iter__, __str__, __len__, __nonzero__, __eq__, __ne__, __bool__, __hash__ - # to actually raise the exception. - # (see Jinja2 source of StrictUndefined to get up to date info) - # Otherwise the undefined error would be raised on the next access which might not be properly handled. - # See https://github.com/ansible/ansible/issues/52158 - # We do that only here because it is taken care of by to_text() in the else block below already. - str(out) - if isinstance(out, NativeJinjaText): # Sometimes (e.g. ``| string``) we need to mark variables # in a special way so that they remain strings and are not @@ -67,9 +79,7 @@ def ansible_native_concat(nodes): else: if isinstance(nodes, types.GeneratorType): nodes = chain(head, nodes) - # Stringifying the nodes is important as it takes care of - # StrictUndefined by side-effect - by raising an exception. - out = u''.join([to_text(v) for v in nodes]) + out = u''.join([to_text(_fail_on_undefined(v)) for v in nodes]) try: out = literal_eval(out) diff --git a/test/integration/targets/jinja2_native_types/inventory.jinja2_native_types b/test/integration/targets/jinja2_native_types/inventory.jinja2_native_types deleted file mode 100644 index c1ed0a24cfe..00000000000 --- a/test/integration/targets/jinja2_native_types/inventory.jinja2_native_types +++ /dev/null @@ -1 +0,0 @@ -host1 diff --git a/test/integration/targets/jinja2_native_types/nested_undefined.yml b/test/integration/targets/jinja2_native_types/nested_undefined.yml new file mode 100644 index 00000000000..c808ffb73cd --- /dev/null +++ b/test/integration/targets/jinja2_native_types/nested_undefined.yml @@ -0,0 +1,24 @@ +- hosts: localhost + gather_facts: no + tasks: + - block: + - name: Test nested undefined var fails, single node + debug: + msg: "{{ [{ 'key': nested_and_undefined }] }}" + register: result + ignore_errors: yes + + - assert: + that: + - "\"'nested_and_undefined' is undefined\" in result.msg" + + - name: Test nested undefined var fails, multiple nodes + debug: + msg: "{{ [{ 'key': nested_and_undefined}] }} second_node" + register: result + ignore_errors: yes + + - assert: + that: + - "\"'nested_and_undefined' is undefined\" in result.msg" + when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=') diff --git a/test/integration/targets/jinja2_native_types/runme.sh b/test/integration/targets/jinja2_native_types/runme.sh index 0326780b4b3..f648f875021 100755 --- a/test/integration/targets/jinja2_native_types/runme.sh +++ b/test/integration/targets/jinja2_native_types/runme.sh @@ -2,6 +2,9 @@ set -eux -ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types runtests.yml -v "$@" -ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types --vault-password-file test_vault_pass test_vault.yml -v "$@" -ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types test_hostvars.yml -v "$@" +export ANSIBLE_JINJA2_NATIVE=1 +ansible-playbook runtests.yml -v "$@" +ansible-playbook --vault-password-file test_vault_pass test_vault.yml -v "$@" +ansible-playbook test_hostvars.yml -v "$@" +ansible-playbook nested_undefined.yml -v "$@" +unset ANSIBLE_JINJA2_NATIVE diff --git a/test/integration/targets/jinja2_native_types/test_hostvars.yml b/test/integration/targets/jinja2_native_types/test_hostvars.yml index 31f5ec9d3ac..ef0047b863e 100644 --- a/test/integration/targets/jinja2_native_types/test_hostvars.yml +++ b/test/integration/targets/jinja2_native_types/test_hostvars.yml @@ -1,4 +1,4 @@ -- hosts: host1 +- hosts: localhost gather_facts: no tasks: - name: Print vars