native types: properly handle Undefined in nested data (#68432) (#71105)

(cherry picked from commit 5ca3aec3c4)

changelogs/fragments/native-jinja2-types-properly-handle-nested-undefined.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - native jinja2 types - properly handle Undefined in nested data.

lib/ansible/template/native_helpers.py

@@ -13,6 +13,7 @@ import types
 from jinja2.runtime import StrictUndefined
 
 from ansible.module_utils._text import to_text
+from ansible.module_utils.common.collections import is_sequence, Mapping
 from ansible.module_utils.common.text.converters import container_to_text
 from ansible.module_utils.six import PY2, text_type
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
@@ -22,39 +23,50 @@ class NativeJinjaText(text_type):
     pass
 
 
+def _fail_on_undefined(data):
+    """Recursively find an undefined value in a nested data structure
+    and properly raise the undefined exception.
+    """
+    if isinstance(data, Mapping):
+        for value in data.values():
+            _fail_on_undefined(value)
+    elif is_sequence(data):
+        for item in data:
+            _fail_on_undefined(item)
+    else:
+        if isinstance(data, StrictUndefined):
+            # To actually raise the undefined exception we need to
+            # access the undefined object otherwise the exception would
+            # be raised on the next access which might not be properly
+            # handled.
+            # See https://github.com/ansible/ansible/issues/52158
+            # and StrictUndefined implementation in upstream Jinja2.
+            str(data)
+
+    return data
+
+
 def ansible_native_concat(nodes):
     """Return a native Python type from the list of compiled nodes. If the
     result is a single node, its value is returned. Otherwise, the nodes are
     concatenated as strings. If the result can be parsed with
     :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
     string is returned.
+
+    https://github.com/pallets/jinja/blob/master/src/jinja2/nativetypes.py
     """
-
-    # https://github.com/pallets/jinja/blob/master/jinja2/nativetypes.py
-
     head = list(islice(nodes, 2))
 
     if not head:
         return None
 
     if len(head) == 1:
-        out = head[0]
+        out = _fail_on_undefined(head[0])
 
         # TODO send unvaulted data to literal_eval?
         if isinstance(out, AnsibleVaultEncryptedUnicode):
             return out.data
 
-        if isinstance(out, StrictUndefined):
-            # A hack to raise proper UndefinedError/AnsibleUndefinedVariable exception.
-            # We need to access the AnsibleUndefined(StrictUndefined) object by either of the following:
-            # __iter__, __str__, __len__, __nonzero__, __eq__, __ne__, __bool__, __hash__
-            # to actually raise the exception.
-            # (see Jinja2 source of StrictUndefined to get up to date info)
-            # Otherwise the undefined error would be raised on the next access which might not be properly handled.
-            # See https://github.com/ansible/ansible/issues/52158
-            # We do that only here because it is taken care of by to_text() in the else block below already.
-            str(out)
-
         if isinstance(out, NativeJinjaText):
             # Sometimes (e.g. ``| string``) we need to mark variables
             # in a special way so that they remain strings and are not
@@ -67,9 +79,7 @@ def ansible_native_concat(nodes):
     else:
         if isinstance(nodes, types.GeneratorType):
             nodes = chain(head, nodes)
-        # Stringifying the nodes is important as it takes care of
-        # StrictUndefined by side-effect - by raising an exception.
-        out = u''.join([to_text(v) for v in nodes])
+        out = u''.join([to_text(_fail_on_undefined(v)) for v in nodes])
 
     try:
         out = literal_eval(out)

test/integration/targets/jinja2_native_types/inventory.jinja2_native_types

@@ -1 +0,0 @@
-host1

test/integration/targets/jinja2_native_types/nested_undefined.yml

@@ -0,0 +1,24 @@
+- hosts: localhost
+  gather_facts: no
+  tasks:
+    - block:
+      - name: Test nested undefined var fails, single node
+        debug:
+          msg: "{{ [{ 'key': nested_and_undefined }] }}"
+        register: result
+        ignore_errors: yes
+
+      - assert:
+          that:
+            - "\"'nested_and_undefined' is undefined\" in result.msg"
+
+      - name: Test nested undefined var fails, multiple nodes
+        debug:
+          msg: "{{ [{ 'key': nested_and_undefined}] }} second_node"
+        register: result
+        ignore_errors: yes
+
+      - assert:
+          that:
+            - "\"'nested_and_undefined' is undefined\" in result.msg"
+      when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=')

test/integration/targets/jinja2_native_types/runme.sh

@@ -2,6 +2,9 @@
 
 set -eux
 
-ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types runtests.yml -v "$@"
-ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types --vault-password-file test_vault_pass test_vault.yml -v "$@"
-ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types test_hostvars.yml -v "$@"
+export ANSIBLE_JINJA2_NATIVE=1
+ansible-playbook runtests.yml -v "$@"
+ansible-playbook --vault-password-file test_vault_pass test_vault.yml -v "$@"
+ansible-playbook test_hostvars.yml -v "$@"
+ansible-playbook nested_undefined.yml -v "$@"
+unset ANSIBLE_JINJA2_NATIVE

test/integration/targets/jinja2_native_types/test_hostvars.yml

@@ -1,4 +1,4 @@
-- hosts: host1
+- hosts: localhost
   gather_facts: no
   tasks:
     - name: Print vars