Properly JSON encode AnsibleUnsafe, using a pre-processor (#60602)

* Properly JSON encode AnsibleUnsafe, using a pre-processor. Fixes #47295

* Add AnsibleUnsafe json tests

* Require preprocess_unsafe to be enabled for that functionality

* Support older json

* sort keys in tests

* Decouple AnsibleJSONEncoder from isinstance checks in preparation to move to module_utils

* Move AnsibleJSONEncoder to module_utils, consolidate instances

* add missing boilerplate

* remove removed.py from ignore

lib/ansible/cli/inventory.py

@@ -180,7 +180,7 @@ class InventoryCLI(CLI):
         else:
             import json
             from ansible.parsing.ajson import AnsibleJSONEncoder
-            results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)
+            results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4, preprocess_unsafe=True)
 
         return results
 

lib/ansible/module_utils/common/json.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+import datetime
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.common._collections_compat import Mapping
+from ansible.module_utils.common.collections import is_sequence
+
+
+def _preprocess_unsafe_encode(value):
+    """Recursively preprocess a data structure converting instances of ``AnsibleUnsafe``
+    into their JSON dict representations
+
+    Used in ``AnsibleJSONEncoder.iterencode``
+    """
+    if getattr(value, '__UNSAFE__', False) and not getattr(value, '__ENCRYPTED__', False):
+        value = {'__ansible_unsafe': to_text(value, errors='surrogate_or_strict', nonstring='strict')}
+    elif is_sequence(value):
+        value = [_preprocess_unsafe_encode(v) for v in value]
+    elif isinstance(value, Mapping):
+        value = dict((k, _preprocess_unsafe_encode(v)) for k, v in value.items())
+
+    return value
+
+
+class AnsibleJSONEncoder(json.JSONEncoder):
+    '''
+    Simple encoder class to deal with JSON encoding of Ansible internal types
+    '''
+
+    def __init__(self, preprocess_unsafe=False, **kwargs):
+        self._preprocess_unsafe = preprocess_unsafe
+        super(AnsibleJSONEncoder, self).__init__(**kwargs)
+
+    # NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback
+    def default(self, o):
+        if getattr(o, '__ENCRYPTED__', False):
+            # vault object
+            value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
+        elif getattr(o, '__UNSAFE__', False):
+            # unsafe object, this will never be triggered, see ``AnsibleJSONEncoder.iterencode``
+            value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
+        elif isinstance(o, Mapping):
+            # hostvars and other objects
+            value = dict(o)
+        elif isinstance(o, (datetime.date, datetime.datetime)):
+            # date object
+            value = o.isoformat()
+        else:
+            # use default encoder
+            value = super(AnsibleJSONEncoder, self).default(o)
+        return value
+
+    def iterencode(self, o, **kwargs):
+        """Custom iterencode, primarily design to handle encoding ``AnsibleUnsafe``
+        as the ``AnsibleUnsafe`` subclasses inherit from string types and
+        ``json.JSONEncoder`` does not support custom encoders for string types
+        """
+        if self._preprocess_unsafe:
+            o = _preprocess_unsafe_encode(o)
+
+        return super(AnsibleJSONEncoder, self).iterencode(o, **kwargs)

lib/ansible/module_utils/common/removed.py

@@ -1,6 +1,10 @@
 # Copyright (c) 2018, Ansible Project
 # Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
 
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
 import json
 import sys
 

lib/ansible/module_utils/connection.py

@@ -33,11 +33,10 @@ import socket
 import struct
 import traceback
 import uuid
-from datetime import date, datetime
 
 from functools import partial
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.module_utils.common._collections_compat import Mapping
+from ansible.module_utils.common.json import AnsibleJSONEncoder
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.six.moves import cPickle
 
@@ -202,29 +201,3 @@ class Connection(object):
         sf.close()
 
         return to_text(response, errors='surrogate_or_strict')
-
-
-# NOTE: This is a modified copy of the class in parsing.ajson to get around not
-#       being able to import that directly, nor some of the type classes
-class AnsibleJSONEncoder(json.JSONEncoder):
-    '''
-    Simple encoder class to deal with JSON encoding of Ansible internal types
-    '''
-
-    def default(self, o):
-        if type(o).__name__ == 'AnsibleVaultEncryptedUnicode':
-            # vault object
-            value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
-        elif type(o).__name__ == 'AnsibleUnsafe':
-            # unsafe object
-            value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
-        elif isinstance(o, Mapping):
-            # hostvars and other objects
-            value = dict(o)
-        elif isinstance(o, (date, datetime)):
-            # date object
-            value = o.isoformat()
-        else:
-            # use default encoder
-            value = super(AnsibleJSONEncoder, self).default(o)
-        return value

lib/ansible/parsing/ajson.py

@@ -7,13 +7,12 @@ __metaclass__ = type
 
 import json
 
-from datetime import date, datetime
+# Imported for backwards compat
+from ansible.module_utils.common.json import AnsibleJSONEncoder
 
-from ansible.module_utils._text import to_text
-from ansible.module_utils.common._collections_compat import Mapping
-from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
-from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var
 from ansible.parsing.vault import VaultLib
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.unsafe_proxy import wrap_var
 
 
 class AnsibleJSONDecoder(json.JSONDecoder):
@@ -38,32 +37,6 @@ class AnsibleJSONDecoder(json.JSONDecoder):
                     value.vault = self._vaults['default']
                 return value
             elif key == '__ansible_unsafe':
-                return wrap_var(value.get('__ansible_unsafe'))
+                return wrap_var(value)
 
         return pairs
-
-
-# TODO: find way to integrate with the encoding modules do in module_utils
-class AnsibleJSONEncoder(json.JSONEncoder):
-    '''
-    Simple encoder class to deal with JSON encoding of Ansible internal types
-    '''
-
-    # NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback
-    def default(self, o):
-        if isinstance(o, AnsibleVaultEncryptedUnicode):
-            # vault object
-            value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
-        elif isinstance(o, AnsibleUnsafe):
-            # unsafe object
-            value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
-        elif isinstance(o, Mapping):
-            # hostvars and other objects
-            value = dict(o)
-        elif isinstance(o, (date, datetime)):
-            # date object
-            value = o.isoformat()
-        else:
-            # use default encoder
-            value = super(AnsibleJSONEncoder, self).default(o)
-        return value

test/sanity/ignore.txt

@@ -203,8 +203,6 @@ lib/ansible/module_utils/cloud.py future-import-boilerplate
 lib/ansible/module_utils/cloud.py metaclass-boilerplate
 lib/ansible/module_utils/common/network.py future-import-boilerplate
 lib/ansible/module_utils/common/network.py metaclass-boilerplate
-lib/ansible/module_utils/common/removed.py future-import-boilerplate
-lib/ansible/module_utils/common/removed.py metaclass-boilerplate
 lib/ansible/module_utils/compat/ipaddress.py future-import-boilerplate
 lib/ansible/module_utils/compat/ipaddress.py metaclass-boilerplate
 lib/ansible/module_utils/compat/ipaddress.py no-assert

test/units/parsing/test_ajson.py

@@ -16,6 +16,7 @@ from pytz import timezone as tz
 from ansible.module_utils.common._collections_compat import Mapping
 from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.unsafe_proxy import AnsibleUnsafeText
 
 
 def test_AnsibleJSONDecoder_vault():
@@ -27,6 +28,20 @@ def test_AnsibleJSONDecoder_vault():
     assert isinstance(data['foo']['password'], AnsibleVaultEncryptedUnicode)
 
 
+def test_encode_decode_unsafe():
+    data = {
+        'key_value': AnsibleUnsafeText(u'{#NOTACOMMENT#}'),
+        'list': [AnsibleUnsafeText(u'{#NOTACOMMENT#}')],
+        'list_dict': [{'key_value': AnsibleUnsafeText(u'{#NOTACOMMENT#}')}]}
+    json_expected = (
+        '{"key_value": {"__ansible_unsafe": "{#NOTACOMMENT#}"}, '
+        '"list": [{"__ansible_unsafe": "{#NOTACOMMENT#}"}], '
+        '"list_dict": [{"key_value": {"__ansible_unsafe": "{#NOTACOMMENT#}"}}]}'
+    )
+    assert json.dumps(data, cls=AnsibleJSONEncoder, preprocess_unsafe=True, sort_keys=True) == json_expected
+    assert json.loads(json_expected, cls=AnsibleJSONDecoder) == data
+
+
 def vault_data():
     """
     Prepare AnsibleVaultEncryptedUnicode test data for AnsibleJSONEncoder.default().