Properly JSON encode AnsibleUnsafe, using a pre-processor (#60602)
* Properly JSON encode AnsibleUnsafe, using a pre-processor. Fixes #47295 * Add AnsibleUnsafe json tests * Require preprocess_unsafe to be enabled for that functionality * Support older json * sort keys in tests * Decouple AnsibleJSONEncoder from isinstance checks in preparation to move to module_utils * Move AnsibleJSONEncoder to module_utils, consolidate instances * add missing boilerplate * remove removed.py from ignore
This commit is contained in:
parent
1d405fdd60
commit
5941e4c843
7 changed files with 96 additions and 63 deletions
|
@ -180,7 +180,7 @@ class InventoryCLI(CLI):
|
||||||
else:
|
else:
|
||||||
import json
|
import json
|
||||||
from ansible.parsing.ajson import AnsibleJSONEncoder
|
from ansible.parsing.ajson import AnsibleJSONEncoder
|
||||||
results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)
|
results = json.dumps(stuff, cls=AnsibleJSONEncoder, sort_keys=True, indent=4, preprocess_unsafe=True)
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
70
lib/ansible/module_utils/common/json.py
Normal file
70
lib/ansible/module_utils/common/json.py
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) 2019 Ansible Project
|
||||||
|
# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
|
||||||
|
|
||||||
|
# Make coding more python3-ish
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
from ansible.module_utils._text import to_text
|
||||||
|
from ansible.module_utils.common._collections_compat import Mapping
|
||||||
|
from ansible.module_utils.common.collections import is_sequence
|
||||||
|
|
||||||
|
|
||||||
|
def _preprocess_unsafe_encode(value):
|
||||||
|
"""Recursively preprocess a data structure converting instances of ``AnsibleUnsafe``
|
||||||
|
into their JSON dict representations
|
||||||
|
|
||||||
|
Used in ``AnsibleJSONEncoder.iterencode``
|
||||||
|
"""
|
||||||
|
if getattr(value, '__UNSAFE__', False) and not getattr(value, '__ENCRYPTED__', False):
|
||||||
|
value = {'__ansible_unsafe': to_text(value, errors='surrogate_or_strict', nonstring='strict')}
|
||||||
|
elif is_sequence(value):
|
||||||
|
value = [_preprocess_unsafe_encode(v) for v in value]
|
||||||
|
elif isinstance(value, Mapping):
|
||||||
|
value = dict((k, _preprocess_unsafe_encode(v)) for k, v in value.items())
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class AnsibleJSONEncoder(json.JSONEncoder):
|
||||||
|
'''
|
||||||
|
Simple encoder class to deal with JSON encoding of Ansible internal types
|
||||||
|
'''
|
||||||
|
|
||||||
|
def __init__(self, preprocess_unsafe=False, **kwargs):
|
||||||
|
self._preprocess_unsafe = preprocess_unsafe
|
||||||
|
super(AnsibleJSONEncoder, self).__init__(**kwargs)
|
||||||
|
|
||||||
|
# NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback
|
||||||
|
def default(self, o):
|
||||||
|
if getattr(o, '__ENCRYPTED__', False):
|
||||||
|
# vault object
|
||||||
|
value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
|
||||||
|
elif getattr(o, '__UNSAFE__', False):
|
||||||
|
# unsafe object, this will never be triggered, see ``AnsibleJSONEncoder.iterencode``
|
||||||
|
value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
|
||||||
|
elif isinstance(o, Mapping):
|
||||||
|
# hostvars and other objects
|
||||||
|
value = dict(o)
|
||||||
|
elif isinstance(o, (datetime.date, datetime.datetime)):
|
||||||
|
# date object
|
||||||
|
value = o.isoformat()
|
||||||
|
else:
|
||||||
|
# use default encoder
|
||||||
|
value = super(AnsibleJSONEncoder, self).default(o)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def iterencode(self, o, **kwargs):
|
||||||
|
"""Custom iterencode, primarily design to handle encoding ``AnsibleUnsafe``
|
||||||
|
as the ``AnsibleUnsafe`` subclasses inherit from string types and
|
||||||
|
``json.JSONEncoder`` does not support custom encoders for string types
|
||||||
|
"""
|
||||||
|
if self._preprocess_unsafe:
|
||||||
|
o = _preprocess_unsafe_encode(o)
|
||||||
|
|
||||||
|
return super(AnsibleJSONEncoder, self).iterencode(o, **kwargs)
|
|
@ -1,6 +1,10 @@
|
||||||
# Copyright (c) 2018, Ansible Project
|
# Copyright (c) 2018, Ansible Project
|
||||||
# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
|
# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
|
||||||
|
|
||||||
|
# Make coding more python3-ish
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
|
@ -33,11 +33,10 @@ import socket
|
||||||
import struct
|
import struct
|
||||||
import traceback
|
import traceback
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import date, datetime
|
|
||||||
|
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from ansible.module_utils._text import to_bytes, to_text
|
from ansible.module_utils._text import to_bytes, to_text
|
||||||
from ansible.module_utils.common._collections_compat import Mapping
|
from ansible.module_utils.common.json import AnsibleJSONEncoder
|
||||||
from ansible.module_utils.six import iteritems
|
from ansible.module_utils.six import iteritems
|
||||||
from ansible.module_utils.six.moves import cPickle
|
from ansible.module_utils.six.moves import cPickle
|
||||||
|
|
||||||
|
@ -202,29 +201,3 @@ class Connection(object):
|
||||||
sf.close()
|
sf.close()
|
||||||
|
|
||||||
return to_text(response, errors='surrogate_or_strict')
|
return to_text(response, errors='surrogate_or_strict')
|
||||||
|
|
||||||
|
|
||||||
# NOTE: This is a modified copy of the class in parsing.ajson to get around not
|
|
||||||
# being able to import that directly, nor some of the type classes
|
|
||||||
class AnsibleJSONEncoder(json.JSONEncoder):
|
|
||||||
'''
|
|
||||||
Simple encoder class to deal with JSON encoding of Ansible internal types
|
|
||||||
'''
|
|
||||||
|
|
||||||
def default(self, o):
|
|
||||||
if type(o).__name__ == 'AnsibleVaultEncryptedUnicode':
|
|
||||||
# vault object
|
|
||||||
value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
|
|
||||||
elif type(o).__name__ == 'AnsibleUnsafe':
|
|
||||||
# unsafe object
|
|
||||||
value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
|
|
||||||
elif isinstance(o, Mapping):
|
|
||||||
# hostvars and other objects
|
|
||||||
value = dict(o)
|
|
||||||
elif isinstance(o, (date, datetime)):
|
|
||||||
# date object
|
|
||||||
value = o.isoformat()
|
|
||||||
else:
|
|
||||||
# use default encoder
|
|
||||||
value = super(AnsibleJSONEncoder, self).default(o)
|
|
||||||
return value
|
|
||||||
|
|
|
@ -7,13 +7,12 @@ __metaclass__ = type
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from datetime import date, datetime
|
# Imported for backwards compat
|
||||||
|
from ansible.module_utils.common.json import AnsibleJSONEncoder
|
||||||
|
|
||||||
from ansible.module_utils._text import to_text
|
|
||||||
from ansible.module_utils.common._collections_compat import Mapping
|
|
||||||
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
|
|
||||||
from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var
|
|
||||||
from ansible.parsing.vault import VaultLib
|
from ansible.parsing.vault import VaultLib
|
||||||
|
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
|
||||||
|
from ansible.utils.unsafe_proxy import wrap_var
|
||||||
|
|
||||||
|
|
||||||
class AnsibleJSONDecoder(json.JSONDecoder):
|
class AnsibleJSONDecoder(json.JSONDecoder):
|
||||||
|
@ -38,32 +37,6 @@ class AnsibleJSONDecoder(json.JSONDecoder):
|
||||||
value.vault = self._vaults['default']
|
value.vault = self._vaults['default']
|
||||||
return value
|
return value
|
||||||
elif key == '__ansible_unsafe':
|
elif key == '__ansible_unsafe':
|
||||||
return wrap_var(value.get('__ansible_unsafe'))
|
return wrap_var(value)
|
||||||
|
|
||||||
return pairs
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
# TODO: find way to integrate with the encoding modules do in module_utils
|
|
||||||
class AnsibleJSONEncoder(json.JSONEncoder):
|
|
||||||
'''
|
|
||||||
Simple encoder class to deal with JSON encoding of Ansible internal types
|
|
||||||
'''
|
|
||||||
|
|
||||||
# NOTE: ALWAYS inform AWS/Tower when new items get added as they consume them downstream via a callback
|
|
||||||
def default(self, o):
|
|
||||||
if isinstance(o, AnsibleVaultEncryptedUnicode):
|
|
||||||
# vault object
|
|
||||||
value = {'__ansible_vault': to_text(o._ciphertext, errors='surrogate_or_strict', nonstring='strict')}
|
|
||||||
elif isinstance(o, AnsibleUnsafe):
|
|
||||||
# unsafe object
|
|
||||||
value = {'__ansible_unsafe': to_text(o, errors='surrogate_or_strict', nonstring='strict')}
|
|
||||||
elif isinstance(o, Mapping):
|
|
||||||
# hostvars and other objects
|
|
||||||
value = dict(o)
|
|
||||||
elif isinstance(o, (date, datetime)):
|
|
||||||
# date object
|
|
||||||
value = o.isoformat()
|
|
||||||
else:
|
|
||||||
# use default encoder
|
|
||||||
value = super(AnsibleJSONEncoder, self).default(o)
|
|
||||||
return value
|
|
||||||
|
|
|
@ -203,8 +203,6 @@ lib/ansible/module_utils/cloud.py future-import-boilerplate
|
||||||
lib/ansible/module_utils/cloud.py metaclass-boilerplate
|
lib/ansible/module_utils/cloud.py metaclass-boilerplate
|
||||||
lib/ansible/module_utils/common/network.py future-import-boilerplate
|
lib/ansible/module_utils/common/network.py future-import-boilerplate
|
||||||
lib/ansible/module_utils/common/network.py metaclass-boilerplate
|
lib/ansible/module_utils/common/network.py metaclass-boilerplate
|
||||||
lib/ansible/module_utils/common/removed.py future-import-boilerplate
|
|
||||||
lib/ansible/module_utils/common/removed.py metaclass-boilerplate
|
|
||||||
lib/ansible/module_utils/compat/ipaddress.py future-import-boilerplate
|
lib/ansible/module_utils/compat/ipaddress.py future-import-boilerplate
|
||||||
lib/ansible/module_utils/compat/ipaddress.py metaclass-boilerplate
|
lib/ansible/module_utils/compat/ipaddress.py metaclass-boilerplate
|
||||||
lib/ansible/module_utils/compat/ipaddress.py no-assert
|
lib/ansible/module_utils/compat/ipaddress.py no-assert
|
||||||
|
|
|
@ -16,6 +16,7 @@ from pytz import timezone as tz
|
||||||
from ansible.module_utils.common._collections_compat import Mapping
|
from ansible.module_utils.common._collections_compat import Mapping
|
||||||
from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder
|
from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder
|
||||||
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
|
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
|
||||||
|
from ansible.utils.unsafe_proxy import AnsibleUnsafeText
|
||||||
|
|
||||||
|
|
||||||
def test_AnsibleJSONDecoder_vault():
|
def test_AnsibleJSONDecoder_vault():
|
||||||
|
@ -27,6 +28,20 @@ def test_AnsibleJSONDecoder_vault():
|
||||||
assert isinstance(data['foo']['password'], AnsibleVaultEncryptedUnicode)
|
assert isinstance(data['foo']['password'], AnsibleVaultEncryptedUnicode)
|
||||||
|
|
||||||
|
|
||||||
|
def test_encode_decode_unsafe():
|
||||||
|
data = {
|
||||||
|
'key_value': AnsibleUnsafeText(u'{#NOTACOMMENT#}'),
|
||||||
|
'list': [AnsibleUnsafeText(u'{#NOTACOMMENT#}')],
|
||||||
|
'list_dict': [{'key_value': AnsibleUnsafeText(u'{#NOTACOMMENT#}')}]}
|
||||||
|
json_expected = (
|
||||||
|
'{"key_value": {"__ansible_unsafe": "{#NOTACOMMENT#}"}, '
|
||||||
|
'"list": [{"__ansible_unsafe": "{#NOTACOMMENT#}"}], '
|
||||||
|
'"list_dict": [{"key_value": {"__ansible_unsafe": "{#NOTACOMMENT#}"}}]}'
|
||||||
|
)
|
||||||
|
assert json.dumps(data, cls=AnsibleJSONEncoder, preprocess_unsafe=True, sort_keys=True) == json_expected
|
||||||
|
assert json.loads(json_expected, cls=AnsibleJSONDecoder) == data
|
||||||
|
|
||||||
|
|
||||||
def vault_data():
|
def vault_data():
|
||||||
"""
|
"""
|
||||||
Prepare AnsibleVaultEncryptedUnicode test data for AnsibleJSONEncoder.default().
|
Prepare AnsibleVaultEncryptedUnicode test data for AnsibleJSONEncoder.default().
|
||||||
|
|
Loading…
Reference in a new issue