From bbdf77a59f50306b34161efc148b572ab257a537 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 1 May 2020 15:03:07 -0400 Subject: [PATCH] preserve json parsing error (#58461) * preserve json parsing error * added test --- lib/ansible/parsing/dataloader.py | 8 ++++---- lib/ansible/parsing/utils/yaml.py | 19 ++++++++++++------- lib/ansible/plugins/inventory/script.py | 2 +- test/integration/targets/dataloader/aliases | 1 + .../attempt_to_load_invalid_json.yml | 4 ++++ test/integration/targets/dataloader/runme.sh | 6 ++++++ .../targets/dataloader/vars/invalid.json | 1 + 7 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 test/integration/targets/dataloader/aliases create mode 100644 test/integration/targets/dataloader/attempt_to_load_invalid_json.yml create mode 100755 test/integration/targets/dataloader/runme.sh create mode 100644 test/integration/targets/dataloader/vars/invalid.json diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py index 41095147346..c09683b31f8 100644 --- a/lib/ansible/parsing/dataloader.py +++ b/lib/ansible/parsing/dataloader.py @@ -75,11 +75,11 @@ class DataLoader: def set_vault_secrets(self, vault_secrets): self._vault.secrets = vault_secrets - def load(self, data, file_name='', show_content=True): + def load(self, data, file_name='', show_content=True, json_only=False): '''Backwards compat for now''' - return from_yaml(data, file_name, show_content, self._vault.secrets) + return from_yaml(data, file_name, show_content, self._vault.secrets, json_only=json_only) - def load_from_file(self, file_name, cache=True, unsafe=False): + def load_from_file(self, file_name, cache=True, unsafe=False, json_only=False): ''' Loads data from a file, which can contain either JSON or YAML. ''' file_name = self.path_dwim(file_name) @@ -94,7 +94,7 @@ class DataLoader: (b_file_data, show_content) = self._get_file_contents(file_name) file_data = to_text(b_file_data, errors='surrogate_or_strict') - parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content) + parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content, json_only=json_only) # cache the file contents for next time self._FILE_CACHE[file_name] = parsed_data diff --git a/lib/ansible/parsing/utils/yaml.py b/lib/ansible/parsing/utils/yaml.py index ac0b9970b6a..8dd0550ef8f 100644 --- a/lib/ansible/parsing/utils/yaml.py +++ b/lib/ansible/parsing/utils/yaml.py @@ -13,7 +13,7 @@ from yaml import YAMLError from ansible.errors import AnsibleParserError from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR -from ansible.module_utils._text import to_native +from ansible.module_utils._text import to_native, to_text from ansible.parsing.yaml.loader import AnsibleLoader from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject from ansible.parsing.ajson import AnsibleJSONDecoder @@ -22,7 +22,7 @@ from ansible.parsing.ajson import AnsibleJSONDecoder __all__ = ('from_yaml',) -def _handle_error(yaml_exc, file_name, show_content): +def _handle_error(json_exc, yaml_exc, file_name, show_content): ''' Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the file name/position where a YAML exception occurred, and raises an AnsibleParserError @@ -36,9 +36,10 @@ def _handle_error(yaml_exc, file_name, show_content): err_obj = AnsibleBaseYAMLObject() err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) - err_msg = getattr(yaml_exc, 'problem', '') + err_msg = 'We were unable to read either as JSON nor YAML, these are the errors we got from each:\n' \ + 'JSON: %s\n\n' % to_text(json_exc) + YAML_SYNTAX_ERROR % getattr(yaml_exc, 'problem', '') - raise AnsibleParserError(YAML_SYNTAX_ERROR % to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc) + raise AnsibleParserError(to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc) def _safe_load(stream, file_name=None, vault_secrets=None): @@ -54,7 +55,7 @@ def _safe_load(stream, file_name=None, vault_secrets=None): pass # older versions of yaml don't have dispose function, ignore -def from_yaml(data, file_name='', show_content=True, vault_secrets=None): +def from_yaml(data, file_name='', show_content=True, vault_secrets=None, json_only=False): ''' Creates a python datastructure from the given data, which can be either a JSON or YAML string. @@ -68,11 +69,15 @@ def from_yaml(data, file_name='', show_content=True, vault_secrets=None) # we first try to load this data as JSON. # Fixes issues with extra vars json strings not being parsed correctly by the yaml parser new_data = json.loads(data, cls=AnsibleJSONDecoder) - except Exception: + except Exception as json_exc: + + if json_only: + raise AnsibleParserError(to_native(json_exc), orig_exc=json_exc) + # must not be JSON, let the rest try try: new_data = _safe_load(data, file_name=file_name, vault_secrets=vault_secrets) except YAMLError as yaml_exc: - _handle_error(yaml_exc, file_name, show_content) + _handle_error(json_exc, yaml_exc, file_name, show_content) return new_data diff --git a/lib/ansible/plugins/inventory/script.py b/lib/ansible/plugins/inventory/script.py index ed8dd5aeea5..d28dc3c4bf3 100644 --- a/lib/ansible/plugins/inventory/script.py +++ b/lib/ansible/plugins/inventory/script.py @@ -128,7 +128,7 @@ class InventoryModule(BaseInventoryPlugin, Cacheable): raise AnsibleError("Inventory {0} contained characters that cannot be interpreted as UTF-8: {1}".format(path, to_native(e))) try: - processed = self.loader.load(data) + processed = self.loader.load(data, json_only=True) except Exception as e: raise AnsibleError("failed to parse executable inventory script results from {0}: {1}\n{2}".format(path, to_native(e), err)) diff --git a/test/integration/targets/dataloader/aliases b/test/integration/targets/dataloader/aliases new file mode 100644 index 00000000000..a6dafcf8cd8 --- /dev/null +++ b/test/integration/targets/dataloader/aliases @@ -0,0 +1 @@ +shippable/posix/group1 diff --git a/test/integration/targets/dataloader/attempt_to_load_invalid_json.yml b/test/integration/targets/dataloader/attempt_to_load_invalid_json.yml new file mode 100644 index 00000000000..536e6daa834 --- /dev/null +++ b/test/integration/targets/dataloader/attempt_to_load_invalid_json.yml @@ -0,0 +1,4 @@ +- hosts: localhost + gather_facts: false + vars_files: + - vars/invalid.json diff --git a/test/integration/targets/dataloader/runme.sh b/test/integration/targets/dataloader/runme.sh new file mode 100755 index 00000000000..6a1bc9a0c2e --- /dev/null +++ b/test/integration/targets/dataloader/runme.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -eux + +# check if we get proper json error +ansible-playbook -i ../../inventory attempt_to_load_invalid_json.yml "$@" 2>&1|grep 'JSON:' diff --git a/test/integration/targets/dataloader/vars/invalid.json b/test/integration/targets/dataloader/vars/invalid.json new file mode 100644 index 00000000000..8d4e4304b8f --- /dev/null +++ b/test/integration/targets/dataloader/vars/invalid.json @@ -0,0 +1 @@ +{ }}