diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py index 611df540a5f..6366886cefc 100644 --- a/lib/ansible/parsing/dataloader.py +++ b/lib/ansible/parsing/dataloader.py @@ -7,23 +7,18 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import copy -import json import os import os.path import re import tempfile -from yaml import YAMLError - from ansible.errors import AnsibleFileNotFound, AnsibleParserError -from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR from ansible.module_utils.basic import is_executable from ansible.module_utils.six import binary_type, text_type from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.parsing.quoting import unquote +from ansible.parsing.utils.yaml import from_yaml from ansible.parsing.vault import VaultLib, b_HEADER, is_encrypted, is_encrypted_file, parse_vaulttext_envelope -from ansible.parsing.yaml.loader import AnsibleLoader -from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode from ansible.utils.path import unfrackpath try: @@ -75,45 +70,8 @@ class DataLoader: self._vault.secrets = vault_secrets def load(self, data, file_name='', show_content=True): - ''' - Creates a python datastructure from the given data, which can be either - a JSON or YAML string. - ''' - new_data = None - - # YAML parser will take JSON as it is a subset. - if isinstance(data, AnsibleUnicode): - # The PyYAML's libyaml bindings use PyUnicode_CheckExact so - # they are unable to cope with our subclass. - # Unwrap and re-wrap the unicode so we can keep track of line - # numbers - in_data = text_type(data) - else: - in_data = data - - try: - # we first try to load this data as JSON - new_data = json.loads(data) - except: - # must not be JSON, let the rest try - if isinstance(data, AnsibleUnicode): - # The PyYAML's libyaml bindings use PyUnicode_CheckExact so - # they are unable to cope with our subclass. - # Unwrap and re-wrap the unicode so we can keep track of line - # numbers - in_data = text_type(data) - else: - in_data = data - try: - new_data = self._safe_load(in_data, file_name=file_name) - except YAMLError as yaml_exc: - self._handle_error(yaml_exc, file_name, show_content) - - if isinstance(data, AnsibleUnicode): - new_data = AnsibleUnicode(new_data) - new_data.ansible_pos = data.ansible_pos - - return new_data + '''Backwards compat for now''' + return from_yaml(data, file_name, show_content, self._vault.secrets) def load_from_file(self, file_name, cache=True, unsafe=False): ''' Loads data from a file, which can contain either JSON or YAML. ''' @@ -162,18 +120,6 @@ class DataLoader: path = self.path_dwim(path) return is_executable(path) - def _safe_load(self, stream, file_name=None): - ''' Implements yaml.safe_load(), except using our custom loader class. ''' - - loader = AnsibleLoader(stream, file_name, self._vault.secrets) - try: - return loader.get_single_data() - finally: - try: - loader.dispose() - except AttributeError: - pass # older versions of yaml don't have dispose function, ignore - def _decrypt_if_vault_data(self, b_vault_data, b_file_name=None): '''Decrypt b_vault_data if encrypted and return b_data and the show_content flag''' @@ -217,24 +163,6 @@ class DataLoader: except (IOError, OSError) as e: raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)), orig_exc=e) - def _handle_error(self, yaml_exc, file_name, show_content): - ''' - Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the - file name/position where a YAML exception occurred, and raises an AnsibleParserError - to display the syntax exception information. - ''' - - # if the YAML exception contains a problem mark, use it to construct - # an object the error class can use to display the faulty line - err_obj = None - if hasattr(yaml_exc, 'problem_mark'): - err_obj = AnsibleBaseYAMLObject() - err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) - - err_msg = getattr(yaml_exc, 'problem', '') - - raise AnsibleParserError(YAML_SYNTAX_ERROR % to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc) - def get_basedir(self): ''' returns the current basedir ''' return self._basedir diff --git a/lib/ansible/parsing/utils/yaml.py b/lib/ansible/parsing/utils/yaml.py new file mode 100644 index 00000000000..de36928c020 --- /dev/null +++ b/lib/ansible/parsing/utils/yaml.py @@ -0,0 +1,91 @@ +# (c) 2012-2014, Michael DeHaan +# Copyright: (c) 2017, Ansible Project +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +from yaml import YAMLError + +from ansible.errors import AnsibleParserError +from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR +from ansible.module_utils.six import text_type +from ansible.module_utils._text import to_native +from ansible.parsing.yaml.loader import AnsibleLoader +from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode + + +__all__ = ('from_yaml',) + + +def _handle_error(yaml_exc, file_name, show_content): + ''' + Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the + file name/position where a YAML exception occurred, and raises an AnsibleParserError + to display the syntax exception information. + ''' + + # if the YAML exception contains a problem mark, use it to construct + # an object the error class can use to display the faulty line + err_obj = None + if hasattr(yaml_exc, 'problem_mark'): + err_obj = AnsibleBaseYAMLObject() + err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1) + + err_msg = getattr(yaml_exc, 'problem', '') + + raise AnsibleParserError(YAML_SYNTAX_ERROR % to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc) + + +def _safe_load(stream, file_name=None, vault_secrets=None): + ''' Implements yaml.safe_load(), except using our custom loader class. ''' + + loader = AnsibleLoader(stream, file_name, vault_secrets) + try: + return loader.get_single_data() + finally: + try: + loader.dispose() + except AttributeError: + pass # older versions of yaml don't have dispose function, ignore + + +def from_yaml(data, file_name='', show_content=True): + ''' + Creates a python datastructure from the given data, which can be either + a JSON or YAML string. + ''' + new_data = None + + if isinstance(data, AnsibleUnicode): + # The PyYAML's libyaml bindings use PyUnicode_CheckExact so + # they are unable to cope with our subclass. + # Unwrap and re-wrap the unicode so we can keep track of line + # numbers + # Note: Cannot use to_text() because AnsibleUnicode is a subclass of the text_type. + # Should not have to worry about tracebacks because python's text constructors (unicode() on + # python2 and str() on python3) can handle a subtype of themselves. + in_data = text_type(data) + else: + in_data = data + + try: + # we first try to load this data as JSON. Fixes issues with extra vars json strings not + # being parsed correctly by the yaml parser + new_data = json.loads(in_data) + except Exception: + # must not be JSON, let the rest try + try: + new_data = _safe_load(in_data, file_name=file_name) + except YAMLError as yaml_exc: + _handle_error(yaml_exc, file_name, show_content) + + if isinstance(data, AnsibleUnicode): + new_data = AnsibleUnicode(new_data) + new_data.ansible_pos = data.ansible_pos + + return new_data diff --git a/test/units/parsing/utils/test_yaml.py b/test/units/parsing/utils/test_yaml.py new file mode 100644 index 00000000000..27b2905ac32 --- /dev/null +++ b/test/units/parsing/utils/test_yaml.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# (c) 2017, Ansible Project +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import pytest + +from ansible.errors import AnsibleParserError +from ansible.parsing.utils.yaml import from_yaml + + +def test_from_yaml_simple(): + assert from_yaml(u'---\n- test: 1\n test2: "2"\n- caf\xe9: "caf\xe9"') == [{u'test': 1, u'test2': u"2"}, {u"caf\xe9": u"caf\xe9"}] + + +def test_bad_yaml(): + with pytest.raises(AnsibleParserError): + from_yaml(u'foo: bar: baz')