Move yaml loading functions from DataLoader into a utility module.
This commit is contained in:
Toshio Kuratomi
parent
2a4b3ef0c1
commit
14c3b4d8e5
|
|
@ -7,23 +7,18 @@ from __future__ import (absolute_import, division, print_function)
|
|||
__metaclass__ = type
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
from yaml import YAMLError
|
||||
|
||||
from ansible.errors import AnsibleFileNotFound, AnsibleParserError
|
||||
from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
|
||||
from ansible.module_utils.basic import is_executable
|
||||
from ansible.module_utils.six import binary_type, text_type
|
||||
from ansible.module_utils._text import to_bytes, to_native, to_text
|
||||
from ansible.parsing.quoting import unquote
|
||||
from ansible.parsing.utils.yaml import from_yaml
|
||||
from ansible.parsing.vault import VaultLib, b_HEADER, is_encrypted, is_encrypted_file, parse_vaulttext_envelope
|
||||
from ansible.parsing.yaml.loader import AnsibleLoader
|
||||
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
|
||||
from ansible.utils.path import unfrackpath
|
||||
|
||||
try:
|
||||
|
|
@ -75,45 +70,8 @@ class DataLoader:
|
|||
self._vault.secrets = vault_secrets
|
||||
|
||||
def load(self, data, file_name='<string>', show_content=True):
|
||||
'''
|
||||
Creates a python datastructure from the given data, which can be either
|
||||
a JSON or YAML string.
|
||||
'''
|
||||
new_data = None
|
||||
|
||||
# YAML parser will take JSON as it is a subset.
|
||||
if isinstance(data, AnsibleUnicode):
|
||||
# The PyYAML's libyaml bindings use PyUnicode_CheckExact so
|
||||
# they are unable to cope with our subclass.
|
||||
# Unwrap and re-wrap the unicode so we can keep track of line
|
||||
# numbers
|
||||
in_data = text_type(data)
|
||||
else:
|
||||
in_data = data
|
||||
|
||||
try:
|
||||
# we first try to load this data as JSON
|
||||
new_data = json.loads(data)
|
||||
except:
|
||||
# must not be JSON, let the rest try
|
||||
if isinstance(data, AnsibleUnicode):
|
||||
# The PyYAML's libyaml bindings use PyUnicode_CheckExact so
|
||||
# they are unable to cope with our subclass.
|
||||
# Unwrap and re-wrap the unicode so we can keep track of line
|
||||
# numbers
|
||||
in_data = text_type(data)
|
||||
else:
|
||||
in_data = data
|
||||
try:
|
||||
new_data = self._safe_load(in_data, file_name=file_name)
|
||||
except YAMLError as yaml_exc:
|
||||
self._handle_error(yaml_exc, file_name, show_content)
|
||||
|
||||
if isinstance(data, AnsibleUnicode):
|
||||
new_data = AnsibleUnicode(new_data)
|
||||
new_data.ansible_pos = data.ansible_pos
|
||||
|
||||
return new_data
|
||||
'''Backwards compat for now'''
|
||||
return from_yaml(data, file_name, show_content, self._vault.secrets)
|
||||
|
||||
def load_from_file(self, file_name, cache=True, unsafe=False):
|
||||
''' Loads data from a file, which can contain either JSON or YAML. '''
|
||||
|
|
@ -162,18 +120,6 @@ class DataLoader:
|
|||
path = self.path_dwim(path)
|
||||
return is_executable(path)
|
||||
|
||||
def _safe_load(self, stream, file_name=None):
|
||||
''' Implements yaml.safe_load(), except using our custom loader class. '''
|
||||
|
||||
loader = AnsibleLoader(stream, file_name, self._vault.secrets)
|
||||
try:
|
||||
return loader.get_single_data()
|
||||
finally:
|
||||
try:
|
||||
loader.dispose()
|
||||
except AttributeError:
|
||||
pass # older versions of yaml don't have dispose function, ignore
|
||||
|
||||
def _decrypt_if_vault_data(self, b_vault_data, b_file_name=None):
|
||||
'''Decrypt b_vault_data if encrypted and return b_data and the show_content flag'''
|
||||
|
||||
|
|
@ -217,24 +163,6 @@ class DataLoader:
|
|||
except (IOError, OSError) as e:
|
||||
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)), orig_exc=e)
|
||||
|
||||
def _handle_error(self, yaml_exc, file_name, show_content):
|
||||
'''
|
||||
Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
|
||||
file name/position where a YAML exception occurred, and raises an AnsibleParserError
|
||||
to display the syntax exception information.
|
||||
'''
|
||||
|
||||
# if the YAML exception contains a problem mark, use it to construct
|
||||
# an object the error class can use to display the faulty line
|
||||
err_obj = None
|
||||
if hasattr(yaml_exc, 'problem_mark'):
|
||||
err_obj = AnsibleBaseYAMLObject()
|
||||
err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
|
||||
|
||||
err_msg = getattr(yaml_exc, 'problem', '')
|
||||
|
||||
raise AnsibleParserError(YAML_SYNTAX_ERROR % to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc)
|
||||
|
||||
def get_basedir(self):
|
||||
''' returns the current basedir '''
|
||||
return self._basedir
|
||||
|
|
|
|||
91
lib/ansible/parsing/utils/yaml.py
Normal file
91
lib/ansible/parsing/utils/yaml.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
|
||||
# Copyright: (c) 2017, Ansible Project
|
||||
# Copyright: (c) 2018, Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import json
|
||||
|
||||
from yaml import YAMLError
|
||||
|
||||
from ansible.errors import AnsibleParserError
|
||||
from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
|
||||
from ansible.module_utils.six import text_type
|
||||
from ansible.module_utils._text import to_native
|
||||
from ansible.parsing.yaml.loader import AnsibleLoader
|
||||
from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
|
||||
|
||||
|
||||
__all__ = ('from_yaml',)
|
||||
|
||||
|
||||
def _handle_error(yaml_exc, file_name, show_content):
|
||||
'''
|
||||
Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
|
||||
file name/position where a YAML exception occurred, and raises an AnsibleParserError
|
||||
to display the syntax exception information.
|
||||
'''
|
||||
|
||||
# if the YAML exception contains a problem mark, use it to construct
|
||||
# an object the error class can use to display the faulty line
|
||||
err_obj = None
|
||||
if hasattr(yaml_exc, 'problem_mark'):
|
||||
err_obj = AnsibleBaseYAMLObject()
|
||||
err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
|
||||
|
||||
err_msg = getattr(yaml_exc, 'problem', '')
|
||||
|
||||
raise AnsibleParserError(YAML_SYNTAX_ERROR % to_native(err_msg), obj=err_obj, show_content=show_content, orig_exc=yaml_exc)
|
||||
|
||||
|
||||
def _safe_load(stream, file_name=None, vault_secrets=None):
|
||||
''' Implements yaml.safe_load(), except using our custom loader class. '''
|
||||
|
||||
loader = AnsibleLoader(stream, file_name, vault_secrets)
|
||||
try:
|
||||
return loader.get_single_data()
|
||||
finally:
|
||||
try:
|
||||
loader.dispose()
|
||||
except AttributeError:
|
||||
pass # older versions of yaml don't have dispose function, ignore
|
||||
|
||||
|
||||
def from_yaml(data, file_name='<string>', show_content=True):
|
||||
'''
|
||||
Creates a python datastructure from the given data, which can be either
|
||||
a JSON or YAML string.
|
||||
'''
|
||||
new_data = None
|
||||
|
||||
if isinstance(data, AnsibleUnicode):
|
||||
# The PyYAML's libyaml bindings use PyUnicode_CheckExact so
|
||||
# they are unable to cope with our subclass.
|
||||
# Unwrap and re-wrap the unicode so we can keep track of line
|
||||
# numbers
|
||||
# Note: Cannot use to_text() because AnsibleUnicode is a subclass of the text_type.
|
||||
# Should not have to worry about tracebacks because python's text constructors (unicode() on
|
||||
# python2 and str() on python3) can handle a subtype of themselves.
|
||||
in_data = text_type(data)
|
||||
else:
|
||||
in_data = data
|
||||
|
||||
try:
|
||||
# we first try to load this data as JSON. Fixes issues with extra vars json strings not
|
||||
# being parsed correctly by the yaml parser
|
||||
new_data = json.loads(in_data)
|
||||
except Exception:
|
||||
# must not be JSON, let the rest try
|
||||
try:
|
||||
new_data = _safe_load(in_data, file_name=file_name)
|
||||
except YAMLError as yaml_exc:
|
||||
_handle_error(yaml_exc, file_name, show_content)
|
||||
|
||||
if isinstance(data, AnsibleUnicode):
|
||||
new_data = AnsibleUnicode(new_data)
|
||||
new_data.ansible_pos = data.ansible_pos
|
||||
|
||||
return new_data
|
||||
34
test/units/parsing/utils/test_yaml.py
Normal file
34
test/units/parsing/utils/test_yaml.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# (c) 2017, Ansible Project
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import pytest
|
||||
|
||||
from ansible.errors import AnsibleParserError
|
||||
from ansible.parsing.utils.yaml import from_yaml
|
||||
|
||||
|
||||
def test_from_yaml_simple():
|
||||
assert from_yaml(u'---\n- test: 1\n test2: "2"\n- caf\xe9: "caf\xe9"') == [{u'test': 1, u'test2': u"2"}, {u"caf\xe9": u"caf\xe9"}]
|
||||
|
||||
|
||||
def test_bad_yaml():
|
||||
with pytest.raises(AnsibleParserError):
|
||||
from_yaml(u'foo: bar: baz')
|
||||
Loading…
Reference in a new issue