From 4203850d1afe7f4bcfd32963900485f99198da6a Mon Sep 17 00:00:00 2001
From: Toshio Kuratomi <toshio@fedoraproject.org>
Date: Mon, 26 Oct 2015 14:23:09 -0700
Subject: [PATCH] Break apart a looped dependency to show a warning when
parsing playbooks
Display a warning when a dict key is overwritten by pyyaml
Fixes #12888
---
hacking/test-module | 2 +-
lib/ansible/cli/adhoc.py | 2 +-
lib/ansible/cli/playbook.py | 2 +-
lib/ansible/cli/vault.py | 5 +-
lib/ansible/constants.py | 2 +-
lib/ansible/executor/task_result.py | 2 +-
lib/ansible/parsing/__init__.py | 277 +---------------------
lib/ansible/parsing/dataloader.py | 294 ++++++++++++++++++++++++
lib/ansible/parsing/quoting.py | 30 +++
lib/ansible/parsing/splitter.py | 10 +-
lib/ansible/parsing/yaml/constructor.py | 39 +++-
lib/ansible/playbook/__init__.py | 4 +-
lib/ansible/playbook/base.py | 6 +-
lib/ansible/vars/__init__.py | 1 -
samples/multi_queues.py | 2 +-
test/units/mock/loader.py | 2 +-
test/units/parsing/test_data_loader.py | 2 +-
test/units/parsing/test_unquote.py | 2 +-
18 files changed, 373 insertions(+), 311 deletions(-)
create mode 100644 lib/ansible/parsing/dataloader.py
create mode 100644 lib/ansible/parsing/quoting.py
diff --git a/hacking/test-module b/hacking/test-module
index 543e803b976..412b39452c3 100755
--- a/hacking/test-module
+++ b/hacking/test-module
@@ -36,7 +36,7 @@ import subprocess
import traceback
import optparse
import ansible.utils.vars as utils_vars
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.parsing.utils.jsonify import jsonify
from ansible.parsing.splitter import parse_kv
import ansible.executor.module_common as module_common
diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py
index f2d6780c931..7d611bf1f0b 100644
--- a/lib/ansible/cli/adhoc.py
+++ b/lib/ansible/cli/adhoc.py
@@ -27,7 +27,7 @@ from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.inventory import Inventory
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.parsing.splitter import parse_kv
from ansible.playbook.play import Play
from ansible.plugins import get_all_plugin_loaders
diff --git a/lib/ansible/cli/playbook.py b/lib/ansible/cli/playbook.py
index 33414601ed9..11c4f796b9c 100644
--- a/lib/ansible/cli/playbook.py
+++ b/lib/ansible/cli/playbook.py
@@ -29,7 +29,7 @@ from ansible.cli import CLI
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.inventory import Inventory
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.utils.vars import load_extra_vars
from ansible.vars import VariableManager
diff --git a/lib/ansible/cli/vault.py b/lib/ansible/cli/vault.py
index e4909cc255e..a4329056f99 100644
--- a/lib/ansible/cli/vault.py
+++ b/lib/ansible/cli/vault.py
@@ -21,14 +21,11 @@ __metaclass__ = type
import os
import sys
-import traceback
-from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.parsing.vault import VaultEditor
from ansible.cli import CLI
-from ansible.utils.display import Display
class VaultCLI(CLI):
""" Vault command line class """
diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py
index 1a919eed598..8270ef22c03 100644
--- a/lib/ansible/constants.py
+++ b/lib/ansible/constants.py
@@ -25,7 +25,7 @@ from string import ascii_letters, digits
from ansible.compat.six import string_types
from ansible.compat.six.moves import configparser
-from ansible.parsing.splitter import unquote
+from ansible.parsing.quoting import unquote
from ansible.errors import AnsibleOptionsError
# copied from utils, avoid circular reference fun :)
diff --git a/lib/ansible/executor/task_result.py b/lib/ansible/executor/task_result.py
index d87f9413efb..da9ab2a11ab 100644
--- a/lib/ansible/executor/task_result.py
+++ b/lib/ansible/executor/task_result.py
@@ -19,7 +19,7 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
class TaskResult:
'''
diff --git a/lib/ansible/parsing/__init__.py b/lib/ansible/parsing/__init__.py
index 8c1833f037b..c5cbe7cf9d2 100644
--- a/lib/ansible/parsing/__init__.py
+++ b/lib/ansible/parsing/__init__.py
@@ -1,4 +1,4 @@
-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2015, Toshio Kuratomi <tkuratomi@ansbile.com>
#
# This file is part of Ansible
#
@@ -18,278 +18,3 @@
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
-
-import copy
-import json
-import os
-import stat
-import subprocess
-
-from yaml import load, YAMLError
-from ansible.compat.six import text_type, string_types
-
-from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
-from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
-from ansible.parsing.vault import VaultLib
-from ansible.parsing.splitter import unquote
-from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
-from ansible.module_utils.basic import is_executable
-from ansible.utils.path import unfrackpath
-from ansible.utils.unicode import to_unicode
-
-class DataLoader():
-
- '''
- The DataLoader class is used to load and parse YAML or JSON content,
- either from a given file name or from a string that was previously
- read in through other means. A Vault password can be specified, and
- any vault-encrypted files will be decrypted.
-
- Data read from files will also be cached, so the file will never be
- read from disk more than once.
-
- Usage:
-
- dl = DataLoader()
- (or)
- dl = DataLoader(vault_password='foo')
-
- ds = dl.load('...')
- ds = dl.load_from_file('/path/to/file')
- '''
-
- def __init__(self):
- self._basedir = '.'
- self._FILE_CACHE = dict()
-
- # initialize the vault stuff with an empty password
- self.set_vault_password(None)
-
- def set_vault_password(self, vault_password):
- self._vault_password = vault_password
- self._vault = VaultLib(password=vault_password)
-
- def load(self, data, file_name='<string>', show_content=True):
- '''
- Creates a python datastructure from the given data, which can be either
- a JSON or YAML string.
- '''
-
- try:
- # we first try to load this data as JSON
- return json.loads(data)
- except:
- # if loading JSON failed for any reason, we go ahead
- # and try to parse it as YAML instead
-
- if isinstance(data, AnsibleUnicode):
- # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
- # they are unable to cope with our subclass.
- # Unwrap and re-wrap the unicode so we can keep track of line
- # numbers
- new_data = text_type(data)
- else:
- new_data = data
- try:
- new_data = self._safe_load(new_data, file_name=file_name)
- except YAMLError as yaml_exc:
- self._handle_error(yaml_exc, file_name, show_content)
-
- if isinstance(data, AnsibleUnicode):
- new_data = AnsibleUnicode(new_data)
- new_data.ansible_pos = data.ansible_pos
- return new_data
-
- def load_from_file(self, file_name):
- ''' Loads data from a file, which can contain either JSON or YAML. '''
-
- file_name = self.path_dwim(file_name)
-
- # if the file has already been read in and cached, we'll
- # return those results to avoid more file/vault operations
- if file_name in self._FILE_CACHE:
- parsed_data = self._FILE_CACHE[file_name]
- else:
- # read the file contents and load the data structure from them
- (file_data, show_content) = self._get_file_contents(file_name)
- parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content)
-
- # cache the file contents for next time
- self._FILE_CACHE[file_name] = parsed_data
-
- # return a deep copy here, so the cache is not affected
- return copy.deepcopy(parsed_data)
-
- def path_exists(self, path):
- path = self.path_dwim(path)
- return os.path.exists(path)
-
- def is_file(self, path):
- path = self.path_dwim(path)
- return os.path.isfile(path) or path == os.devnull
-
- def is_directory(self, path):
- path = self.path_dwim(path)
- return os.path.isdir(path)
-
- def list_directory(self, path):
- path = self.path_dwim(path)
- return os.listdir(path)
-
- def is_executable(self, path):
- '''is the given path executable?'''
- path = self.path_dwim(path)
- return is_executable(path)
-
- def _safe_load(self, stream, file_name=None):
- ''' Implements yaml.safe_load(), except using our custom loader class. '''
-
- loader = AnsibleLoader(stream, file_name)
- try:
- return loader.get_single_data()
- finally:
- loader.dispose()
-
- def _get_file_contents(self, file_name):
- '''
- Reads the file contents from the given file name, and will decrypt them
- if they are found to be vault-encrypted.
- '''
- if not file_name or not isinstance(file_name, string_types):
- raise AnsibleParserError("Invalid filename: '%s'" % str(file_name))
-
- if not self.path_exists(file_name) or not self.is_file(file_name):
- raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name)
-
- show_content = True
- try:
- with open(file_name, 'rb') as f:
- data = f.read()
- if self._vault.is_encrypted(data):
- data = self._vault.decrypt(data)
- show_content = False
-
- data = to_unicode(data, errors='strict')
- return (data, show_content)
-
- except (IOError, OSError) as e:
- raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)))
-
- def _handle_error(self, yaml_exc, file_name, show_content):
- '''
- Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
- file name/position where a YAML exception occurred, and raises an AnsibleParserError
- to display the syntax exception information.
- '''
-
- # if the YAML exception contains a problem mark, use it to construct
- # an object the error class can use to display the faulty line
- err_obj = None
- if hasattr(yaml_exc, 'problem_mark'):
- err_obj = AnsibleBaseYAMLObject()
- err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
-
- raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content)
-
- def get_basedir(self):
- ''' returns the current basedir '''
- return self._basedir
-
- def set_basedir(self, basedir):
- ''' sets the base directory, used to find files when a relative path is given '''
-
- if basedir is not None:
- self._basedir = to_unicode(basedir)
-
- def path_dwim(self, given):
- '''
- make relative paths work like folks expect.
- '''
-
- given = unquote(given)
-
- if given.startswith("/"):
- return os.path.abspath(given)
- elif given.startswith("~"):
- return os.path.abspath(os.path.expanduser(given))
- else:
- return os.path.abspath(os.path.join(self._basedir, given))
-
- def path_dwim_relative(self, path, dirname, source):
- '''
- find one file in either a role or playbook dir with or without
- explicitly named dirname subdirs
-
- Used in action plugins and lookups to find supplemental files that
- could be in either place.
- '''
-
- search = []
- isrole = False
-
- # I have full path, nothing else needs to be looked at
- if source.startswith('~') or source.startswith('/'):
- search.append(self.path_dwim(source))
- else:
- # base role/play path + templates/files/vars + relative filename
- search.append(os.path.join(path, dirname, source))
-
- basedir = unfrackpath(path)
-
- # is it a role and if so make sure you get correct base path
- if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \
- or os.path.exists(os.path.join(path,'tasks/main.yml')):
- isrole = True
- if path.endswith('tasks'):
- basedir = unfrackpath(os.path.dirname(path))
-
- cur_basedir = self._basedir
- self.set_basedir(basedir)
- # resolved base role/play path + templates/files/vars + relative filename
- search.append(self.path_dwim(os.path.join(basedir, dirname, source)))
- self.set_basedir(cur_basedir)
-
- if isrole and not source.endswith(dirname):
- # look in role's tasks dir w/o dirname
- search.append(self.path_dwim(os.path.join(basedir, 'tasks', source)))
-
- # try to create absolute path for loader basedir + templates/files/vars + filename
- search.append(self.path_dwim(os.path.join(dirname,source)))
- search.append(self.path_dwim(os.path.join(basedir, source)))
-
- # try to create absolute path for loader basedir + filename
- search.append(self.path_dwim(source))
-
- for candidate in search:
- if os.path.exists(candidate):
- break
-
- return candidate
-
- def read_vault_password_file(self, vault_password_file):
- """
- Read a vault password from a file or if executable, execute the script and
- retrieve password from STDOUT
- """
-
- this_path = os.path.realpath(os.path.expanduser(vault_password_file))
- if not os.path.exists(this_path):
- raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
-
- if self.is_executable(this_path):
- try:
- # STDERR not captured to make it easier for users to prompt for input in their scripts
- p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
- except OSError as e:
- raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
- stdout, stderr = p.communicate()
- self.set_vault_password(stdout.strip('\r\n'))
- else:
- try:
- f = open(this_path, "rb")
- self.set_vault_password(f.read().strip())
- f.close()
- except (OSError, IOError) as e:
- raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))
-
diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py
new file mode 100644
index 00000000000..aaa878bb5f0
--- /dev/null
+++ b/lib/ansible/parsing/dataloader.py
@@ -0,0 +1,294 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import copy
+import json
+import os
+import stat
+import subprocess
+
+from yaml import load, YAMLError
+from ansible.compat.six import text_type, string_types
+
+from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleError
+from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
+from ansible.parsing.vault import VaultLib
+from ansible.parsing.quoting import unquote
+from ansible.parsing.yaml.loader import AnsibleLoader
+from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
+from ansible.module_utils.basic import is_executable
+from ansible.utils.path import unfrackpath
+from ansible.utils.unicode import to_unicode
+
+class DataLoader():
+
+ '''
+ The DataLoader class is used to load and parse YAML or JSON content,
+ either from a given file name or from a string that was previously
+ read in through other means. A Vault password can be specified, and
+ any vault-encrypted files will be decrypted.
+
+ Data read from files will also be cached, so the file will never be
+ read from disk more than once.
+
+ Usage:
+
+ dl = DataLoader()
+ (or)
+ dl = DataLoader(vault_password='foo')
+
+ ds = dl.load('...')
+ ds = dl.load_from_file('/path/to/file')
+ '''
+
+ def __init__(self):
+ self._basedir = '.'
+ self._FILE_CACHE = dict()
+
+ # initialize the vault stuff with an empty password
+ self.set_vault_password(None)
+
+ def set_vault_password(self, vault_password):
+ self._vault_password = vault_password
+ self._vault = VaultLib(password=vault_password)
+
+ def load(self, data, file_name='<string>', show_content=True):
+ '''
+ Creates a python datastructure from the given data, which can be either
+ a JSON or YAML string.
+ '''
+
+ try:
+ # we first try to load this data as JSON
+ return json.loads(data)
+ except:
+ # if loading JSON failed for any reason, we go ahead
+ # and try to parse it as YAML instead
+
+ if isinstance(data, AnsibleUnicode):
+ # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
+ # they are unable to cope with our subclass.
+ # Unwrap and re-wrap the unicode so we can keep track of line
+ # numbers
+ new_data = text_type(data)
+ else:
+ new_data = data
+ try:
+ new_data = self._safe_load(new_data, file_name=file_name)
+ except YAMLError as yaml_exc:
+ self._handle_error(yaml_exc, file_name, show_content)
+
+ if isinstance(data, AnsibleUnicode):
+ new_data = AnsibleUnicode(new_data)
+ new_data.ansible_pos = data.ansible_pos
+ return new_data
+
+ def load_from_file(self, file_name):
+ ''' Loads data from a file, which can contain either JSON or YAML. '''
+
+ file_name = self.path_dwim(file_name)
+
+ # if the file has already been read in and cached, we'll
+ # return those results to avoid more file/vault operations
+ if file_name in self._FILE_CACHE:
+ parsed_data = self._FILE_CACHE[file_name]
+ else:
+ # read the file contents and load the data structure from them
+ (file_data, show_content) = self._get_file_contents(file_name)
+ parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content)
+
+ # cache the file contents for next time
+ self._FILE_CACHE[file_name] = parsed_data
+
+ # return a deep copy here, so the cache is not affected
+ return copy.deepcopy(parsed_data)
+
+ def path_exists(self, path):
+ path = self.path_dwim(path)
+ return os.path.exists(path)
+
+ def is_file(self, path):
+ path = self.path_dwim(path)
+ return os.path.isfile(path) or path == os.devnull
+
+ def is_directory(self, path):
+ path = self.path_dwim(path)
+ return os.path.isdir(path)
+
+ def list_directory(self, path):
+ path = self.path_dwim(path)
+ return os.listdir(path)
+
+ def is_executable(self, path):
+ '''is the given path executable?'''
+ path = self.path_dwim(path)
+ return is_executable(path)
+
+ def _safe_load(self, stream, file_name=None):
+ ''' Implements yaml.safe_load(), except using our custom loader class. '''
+
+ loader = AnsibleLoader(stream, file_name)
+ try:
+ return loader.get_single_data()
+ finally:
+ loader.dispose()
+
+ def _get_file_contents(self, file_name):
+ '''
+ Reads the file contents from the given file name, and will decrypt them
+ if they are found to be vault-encrypted.
+ '''
+ if not file_name or not isinstance(file_name, string_types):
+ raise AnsibleParserError("Invalid filename: '%s'" % str(file_name))
+
+ if not self.path_exists(file_name) or not self.is_file(file_name):
+ raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % file_name)
+
+ show_content = True
+ try:
+ with open(file_name, 'rb') as f:
+ data = f.read()
+ if self._vault.is_encrypted(data):
+ data = self._vault.decrypt(data)
+ show_content = False
+
+ data = to_unicode(data, errors='strict')
+ return (data, show_content)
+
+ except (IOError, OSError) as e:
+ raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)))
+
+ def _handle_error(self, yaml_exc, file_name, show_content):
+ '''
+ Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
+ file name/position where a YAML exception occurred, and raises an AnsibleParserError
+ to display the syntax exception information.
+ '''
+
+ # if the YAML exception contains a problem mark, use it to construct
+ # an object the error class can use to display the faulty line
+ err_obj = None
+ if hasattr(yaml_exc, 'problem_mark'):
+ err_obj = AnsibleBaseYAMLObject()
+ err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
+
+ raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content)
+
+ def get_basedir(self):
+ ''' returns the current basedir '''
+ return self._basedir
+
+ def set_basedir(self, basedir):
+ ''' sets the base directory, used to find files when a relative path is given '''
+
+ if basedir is not None:
+ self._basedir = to_unicode(basedir)
+
+ def path_dwim(self, given):
+ '''
+ make relative paths work like folks expect.
+ '''
+
+ given = unquote(given)
+
+ if given.startswith("/"):
+ return os.path.abspath(given)
+ elif given.startswith("~"):
+ return os.path.abspath(os.path.expanduser(given))
+ else:
+ return os.path.abspath(os.path.join(self._basedir, given))
+
+ def path_dwim_relative(self, path, dirname, source):
+ '''
+ find one file in either a role or playbook dir with or without
+ explicitly named dirname subdirs
+
+ Used in action plugins and lookups to find supplemental files that
+ could be in either place.
+ '''
+
+ search = []
+ isrole = False
+
+ # I have full path, nothing else needs to be looked at
+ if source.startswith('~') or source.startswith('/'):
+ search.append(self.path_dwim(source))
+ else:
+ # base role/play path + templates/files/vars + relative filename
+ search.append(os.path.join(path, dirname, source))
+
+ basedir = unfrackpath(path)
+
+ # is it a role and if so make sure you get correct base path
+ if path.endswith('tasks') and os.path.exists(os.path.join(path,'main.yml')) \
+ or os.path.exists(os.path.join(path,'tasks/main.yml')):
+ isrole = True
+ if path.endswith('tasks'):
+ basedir = unfrackpath(os.path.dirname(path))
+
+ cur_basedir = self._basedir
+ self.set_basedir(basedir)
+ # resolved base role/play path + templates/files/vars + relative filename
+ search.append(self.path_dwim(os.path.join(basedir, dirname, source)))
+ self.set_basedir(cur_basedir)
+
+ if isrole and not source.endswith(dirname):
+ # look in role's tasks dir w/o dirname
+ search.append(self.path_dwim(os.path.join(basedir, 'tasks', source)))
+
+ # try to create absolute path for loader basedir + templates/files/vars + filename
+ search.append(self.path_dwim(os.path.join(dirname,source)))
+ search.append(self.path_dwim(os.path.join(basedir, source)))
+
+ # try to create absolute path for loader basedir + filename
+ search.append(self.path_dwim(source))
+
+ for candidate in search:
+ if os.path.exists(candidate):
+ break
+
+ return candidate
+
+ def read_vault_password_file(self, vault_password_file):
+ """
+ Read a vault password from a file or if executable, execute the script and
+ retrieve password from STDOUT
+ """
+
+ this_path = os.path.realpath(os.path.expanduser(vault_password_file))
+ if not os.path.exists(this_path):
+ raise AnsibleFileNotFound("The vault password file %s was not found" % this_path)
+
+ if self.is_executable(this_path):
+ try:
+ # STDERR not captured to make it easier for users to prompt for input in their scripts
+ p = subprocess.Popen(this_path, stdout=subprocess.PIPE)
+ except OSError as e:
+ raise AnsibleError("Problem running vault password script %s (%s). If this is not a script, remove the executable bit from the file." % (' '.join(this_path), e))
+ stdout, stderr = p.communicate()
+ self.set_vault_password(stdout.strip('\r\n'))
+ else:
+ try:
+ f = open(this_path, "rb")
+ self.set_vault_password(f.read().strip())
+ f.close()
+ except (OSError, IOError) as e:
+ raise AnsibleError("Could not read vault password file %s: %s" % (this_path, e))
diff --git a/lib/ansible/parsing/quoting.py b/lib/ansible/parsing/quoting.py
new file mode 100644
index 00000000000..7b94f906538
--- /dev/null
+++ b/lib/ansible/parsing/quoting.py
@@ -0,0 +1,30 @@
+# (c) 2014 James Cammarata, <jcammarata@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def is_quoted(data):
+ return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\'
+
+def unquote(data):
+ ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
+ if is_quoted(data):
+ return data[1:-1]
+ return data
diff --git a/lib/ansible/parsing/splitter.py b/lib/ansible/parsing/splitter.py
index ef8fc89b008..3bbf6e7e9c9 100644
--- a/lib/ansible/parsing/splitter.py
+++ b/lib/ansible/parsing/splitter.py
@@ -23,6 +23,7 @@ import re
import codecs
from ansible.errors import AnsibleError
+from ansible.parsing.quoting import unquote
# Decode escapes adapted from rspeer's answer here:
# http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
@@ -263,12 +264,3 @@ def split_args(args):
raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")
return params
-
-def is_quoted(data):
- return len(data) > 1 and data[0] == data[-1] and data[0] in ('"', "'") and data[-2] != '\\'
-
-def unquote(data):
- ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
- if is_quoted(data):
- return data[1:-1]
- return data
diff --git a/lib/ansible/parsing/yaml/constructor.py b/lib/ansible/parsing/yaml/constructor.py
index d1a2a01bc28..541165a42fd 100644
--- a/lib/ansible/parsing/yaml/constructor.py
+++ b/lib/ansible/parsing/yaml/constructor.py
@@ -19,9 +19,17 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
-from yaml.constructor import Constructor
+from yaml.constructor import Constructor, ConstructorError
+from yaml.nodes import MappingNode
from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
+try:
+ from __main__ import display
+except ImportError:
+ from ansible.utils.display import Display
+ display = Display()
+
+
class AnsibleConstructor(Constructor):
def __init__(self, file_name=None):
self._ansible_file_name = file_name
@@ -35,10 +43,33 @@ class AnsibleConstructor(Constructor):
data.ansible_pos = self._node_position_info(node)
def construct_mapping(self, node, deep=False):
- ret = AnsibleMapping(super(Constructor, self).construct_mapping(node, deep))
- ret.ansible_pos = self._node_position_info(node)
+ # This first part fixes a problem with pyyaml's handling of duplicate
+ # dict keys.
+ # from SafeConstructor
+ if not isinstance(node, MappingNode):
+ raise ConstructorError(None, None,
+ "expected a mapping node, but found %s" % node.id,
+ node.start_mark)
+ self.flatten_mapping(node)
+ mapping = AnsibleMapping()
+ for key_node, value_node in node.value:
+ key = self.construct_object(key_node, deep=deep)
+ try:
+ hash(key)
+ except TypeError as exc:
+ raise ConstructorError("while constructing a mapping", node.start_mark,
+ "found unacceptable key (%s)" % exc, key_node.start_mark)
- return ret
+ if key in mapping:
+ display.warning('While constructing a mapping%s found a duplicate dict key (%s). Using last value only.' % (node.start_mark, key_node.start_mark))
+
+ value = self.construct_object(value_node, deep=deep)
+ mapping[key] = value
+
+ # Add our extra information to the returned value
+ mapping.ansible_pos = self._node_position_info(node)
+
+ return mapping
def construct_yaml_str(self, node):
# Override the default string handling function
diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py
index 37b3ee75034..888299e1d9e 100644
--- a/lib/ansible/playbook/__init__.py
+++ b/lib/ansible/playbook/__init__.py
@@ -21,9 +21,7 @@ __metaclass__ = type
import os
-from ansible.errors import AnsibleError, AnsibleParserError
-from ansible.parsing import DataLoader
-from ansible.playbook.attribute import Attribute, FieldAttribute
+from ansible.errors import AnsibleParserError
from ansible.playbook.play import Play
from ansible.playbook.playbook_include import PlaybookInclude
from ansible.plugins import get_all_plugin_loaders
diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py
index e8759c32a2d..06681abe763 100644
--- a/lib/ansible/playbook/base.py
+++ b/lib/ansible/playbook/base.py
@@ -26,20 +26,16 @@ import uuid
from functools import partial
from inspect import getmembers
-from io import FileIO
from ansible.compat.six import iteritems, string_types, text_type
from jinja2.exceptions import UndefinedError
from ansible.errors import AnsibleParserError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.playbook.attribute import Attribute, FieldAttribute
-from ansible.template import Templar
from ansible.utils.boolean import boolean
-from ansible.utils.debug import debug
from ansible.utils.vars import combine_vars, isidentifier
-from ansible.template import template
class Base:
diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py
index 7ce24f0b0a9..a8972293d6e 100644
--- a/lib/ansible/vars/__init__.py
+++ b/lib/ansible/vars/__init__.py
@@ -37,7 +37,6 @@ from ansible.cli import CLI
from ansible.compat.six import string_types
from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound
from ansible.inventory.host import Host
-from ansible.parsing import DataLoader
from ansible.plugins import lookup_loader
from ansible.plugins.cache import FactCache
from ansible.template import Templar
diff --git a/samples/multi_queues.py b/samples/multi_queues.py
index 673bb01de60..3b647071c71 100644
--- a/samples/multi_queues.py
+++ b/samples/multi_queues.py
@@ -13,7 +13,7 @@ from ansible.playbook.play_context import PlayContext
from ansible.playbook.task import Task
from ansible.executor.task_executor import TaskExecutor
from ansible.executor.task_result import TaskResult
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.vars import VariableManager
from ansible.utils.debug import debug
diff --git a/test/units/mock/loader.py b/test/units/mock/loader.py
index db69b862a73..2027e183c19 100644
--- a/test/units/mock/loader.py
+++ b/test/units/mock/loader.py
@@ -22,7 +22,7 @@ __metaclass__ = type
import os
from ansible.errors import AnsibleParserError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
class DictDataLoader(DataLoader):
diff --git a/test/units/parsing/test_data_loader.py b/test/units/parsing/test_data_loader.py
index d4d4d9a1d51..19e3107447f 100644
--- a/test/units/parsing/test_data_loader.py
+++ b/test/units/parsing/test_data_loader.py
@@ -26,7 +26,7 @@ from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, mock_open
from ansible.errors import AnsibleParserError
-from ansible.parsing import DataLoader
+from ansible.parsing.dataloader import DataLoader
from ansible.parsing.yaml.objects import AnsibleMapping
class TestDataLoader(unittest.TestCase):
diff --git a/test/units/parsing/test_unquote.py b/test/units/parsing/test_unquote.py
index afb11d4e238..4e5d632066b 100644
--- a/test/units/parsing/test_unquote.py
+++ b/test/units/parsing/test_unquote.py
@@ -23,7 +23,7 @@ __metaclass__ = type
from nose import tools
from ansible.compat.tests import unittest
-from ansible.parsing.splitter import unquote
+from ansible.parsing.quoting import unquote
# Tests using nose's test generators cannot use unittest base class.