cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Include vars updated to work with directories (#17207)

Browse source 
* New features for include_vars

include_vars.py now allows you to include an entire directory and its nested directories of variable files.

Added Features..

* Ignore by default *.md, *.py, and *.pyc
* Ignore any list of files.
* Only include files nested by depth (default=unlimited)
* Match only files matching (valid regex)
* Sort files alphabetically and load in that order.
* Sort directories alphabetically and load in that order.

```
    - include_vars: 'vars/all.yml'

    - name: include all.yml
      include_vars:
        file: 'vars/all.yml'

    - name: include all yml files in vars/all and all nested directories
      include_vars:
        dir: 'vars/all'

    - name: include all yml files in vars/all and all nested directories and save the output in test.
      include_vars:
        dir: 'vars/all'
        name: test

    - name: include all yml files in vars/services
      include_vars:
        dir: 'vars/services'
        depth: 1

    - name: include only bastion.yml files
      include_vars:
        dir: 'vars'
        files_matching: 'bastion.yml'

    - name: include only all yml files exception bastion.yml
      include_vars:
        dir: 'vars'
        ignore_files: 'bastion.yml'
```

* Added whitelist for file extensisions (yaml, yml, json)

* Removed unit tests in favor of integration tests
This commit is contained in:
Allen Sanabria 2016-08-30 14:34:31 -07:00 committed by Toshio Kuratomi
parent 26118a51f8
commit 03132041fb
8 changed files with 368 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

292
lib/ansible/plugins/action/include_vars.py
View file

@ -1,4 +1,4 @@
# (c) 2013-2014, Benno Joy <benno@ansible.com>
# (c) 2016, Allen Sanabria <asanabria@linuxdynasty.org>
#
# This file is part of Ansible
#
@ -14,53 +14,281 @@
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from os import path, walk
import re
from ansible.errors import AnsibleError
from ansible.plugins.action import ActionBase
from ansible.utils.unicode import to_str
class ActionModule(ActionBase):
TRANSFERS_FILES = False
def _mutually_exclusive(self):
dir_arguments = [
self.source_dir, self.files_matching, self.ignore_files,
self.depth
]
if self.source_file and None not in dir_arguments:
err_msg = (
"Can not include {0} with file argument"
.format(", ".join(self.VALID_DIR_ARGUMENTS))
)
raise AnsibleError(err_msg)
elif self.source_dir and self.source_file:
err_msg = (
"Need to pass either file or dir"
)
raise AnsibleError(err_msg)
def _set_dir_defaults(self):
if not self.depth:
self.depth = 0
if self.files_matching:
self.matcher = re.compile(r'{0}'.format(self.files_matching))
else:
self.matcher = None
if not self.ignore_files:
self.ignore_files = list()
if isinstance(self.ignore_files, str):
self.ignore_files = self.ignore_files.split()
elif isinstance(self.ignore_files, dict):
return {
'failed': True,
'message': '{0} must be a list'.format(self.ignore_files)
}
def _set_args(self):
""" Set instance variables based on the arguments that were passed
"""
self.VALID_DIR_ARGUMENTS = [
'dir', 'depth', 'files_matching', 'ignore_files'
]
self.VALID_FILE_ARGUMENTS = ['file', '_raw_params']
self.GLOBAL_FILE_ARGUMENTS = ['name']
self.VALID_ARGUMENTS = (
self.VALID_DIR_ARGUMENTS + self.VALID_FILE_ARGUMENTS +
self.GLOBAL_FILE_ARGUMENTS
)
for arg in self._task.args:
if arg not in self.VALID_ARGUMENTS:
err_msg = '{0} is not a valid option in debug'.format(arg)
raise AnsibleError(err_msg)
self.return_results_as_name = self._task.args.get('name', None)
self.source_dir = self._task.args.get('dir', None)
self.source_file = self._task.args.get('file', None)
if not self.source_dir and not self.source_file:
self.source_file = self._task.args.get('_raw_params')
self.depth = self._task.args.get('depth', None)
self.files_matching = self._task.args.get('files_matching', None)
self.ignore_files = self._task.args.get('ignore_files', None)
self._mutually_exclusive()
def run(self, tmp=None, task_vars=None):
varname = self._task.args.get('name')
source = self._task.args.get('file')
if not source:
source = self._task.args.get('_raw_params')
if source is None:
raise AnsibleError("No filename was found for the included vars. " + \
"Use `- include_vars: <filename>` or the `file:` option " + \
"to specify the vars filename.", self._task._ds)
if task_vars is None:
""" Load yml files recursively from a directory.
"""
self.VALID_FILE_EXTENSIONS = ['yaml', 'yml', '.json']
if not task_vars:
task_vars = dict()
self.show_content = True
self._set_args()
results = dict()
if self.source_dir:
self._set_dir_defaults()
self._set_root_dir()
if path.exists(self.source_dir):
for root_dir, filenames in self._traverse_dir_depth():
failed, err_msg, updated_results = (
self._load_files_in_dir(root_dir, filenames)
)
if not failed:
results.update(updated_results)
else:
break
else:
failed = True
err_msg = (
'{0} directory does not exist'.format(self.source_dir)
)
else:
try:
self.source_file = self._find_needle('vars', self.source_file)
failed, err_msg, updated_results = (
self._load_files(self.source_file)
)
if not failed:
results.update(updated_results)
except AnsibleError as e:
err_msg = to_str(e)
raise AnsibleError(err_msg)
if self.return_results_as_name:
scope = dict()
scope[self.return_results_as_name] = results
results = scope
result = super(ActionModule, self).run(tmp, task_vars)
try:
source = self._find_needle('vars', source)
except AnsibleError as e:
result['failed'] = True
result['message'] = to_str(e)
return result
if failed:
result['failed'] = failed
result['message'] = err_msg
(data, show_content) = self._loader._get_file_contents(source)
data = self._loader.load(data, show_content)
if data is None:
data = {}
if not isinstance(data, dict):
result['failed'] = True
result['message'] = "%s must be stored as a dictionary/hash" % source
else:
if varname:
scope = {}
scope[varname] = data
data = scope
result['ansible_facts'] = data
result['_ansible_no_log'] = not show_content
result['ansible_facts'] = results
result['_ansible_no_log'] = not self.show_content
return result
def _set_root_dir(self):
if self._task._role:
if self.source_dir.split('/')[0] == 'vars':
path_to_use = (
path.join(self._task._role._role_path, self.source_dir)
)
if path.exists(path_to_use):
self.source_dir = path_to_use
else:
path_to_use = (
path.join(
self._task._role._role_path, 'vars', self.source_dir
)
)
self.source_dir = path_to_use
else:
current_dir = (
"/".join(self._task._ds._data_source.split('/')[:-1])
)
self.source_dir = path.join(current_dir, self.source_dir)
def _traverse_dir_depth(self):
""" Recursively iterate over a directory and sort the files in
alphabetical order. Do not iterate pass the set depth.
The default depth is unlimited.
"""
current_depth = 0
sorted_walk = list(walk(self.source_dir))
sorted_walk.sort(key=lambda x: x[0])
for current_root, current_dir, current_files in sorted_walk:
current_depth += 1
if current_depth <= self.depth or self.depth == 0:
current_files.sort()
yield (current_root, current_files)
else:
break
def _ignore_file(self, filename):
""" Return True if a file matches the list of ignore_files.
Args:
filename (str): The filename that is being matched against.
Returns:
Boolean
"""
for file_type in self.ignore_files:
try:
if re.search(r'{0}$'.format(file_type), filename):
return True
except Exception:
err_msg = 'Invalid regular expression: {0}'.format(file_type)
raise AnsibleError(err_msg)
return False
def _is_valid_file_ext(self, source_file):
""" Verify if source file has a valid extension
Args:
source_file (str): The full path of source file or source file.
Returns:
Bool
"""
success = False
file_ext = source_file.split('.')
if len(file_ext) >= 1:
if file_ext[-1] in self.VALID_FILE_EXTENSIONS:
success = True
return success
return success
def _load_files(self, filename):
""" Loads a file and converts the output into a valid Python dict.
Args:
filename (str): The source file.
Returns:
Tuple (bool, str, dict)
"""
results = dict()
failed = False
err_msg = ''
if not self._is_valid_file_ext(filename):
failed = True
err_msg = (
'{0} does not have a valid extension: {1}'
.format(filename, ', '.join(self.VALID_FILE_EXTENSIONS))
)
return failed, err_msg, results
data, show_content = self._loader._get_file_contents(filename)
self.show_content = show_content
data = self._loader.load(data, show_content)
if not data:
data = dict()
if not isinstance(data, dict):
failed = True
err_msg = (
'{0} must be stored as a dictionary/hash'
.format(filename)
)
else:
results.update(data)
return failed, err_msg, results
def _load_files_in_dir(self, root_dir, var_files):
""" Load the found yml files and update/overwrite the dictionary.
Args:
root_dir (str): The base directory of the list of files that is being passed.
var_files: (list): List of files to iterate over and load into a dictionary.
Returns:
Tuple (bool, str, dict)
"""
results = dict()
failed = False
err_msg = ''
for filename in var_files:
stop_iter = False
# Never include main.yml from a role, as that is the default included by the role
if self._task._role:
if filename == 'main.yml':
stop_iter = True
continue
filepath = path.join(root_dir, filename)
if self.files_matching:
if not self.matcher.search(filename):
stop_iter = True
if not stop_iter and not failed:
if path.exists(filepath) and not self._ignore_file(filename):
failed, err_msg, loaded_data = self._load_files(filepath)
if not failed:
results.update(loaded_data)
return failed, err_msg, results

3
test/integration/roles/test_include_vars/defaults/main.yml Normal file
View file

@ -0,0 +1,3 @@
---
testing: 1
base_dir: defaults

86
test/integration/roles/test_include_vars/tasks/main.yml Normal file
View file

@ -0,0 +1,86 @@
---
- name: verify that the default value is indeed 1
assert:
that:
- "testing == 1"
- "base_dir == 'defaults'"
- name: include the vars/environments/development/all.yml
include_vars:
file: environments/development/all.yml
- name: verify that the default value is indeed 789
assert:
that:
- "testing == 789"
- "base_dir == 'environments/development'"
- name: include the vars/environments/development/all.yml and save results in all
include_vars:
file: environments/development/all.yml
name: all
- name: verify that the values are stored in the all variable
assert:
that:
- "all['testing'] == 789"
- "all['base_dir'] == 'environments/development'"
- name: include the all directory in vars
include_vars:
dir: all
depth: 1
- name: verify that the default value is indeed 123
assert:
that:
- "testing == 123"
- "base_dir == 'all'"
- name: include every directory in vars
include_vars:
dir: vars
- name: verify that the variable overwrite based on alphabetical order
assert:
that:
- "testing == 456"
- "base_dir == 'services'"
- "webapp_containers == 10"
- name: include every directory in vars except files matching webapp.yml
include_vars:
dir: vars
ignore_files:
- webapp.yml
- name: verify that the webapp.yml file was not included
assert:
that:
- "testing == 789"
- "base_dir == 'environments/development'"
- name: include only files matching webapp.yml
include_vars:
dir: environments
files_matching: webapp.yml
- name: verify that only files matching webapp.yml and in the environments directory get loaded.
assert:
that:
- "testing == 101112"
- "base_dir == 'development/services'"
- "webapp_containers == 20"
- name: include only files matching webapp.yml and store results in webapp
include_vars:
dir: environments
files_matching: webapp.yml
name: webapp
- name: verify that only files matching webapp.yml and in the environments directory get loaded into stored variable webapp.
assert:
that:
- "webapp['testing'] == 101112"
- "webapp['base_dir'] == 'development/services'"
- "webapp['webapp_containers'] == 20"

3
test/integration/roles/test_include_vars/vars/all/all.yml Normal file
View file

@ -0,0 +1,3 @@
---
testing: 123
base_dir: all

3
test/integration/roles/test_include_vars/vars/environments/development/all.yml Normal file
View file

@ -0,0 +1,3 @@
---
testing: 789
base_dir: 'environments/development'

4
test/integration/roles/test_include_vars/vars/environments/development/services/webapp.yml Normal file
View file

@ -0,0 +1,4 @@
---
testing: 101112
base_dir: 'development/services'
webapp_containers: 20

4
test/integration/roles/test_include_vars/vars/services/webapp.yml Normal file
View file

@ -0,0 +1,4 @@
---
testing: 456
base_dir: services
webapp_containers: 10

5
test/integration/test_include_vars.yml Normal file
View file

@ -0,0 +1,5 @@
---
- hosts: 127.0.0.1
gather_facts: False
roles:
- { role: test_include_vars }