cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support for setup targets to ansible-test. (#28544)

Browse source 
* Add support for setup targets to ansible-test.
* Code cleanup.
This commit is contained in:
Matt Clay 2017-08-23 14:09:50 -04:00 committed by GitHub
parent 282e743eb0
commit 5ea8a5e34b
6 changed files with 99 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
test/runner/lib/classification.py
View file

@ -247,14 +247,14 @@ class PathMapper(object):
return minimal
if path.startswith('lib/ansible/modules/'):
module = self.module_names_by_path.get(path)
module_name = self.module_names_by_path.get(path)
if module:
if module_name:
return {
'units': module if module in self.units_modules else None,
'integration': self.posix_integration_by_module.get(module) if ext == '.py' else None,
'windows-integration': self.windows_integration_by_module.get(module) if ext == '.ps1' else None,
'network-integration': self.network_integration_by_module.get(module),
'units': module_name if module_name in self.units_modules else None,
'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None,
'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None,
'network-integration': self.network_integration_by_module.get(module_name),
}
return minimal

2
test/runner/lib/cloud/__init__.py
View file

@ -343,7 +343,7 @@ class CloudEnvironment(CloudBase):
def on_failure(self, target, tries):
"""
:type target: TestTarget
:type target: IntegrationTarget
:type tries: int
"""
pass

8
test/runner/lib/cover.py
View file

@ -92,11 +92,11 @@ def command_coverage_combine(args):
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif '/ansible_module_' in filename:
module = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
if module not in modules:
display.warning('Skipping coverage of unknown module: %s' % module)
module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
if module_name not in modules:
display.warning('Skipping coverage of unknown module: %s' % module_name)
continue
new_name = os.path.abspath(modules[module])
new_name = os.path.abspath(modules[module_name])
display.info('%s -> %s' % (filename, new_name), verbosity=3)
filename = new_name
elif re.search('^(/.*?)?/root/ansible/', filename):

76
test/runner/lib/executor.py
View file

@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function
import json
import os
import collections
import re
import tempfile
import time
@ -244,8 +245,9 @@ def command_posix_integration(args):
"""
:type args: PosixIntegrationConfig
"""
internal_targets = command_integration_filter(args, walk_posix_integration_targets())
command_integration_filtered(args, internal_targets)
all_targets = tuple(walk_posix_integration_targets(include_hidden=True))
internal_targets = command_integration_filter(args, all_targets)
command_integration_filtered(args, internal_targets, all_targets)
def command_network_integration(args):
@ -270,7 +272,8 @@ def command_network_integration(args):
'See also inventory template: %s.template' % (filename, default_filename)
)
internal_targets = command_integration_filter(args, walk_network_integration_targets())
all_targets = tuple(walk_network_integration_targets(include_hidden=True))
internal_targets = command_integration_filter(args, all_targets)
platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
if args.platform:
@ -309,7 +312,7 @@ def command_network_integration(args):
else:
install_command_requirements(args)
command_integration_filtered(args, internal_targets)
command_integration_filtered(args, internal_targets, all_targets)
def network_run(args, platform, version):
@ -377,7 +380,8 @@ def command_windows_integration(args):
if not args.explain and not args.windows and not os.path.isfile(filename):
raise ApplicationError('Use the --windows option or provide an inventory file (see %s.template).' % filename)
internal_targets = command_integration_filter(args, walk_windows_integration_targets())
all_targets = tuple(walk_windows_integration_targets(include_hidden=True))
internal_targets = command_integration_filter(args, all_targets)
if args.windows:
instances = [] # type: list [lib.thread.WrappedThread]
@ -405,7 +409,7 @@ def command_windows_integration(args):
install_command_requirements(args)
try:
command_integration_filtered(args, internal_targets)
command_integration_filtered(args, internal_targets, all_targets)
finally:
pass
@ -477,7 +481,7 @@ def command_integration_filter(args, targets):
:type targets: collections.Iterable[IntegrationTarget]
:rtype: tuple[IntegrationTarget]
"""
targets = tuple(targets)
targets = tuple(target for target in targets if 'hidden/' not in target.aliases)
changes = get_changes_filter(args)
require = (args.require or []) + changes
exclude = (args.exclude or [])
@ -507,16 +511,29 @@ def command_integration_filter(args, targets):
return internal_targets
def command_integration_filtered(args, targets):
def command_integration_filtered(args, targets, all_targets):
"""
:type args: IntegrationConfig
:type targets: tuple[IntegrationTarget]
:type all_targets: tuple[IntegrationTarget]
"""
found = False
passed = []
failed = []
targets_iter = iter(targets)
all_targets_dict = dict((target.name, target) for target in all_targets)
setup_errors = []
setup_targets_executed = set()
for target in all_targets:
for setup_target in target.setup_once + target.setup_always:
if setup_target not in all_targets_dict:
setup_errors.append('Target "%s" contains invalid setup target: %s' % (target.name, setup_target))
if setup_errors:
raise ApplicationError('Found %d invalid setup aliases:\n%s' % (len(setup_errors), '\n'.join(setup_errors)))
test_dir = os.path.expanduser('~/ansible_testing')
@ -561,12 +578,15 @@ def command_integration_filtered(args, targets):
while tries:
tries -= 1
if not args.explain:
# create a fresh test directory for each test target
remove_tree(test_dir)
make_dirs(test_dir)
try:
run_setup_targets(args, test_dir, target.setup_once, all_targets_dict, setup_targets_executed, False)
run_setup_targets(args, test_dir, target.setup_always, all_targets_dict, setup_targets_executed, True)
if not args.explain:
# create a fresh test directory for each test target
remove_tree(test_dir)
make_dirs(test_dir)
if target.script_path:
command_integration_script(args, target)
else:
@ -611,6 +631,34 @@ def command_integration_filtered(args, targets):
len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed)))
def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, always):
"""
:param args: IntegrationConfig
:param test_dir: str
:param target_names: list[str]
:param targets_dict: dict[str, IntegrationTarget]
:param targets_executed: set[str]
:param always: bool
"""
for target_name in target_names:
if not always and target_name in targets_executed:
continue
target = targets_dict[target_name]
if not args.explain:
# create a fresh test directory for each test target
remove_tree(test_dir)
make_dirs(test_dir)
if target.script_path:
command_integration_script(args, target)
else:
command_integration_role(args, target, None)
targets_executed.add(target_name)
def integration_environment(args, target, cmd):
"""
:type args: IntegrationConfig
@ -667,7 +715,7 @@ def command_integration_role(args, target, start_at_task):
"""
:type args: IntegrationConfig
:type target: IntegrationTarget
:type start_at_task: str
:type start_at_task: str | None
"""
display.info('Running %s integration test role' % target.name)

1
test/runner/lib/metadata.py
View file

@ -10,6 +10,7 @@ from lib.util import (
from lib.diff import (
parse_diff,
FileDiff,
)

37
test/runner/lib/target.py
View file

@ -2,6 +2,7 @@
from __future__ import absolute_import, print_function
import collections
import os
import re
import errno
@ -11,7 +12,6 @@ import sys
from lib.util import (
ApplicationError,
ABC,
)
MODULE_EXTENSIONS = '.py', '.ps1'
@ -222,30 +222,33 @@ def walk_sanity_targets():
return walk_test_targets(module_path='lib/ansible/modules/')
def walk_posix_integration_targets():
def walk_posix_integration_targets(include_hidden=False):
"""
:type include_hidden: bool
:rtype: collections.Iterable[IntegrationTarget]
"""
for target in walk_integration_targets():
if 'posix/' in target.aliases:
if 'posix/' in target.aliases or (include_hidden and 'hidden/posix/' in target.aliases):
yield target
def walk_network_integration_targets():
def walk_network_integration_targets(include_hidden=False):
"""
:type include_hidden: bool
:rtype: collections.Iterable[IntegrationTarget]
"""
for target in walk_integration_targets():
if 'network/' in target.aliases:
if 'network/' in target.aliases or (include_hidden and 'hidden/network/' in target.aliases):
yield target
def walk_windows_integration_targets():
def walk_windows_integration_targets(include_hidden=False):
"""
:type include_hidden: bool
:rtype: collections.Iterable[IntegrationTarget]
"""
for target in walk_integration_targets():
if 'windows/' in target.aliases:
if 'windows/' in target.aliases or (include_hidden and 'hidden/windows/' in target.aliases):
yield target
@ -338,7 +341,12 @@ def analyze_integration_target_dependencies(integration_targets):
"""
hidden_role_target_names = set(t.name for t in integration_targets if t.type == 'role' and 'hidden/' in t.aliases)
normal_role_targets = [t for t in integration_targets if t.type == 'role' and 'hidden/' not in t.aliases]
dependencies = dict((target_name, set()) for target_name in hidden_role_target_names)
dependencies = collections.defaultdict(set)
# handle setup dependencies
for target in integration_targets:
for setup_target_name in target.setup_always + target.setup_once:
dependencies[setup_target_name].add(target.name)
# intentionally primitive analysis of role meta to avoid a dependency on pyyaml
for role_target in normal_role_targets:
@ -511,13 +519,13 @@ class IntegrationTarget(CompletionTarget):
# modules
if self.name in modules:
module = self.name
module_name = self.name
elif self.name.startswith('win_') and self.name[4:] in modules:
module = self.name[4:]
module_name = self.name[4:]
else:
module = None
module_name = None
self.modules = tuple(sorted(a for a in static_aliases + tuple([module]) if a in modules))
self.modules = tuple(sorted(a for a in static_aliases + tuple([module_name]) if a in modules))
# groups
@ -576,6 +584,11 @@ class IntegrationTarget(CompletionTarget):
self.aliases = tuple(sorted(set(aliases)))
# configuration
self.setup_once = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/once/'))))
self.setup_always = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/always/'))))
class TargetPatternsNotMatched(ApplicationError):
"""One or more targets were not matched when a match was required."""