From b834b29e43d144286eb48072b3afa24e015cd514 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 10 Jan 2019 14:43:21 -0800 Subject: [PATCH] Run integration tests from temporary directory. ci_complete --- test/runner/lib/ansible_util.py | 9 +- test/runner/lib/cache.py | 35 ++++ test/runner/lib/cli.py | 8 + test/runner/lib/config.py | 2 + test/runner/lib/executor.py | 77 ++++----- test/runner/lib/integration/__init__.py | 214 ++++++++++++++++++++++++ test/runner/lib/util.py | 25 +++ 7 files changed, 327 insertions(+), 43 deletions(-) create mode 100644 test/runner/lib/cache.py create mode 100644 test/runner/lib/integration/__init__.py diff --git a/test/runner/lib/ansible_util.py b/test/runner/lib/ansible_util.py index 6b0174affcc..bef1e99beea 100644 --- a/test/runner/lib/ansible_util.py +++ b/test/runner/lib/ansible_util.py @@ -14,10 +14,11 @@ from lib.config import ( ) -def ansible_environment(args, color=True): +def ansible_environment(args, color=True, ansible_config=None): """ :type args: CommonConfig :type color: bool + :type ansible_config: str | None :rtype: dict[str, str] """ env = common_environment() @@ -28,12 +29,14 @@ def ansible_environment(args, color=True): if not path.startswith(ansible_path + os.path.pathsep): path = ansible_path + os.path.pathsep + path - if isinstance(args, IntegrationConfig): + if ansible_config: + pass + elif isinstance(args, IntegrationConfig): ansible_config = 'test/integration/%s.cfg' % args.command else: ansible_config = 'test/%s/ansible.cfg' % args.command - if not os.path.exists(ansible_config): + if not args.explain and not os.path.exists(ansible_config): raise ApplicationError('Configuration not found: %s' % ansible_config) ansible = dict( diff --git a/test/runner/lib/cache.py b/test/runner/lib/cache.py new file mode 100644 index 00000000000..20553caf294 --- /dev/null +++ b/test/runner/lib/cache.py @@ -0,0 +1,35 @@ +"""Cache for commonly shared data that is intended to be immutable.""" + +from __future__ import absolute_import, print_function + + +class CommonCache(object): + """Common cache.""" + def __init__(self, args): + """ + :param args: CommonConfig + """ + self.args = args + + def get(self, key, factory): + """ + :param key: str + :param factory: () -> any + :rtype: any + """ + if key not in self.args.cache: + self.args.cache[key] = factory() + + return self.args.cache[key] + + def get_with_args(self, key, factory): + """ + :param key: str + :param factory: (CommonConfig) -> any + :rtype: any + """ + + if key not in self.args.cache: + self.args.cache[key] = factory(self.args) + + return self.args.cache[key] diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py index cab215fe489..86ecbe4a365 100644 --- a/test/runner/lib/cli.py +++ b/test/runner/lib/cli.py @@ -292,6 +292,14 @@ def parse_args(): action='store_true', help='list matching targets instead of running tests') + integration.add_argument('--no-temp-workdir', + action='store_true', + help='do not run tests from a temporary directory (use only for verifying broken tests)') + + integration.add_argument('--no-temp-unicode', + action='store_true', + help='avoid unicode characters in temporary directory (use only for verifying broken tests)') + subparsers = parser.add_subparsers(metavar='COMMAND') subparsers.required = True # work-around for python 3 bug which makes subparsers optional diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index 208011189d6..a4c3ff09c84 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -189,6 +189,8 @@ class IntegrationConfig(TestConfig): self.tags = args.tags self.skip_tags = args.skip_tags self.diff = args.diff + self.no_temp_workdir = args.no_temp_workdir + self.no_temp_unicode = args.no_temp_unicode if self.list_targets: self.explain = True diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 3834905a016..e593d8b2121 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -7,7 +7,6 @@ import os import collections import datetime import re -import tempfile import time import textwrap import functools @@ -55,6 +54,7 @@ from lib.util import ( generate_pip_command, find_python, get_docker_completion, + named_temporary_file, ) from lib.docker_util import ( @@ -108,6 +108,10 @@ from lib.metadata import ( ChangeDescription, ) +from lib.integration import ( + integration_test_environment, +) + SUPPORTED_PYTHON_VERSIONS = ( '2.6', '2.7', @@ -1102,16 +1106,17 @@ def run_setup_targets(args, test_dir, target_names, targets_dict, targets_execut targets_executed.add(target_name) -def integration_environment(args, target, cmd, test_dir, inventory_path): +def integration_environment(args, target, cmd, test_dir, inventory_path, ansible_config): """ :type args: IntegrationConfig :type target: IntegrationTarget :type cmd: list[str] :type test_dir: str :type inventory_path: str + :type ansible_config: str | None :rtype: dict[str, str] """ - env = ansible_environment(args) + env = ansible_environment(args, ansible_config=ansible_config) if args.inject_httptester: env.update(dict( @@ -1154,15 +1159,16 @@ def command_integration_script(args, target, test_dir, inventory_path): """ display.info('Running %s integration test script' % target.name) - cmd = ['./%s' % os.path.basename(target.script_path)] + with integration_test_environment(args, target, inventory_path) as test_env: + cmd = ['./%s' % os.path.basename(target.script_path)] - if args.verbosity: - cmd.append('-' + ('v' * args.verbosity)) + if args.verbosity: + cmd.append('-' + ('v' * args.verbosity)) - env = integration_environment(args, target, cmd, test_dir, inventory_path) - cwd = target.path + env = integration_environment(args, target, cmd, test_dir, test_env.inventory_path, test_env.ansible_config) + cwd = os.path.join(test_env.integration_dir, 'targets', target.name) - intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd) + intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd) def command_integration_role(args, target, start_at_task, test_dir, inventory_path): @@ -1175,8 +1181,6 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa """ display.info('Running %s integration test role' % target.name) - vars_file = 'integration_config.yml' - if isinstance(args, WindowsIntegrationConfig): hosts = 'windows' gather_facts = False @@ -1199,46 +1203,39 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa - { role: %s } ''' % (hosts, gather_facts, target.name) - inventory = os.path.relpath(inventory_path, 'test/integration') + with integration_test_environment(args, target, inventory_path) as test_env: + with named_temporary_file(args=args, directory=test_env.integration_dir, prefix='%s-' % target.name, suffix='.yml', content=playbook) as playbook_path: + filename = os.path.basename(playbook_path) - if '/' in inventory: - inventory = inventory_path + display.info('>>> Playbook: %s\n%s' % (filename, playbook.strip()), verbosity=3) - with tempfile.NamedTemporaryFile(dir='test/integration', prefix='%s-' % target.name, suffix='.yml') as pb_fd: - pb_fd.write(playbook.encode('utf-8')) - pb_fd.flush() + cmd = ['ansible-playbook', filename, '-i', test_env.inventory_path, '-e', '@%s' % test_env.vars_file] - filename = os.path.basename(pb_fd.name) + if start_at_task: + cmd += ['--start-at-task', start_at_task] - display.info('>>> Playbook: %s\n%s' % (filename, playbook.strip()), verbosity=3) + if args.tags: + cmd += ['--tags', args.tags] - cmd = ['ansible-playbook', filename, '-i', inventory, '-e', '@%s' % vars_file] + if args.skip_tags: + cmd += ['--skip-tags', args.skip_tags] - if start_at_task: - cmd += ['--start-at-task', start_at_task] + if args.diff: + cmd += ['--diff'] - if args.tags: - cmd += ['--tags', args.tags] + if isinstance(args, NetworkIntegrationConfig): + if args.testcase: + cmd += ['-e', 'testcase=%s' % args.testcase] - if args.skip_tags: - cmd += ['--skip-tags', args.skip_tags] + if args.verbosity: + cmd.append('-' + ('v' * args.verbosity)) - if args.diff: - cmd += ['--diff'] + env = integration_environment(args, target, cmd, test_dir, test_env.inventory_path, test_env.ansible_config) + cwd = test_env.integration_dir - if isinstance(args, NetworkIntegrationConfig): - if args.testcase: - cmd += ['-e', 'testcase=%s' % args.testcase] + env['ANSIBLE_ROLES_PATH'] = os.path.abspath(os.path.join(test_env.integration_dir, 'targets')) - if args.verbosity: - cmd.append('-' + ('v' * args.verbosity)) - - env = integration_environment(args, target, cmd, test_dir, inventory_path) - cwd = 'test/integration' - - env['ANSIBLE_ROLES_PATH'] = os.path.abspath('test/integration/targets') - - intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd) + intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd) def command_units(args): diff --git a/test/runner/lib/integration/__init__.py b/test/runner/lib/integration/__init__.py new file mode 100644 index 00000000000..ee7f682dcb8 --- /dev/null +++ b/test/runner/lib/integration/__init__.py @@ -0,0 +1,214 @@ +"""Ansible integration test infrastructure.""" + +from __future__ import absolute_import, print_function + +import contextlib +import os +import shutil +import tempfile + +from lib.target import ( + analyze_integration_target_dependencies, + walk_integration_targets, +) + +from lib.config import ( + NetworkIntegrationConfig, + PosixIntegrationConfig, + WindowsIntegrationConfig, +) + +from lib.util import ( + ApplicationError, + display, + make_dirs, +) + +from lib.cache import ( + CommonCache, +) + + +def generate_dependency_map(integration_targets): + """ + :type integration_targets: list[IntegrationTarget] + :rtype: dict[str, set[IntegrationTarget]] + """ + targets_dict = dict((target.name, target) for target in integration_targets) + target_dependencies = analyze_integration_target_dependencies(integration_targets) + dependency_map = {} + + invalid_targets = set() + + for dependency, dependents in target_dependencies.items(): + dependency_target = targets_dict.get(dependency) + + if not dependency_target: + invalid_targets.add(dependency) + continue + + for dependent in dependents: + if dependent not in dependency_map: + dependency_map[dependent] = set() + + dependency_map[dependent].add(dependency_target) + + if invalid_targets: + raise ApplicationError('Non-existent target dependencies: %s' % ', '.join(sorted(invalid_targets))) + + return dependency_map + + +def get_files_needed(target_dependencies): + """ + :type target_dependencies: list[IntegrationTarget] + :rtype: list[str] + """ + files_needed = [] + + for target_dependency in target_dependencies: + files_needed += target_dependency.needs_file + + files_needed = sorted(set(files_needed)) + + invalid_paths = [path for path in files_needed if not os.path.isfile(path)] + + if invalid_paths: + raise ApplicationError('Invalid "needs/file/*" aliases:\n%s' % '\n'.join(invalid_paths)) + + return files_needed + + +@contextlib.contextmanager +def integration_test_environment(args, target, inventory_path): + """ + :type args: IntegrationConfig + :type target: IntegrationTarget + :type inventory_path: str + """ + vars_file = 'integration_config.yml' + + if args.no_temp_workdir or 'no/temp_workdir/' in target.aliases: + display.warning('Disabling the temp work dir is a temporary debugging feature that may be removed in the future without notice.') + + integration_dir = 'test/integration' + ansible_config = os.path.join(integration_dir, '%s.cfg' % args.command) + + inventory_name = os.path.relpath(inventory_path, integration_dir) + + if '/' in inventory_name: + inventory_name = inventory_path + + yield IntegrationEnvironment(integration_dir, inventory_name, ansible_config, vars_file) + return + + root_temp_dir = os.path.expanduser('~/.ansible/test/tmp') + + prefix = '%s-' % target.name + suffix = u'-\u00c5\u00d1\u015a\u00cc\u03b2\u0141\u00c8' + + if args.no_temp_unicode or 'no/temp_unicode/' in target.aliases: + display.warning('Disabling unicode in the temp work dir is a temporary debugging feature that may be removed in the future without notice.') + suffix = '-ansible' + + if isinstance('', bytes): + suffix = suffix.encode('utf-8') + + if args.explain: + temp_dir = os.path.join(root_temp_dir, '%stemp%s' % (prefix, suffix)) + else: + make_dirs(root_temp_dir) + temp_dir = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_temp_dir) + + try: + display.info('Preparing temporary directory: %s' % temp_dir, verbosity=2) + + inventory_names = { + PosixIntegrationConfig: 'inventory', + WindowsIntegrationConfig: 'inventory.winrm', + NetworkIntegrationConfig: 'inventory.networking', + } + + inventory_name = inventory_names[type(args)] + + cache = IntegrationCache(args) + + target_dependencies = sorted([target] + list(cache.dependency_map.get(target.name, set()))) + + files_needed = get_files_needed(target_dependencies) + + integration_dir = os.path.join(temp_dir, 'test/integration') + ansible_config = os.path.join(integration_dir, '%s.cfg' % args.command) + + file_copies = [ + ('test/integration/%s.cfg' % args.command, ansible_config), + ('test/integration/integration_config.yml', os.path.join(integration_dir, vars_file)), + (inventory_path, os.path.join(integration_dir, inventory_name)), + ] + + file_copies += [(path, os.path.join(temp_dir, path)) for path in files_needed] + + directory_copies = [ + (os.path.join('test/integration/targets', target.name), os.path.join(integration_dir, 'targets', target.name)) for target in target_dependencies + ] + + inventory_dir = os.path.dirname(inventory_path) + + host_vars_dir = os.path.join(inventory_dir, 'host_vars') + group_vars_dir = os.path.join(inventory_dir, 'group_vars') + + if os.path.isdir(host_vars_dir): + directory_copies.append((host_vars_dir, os.path.join(integration_dir, os.path.basename(host_vars_dir)))) + + if os.path.isdir(group_vars_dir): + directory_copies.append((group_vars_dir, os.path.join(integration_dir, os.path.basename(group_vars_dir)))) + + directory_copies = sorted(set(directory_copies)) + file_copies = sorted(set(file_copies)) + + if not args.explain: + make_dirs(integration_dir) + + for dir_src, dir_dst in directory_copies: + display.info('Copying %s/ to %s/' % (dir_src, dir_dst), verbosity=2) + + if not args.explain: + shutil.copytree(dir_src, dir_dst, symlinks=True) + + for file_src, file_dst in file_copies: + display.info('Copying %s to %s' % (file_src, file_dst), verbosity=2) + + if not args.explain: + make_dirs(os.path.dirname(file_dst)) + shutil.copy2(file_src, file_dst) + + yield IntegrationEnvironment(integration_dir, inventory_name, ansible_config, vars_file) + finally: + if not args.explain: + shutil.rmtree(temp_dir) + + +class IntegrationEnvironment(object): + """Details about the integration environment.""" + def __init__(self, integration_dir, inventory_path, ansible_config, vars_file): + self.integration_dir = integration_dir + self.inventory_path = inventory_path + self.ansible_config = ansible_config + self.vars_file = vars_file + + +class IntegrationCache(CommonCache): + """Integration cache.""" + @property + def integration_targets(self): + """ + :rtype: list[IntegrationTarget] + """ + return self.get('integration_targets', lambda: list(walk_integration_targets())) + + @property + def dependency_map(self): + """ + :rtype: dict[str, set[IntegrationTarget]] + """ + return self.get('dependency_map', lambda: generate_dependency_map(self.integration_targets)) diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index 5e3411fdffa..47e51849fd1 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -754,6 +754,8 @@ class CommonConfig(object): if is_shippable(): self.redact = True + self.cache = {} + def docker_qualify_image(name): """ @@ -765,6 +767,29 @@ def docker_qualify_image(name): return config.get('name', name) +@contextlib.contextmanager +def named_temporary_file(args, prefix, suffix, directory, content): + """ + :param args: CommonConfig + :param prefix: str + :param suffix: str + :param directory: str + :param content: str | bytes | unicode + :rtype: str + """ + if not isinstance(content, bytes): + content = content.encode('utf-8') + + if args.explain: + yield os.path.join(directory, '%stemp%s' % (prefix, suffix)) + else: + with tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=directory) as tempfile_fd: + tempfile_fd.write(content) + tempfile_fd.flush() + + yield tempfile_fd.name + + def parse_to_list_of_dict(pattern, value): """ :type pattern: str