From b101fda4c6acce465619421fd6710c0514a8f2f7 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 5 Aug 2019 17:40:00 -0700 Subject: [PATCH] Use constants to construct ansible-test paths. --- test/runner/lib/ansible_util.py | 3 ++- test/runner/lib/cli.py | 8 +++++--- test/runner/lib/delegation.py | 7 ++++--- test/runner/lib/executor.py | 11 ++++++----- test/runner/lib/manage_ci.py | 4 ++-- test/runner/lib/sanity/__init__.py | 7 ++++--- test/runner/lib/sanity/compile.py | 4 ++-- test/runner/lib/sanity/import.py | 3 ++- test/runner/lib/sanity/pep8.py | 4 ++-- test/runner/lib/sanity/pslint.py | 7 ++++--- test/runner/lib/sanity/pylint.py | 9 +++++---- test/runner/lib/sanity/rstcheck.py | 4 ++-- test/runner/lib/sanity/shellcheck.py | 4 ++-- test/runner/lib/sanity/validate_modules.py | 4 ++-- test/runner/lib/sanity/yamllint.py | 4 ++-- test/runner/lib/util.py | 3 ++- test/runner/lib/util_common.py | 3 ++- 17 files changed, 50 insertions(+), 39 deletions(-) diff --git a/test/runner/lib/ansible_util.py b/test/runner/lib/ansible_util.py index 9d8cb7ce581..126b41b14f0 100644 --- a/test/runner/lib/ansible_util.py +++ b/test/runner/lib/ansible_util.py @@ -15,6 +15,7 @@ from lib.util import ( find_python, ApplicationError, ANSIBLE_ROOT, + ANSIBLE_TEST_DATA_ROOT, ) from lib.util_common import ( @@ -96,7 +97,7 @@ def check_pyyaml(args, version): return python = find_python(version) - stdout, _dummy = run_command(args, [python, os.path.join(ANSIBLE_ROOT, 'test/runner/yamlcheck.py')], capture=True) + stdout, _dummy = run_command(args, [python, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'yamlcheck.py')], capture=True) if args.explain: return diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py index 079422276ff..64f86c0cddb 100644 --- a/test/runner/lib/cli.py +++ b/test/runner/lib/cli.py @@ -21,6 +21,7 @@ from lib.util import ( generate_pip_command, read_lines_without_comments, MAXFD, + ANSIBLE_TEST_DATA_ROOT, ) from lib.delegation import ( @@ -826,7 +827,8 @@ def complete_remote_shell(prefix, parsed_args, **_): images = sorted(get_remote_completion().keys()) # 2008 doesn't support SSH so we do not add to the list of valid images - images.extend(["windows/%s" % i for i in read_lines_without_comments('test/runner/completion/windows.txt', remove_blank_lines=True) if i != '2008']) + windows_completion_path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', 'windows.txt') + images.extend(["windows/%s" % i for i in read_lines_without_comments(windows_completion_path, remove_blank_lines=True) if i != '2008']) return [i for i in images if i.startswith(prefix)] @@ -850,7 +852,7 @@ def complete_windows(prefix, parsed_args, **_): :type parsed_args: any :rtype: list[str] """ - images = read_lines_without_comments('test/runner/completion/windows.txt', remove_blank_lines=True) + images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', 'windows.txt'), remove_blank_lines=True) return [i for i in images if i.startswith(prefix) and (not parsed_args.windows or i not in parsed_args.windows)] @@ -861,7 +863,7 @@ def complete_network_platform(prefix, parsed_args, **_): :type parsed_args: any :rtype: list[str] """ - images = read_lines_without_comments('test/runner/completion/network.txt', remove_blank_lines=True) + images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', 'network.txt'), remove_blank_lines=True) return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)] diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index e2d3620e8d6..baeb4585a05 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -42,11 +42,12 @@ from lib.util import ( common_environment, pass_vars, display, + ANSIBLE_ROOT, + ANSIBLE_TEST_DATA_ROOT, ) from lib.util_common import ( run_command, - ANSIBLE_ROOT, ) from lib.docker_util import ( @@ -162,7 +163,7 @@ def delegate_tox(args, exclude, require, integration_targets): } for version in versions: - tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', '')] + tox = ['tox', '-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'tox.ini'), '-e', 'py' + version.replace('.', '')] if args.tox_sitepackages: tox.append('--sitepackages') @@ -304,7 +305,7 @@ def delegate_docker(args, exclude, require, integration_targets): test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start - docker_put(args, test_id, os.path.join(ANSIBLE_ROOT, 'test/runner/setup/docker.sh'), '/root/docker.sh') + docker_put(args, test_id, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'docker.sh'), '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 1d265a81dfd..c00cf9e57e9 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -61,6 +61,7 @@ from lib.util import ( COVERAGE_OUTPUT_PATH, cmd_quote, ANSIBLE_ROOT, + ANSIBLE_TEST_DATA_ROOT, get_available_python_versions, is_subdir, ) @@ -314,8 +315,8 @@ def generate_pip_install(pip, command, packages=None): :type packages: list[str] | None :rtype: list[str] | None """ - constraints = os.path.join(ANSIBLE_ROOT, 'test/runner/requirements/constraints.txt') - requirements = os.path.join(ANSIBLE_ROOT, 'test/runner/requirements/%s.txt' % command) + constraints = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt') + requirements = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', '%s.txt' % command) options = [] @@ -610,7 +611,7 @@ def command_windows_integration(args): for remote in [r for r in remotes if r.version != '2008']: manage = ManageWindowsCI(remote) - manage.upload("test/runner/setup/windows-httptester.ps1", watcher_path) + manage.upload(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'windows-httptester.ps1'), watcher_path) # We cannot pass an array of string with -File so we just use a delimiter for multiple values script = "powershell.exe -NoProfile -ExecutionPolicy Bypass -File .\\%s -Hosts \"%s\"" \ @@ -1371,7 +1372,7 @@ def command_units(args): '--color', 'yes' if args.color else 'no', '-p', 'no:cacheprovider', - '-c', os.path.join(ANSIBLE_ROOT, 'test/runner/pytest.ini'), + '-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'pytest.ini'), '--junit-xml', 'test/results/junit/python%s-units.xml' % version, ] @@ -1863,7 +1864,7 @@ class EnvironmentDescription: versions += SUPPORTED_PYTHON_VERSIONS versions += list(set(v.split('.')[0] for v in SUPPORTED_PYTHON_VERSIONS)) - version_check = os.path.join(ANSIBLE_ROOT, 'test/runner/versions.py') + version_check = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'versions.py') python_paths = dict((v, find_executable('python%s' % v, required=False)) for v in sorted(versions)) pip_paths = dict((v, find_executable('pip%s' % v, required=False)) for v in sorted(versions)) program_versions = dict((v, self.get_version([python_paths[v], version_check], warnings)) for v in sorted(python_paths) if python_paths[v]) diff --git a/test/runner/lib/manage_ci.py b/test/runner/lib/manage_ci.py index b9663078449..732d554ac53 100644 --- a/test/runner/lib/manage_ci.py +++ b/test/runner/lib/manage_ci.py @@ -11,12 +11,12 @@ from lib.util import ( ApplicationError, cmd_quote, display, + ANSIBLE_TEST_DATA_ROOT, ) from lib.util_common import ( intercept_command, run_command, - ANSIBLE_ROOT, ) from lib.core_ci import ( @@ -255,7 +255,7 @@ class ManagePosixCI: """Configure remote host for testing. :type python_version: str """ - self.upload(os.path.join(ANSIBLE_ROOT, 'test/runner/setup/remote.sh'), '/tmp') + self.upload(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'remote.sh'), '/tmp') self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s' % (self.core_ci.platform, python_version)) def upload_source(self): diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py index c2355f23d0b..e8f6120c805 100644 --- a/test/runner/lib/sanity/__init__.py +++ b/test/runner/lib/sanity/__init__.py @@ -72,6 +72,7 @@ from lib.env import ( ) COMMAND = 'sanity' +SANITY_ROOT = os.path.join(ANSIBLE_ROOT, 'test', 'sanity') def command_sanity(args): @@ -212,15 +213,15 @@ def collect_code_smell_tests(): """ :rtype: tuple[SanityFunc] """ - skip_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/skip.txt') - ansible_only_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/ansible-only.txt') + skip_file = os.path.join(SANITY_ROOT, 'code-smell', 'skip.txt') + ansible_only_file = os.path.join(SANITY_ROOT, 'code-smell', 'ansible-only.txt') skip_tests = read_lines_without_comments(skip_file, remove_blank_lines=True, optional=True) if not data_context().content.is_ansible: skip_tests += read_lines_without_comments(ansible_only_file, remove_blank_lines=True) - paths = glob.glob(os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/*.py')) + paths = glob.glob(os.path.join(SANITY_ROOT, 'code-smell', '*.py')) paths = sorted(p for p in paths if os.access(p, os.X_OK) and os.path.isfile(p) and os.path.basename(p) not in skip_tests) tests = tuple(SanityCodeSmellTest(p) for p in paths) diff --git a/test/runner/lib/sanity/compile.py b/test/runner/lib/sanity/compile.py index 43661985ee1..6f6aa3961b7 100644 --- a/test/runner/lib/sanity/compile.py +++ b/test/runner/lib/sanity/compile.py @@ -12,6 +12,7 @@ from lib.sanity import ( SanityFailure, SanitySuccess, SanityTargets, + SANITY_ROOT, ) from lib.target import ( @@ -23,7 +24,6 @@ from lib.util import ( display, find_python, parse_to_list_of_dict, - ANSIBLE_ROOT, is_subdir, ) @@ -53,7 +53,7 @@ class CompileTest(SanityMultipleVersion): paths = [target.path for target in targets.include] - cmd = [find_python(python_version), os.path.join(ANSIBLE_ROOT, 'test/sanity/compile/compile.py')] + cmd = [find_python(python_version), os.path.join(SANITY_ROOT, 'compile', 'compile.py')] data = '\n'.join(paths) diff --git a/test/runner/lib/sanity/import.py b/test/runner/lib/sanity/import.py index 9e585a5abfc..15bedfb2cf1 100644 --- a/test/runner/lib/sanity/import.py +++ b/test/runner/lib/sanity/import.py @@ -11,6 +11,7 @@ from lib.sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SANITY_ROOT, ) from lib.target import ( @@ -92,7 +93,7 @@ class ImportTest(SanityMultipleVersion): # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') if not args.explain: - os.symlink(os.path.abspath(os.path.join(ANSIBLE_ROOT, 'test/sanity/import/importer.py')), importer_path) + os.symlink(os.path.abspath(os.path.join(SANITY_ROOT, 'import', 'importer.py')), importer_path) # create a minimal python library python_path = os.path.abspath('test/runner/.tox/import/lib') diff --git a/test/runner/lib/sanity/pep8.py b/test/runner/lib/sanity/pep8.py index 24e33200474..59e84671b75 100644 --- a/test/runner/lib/sanity/pep8.py +++ b/test/runner/lib/sanity/pep8.py @@ -11,6 +11,7 @@ from lib.sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SANITY_ROOT, ) from lib.target import ( @@ -21,7 +22,6 @@ from lib.util import ( SubprocessError, read_lines_without_comments, parse_to_list_of_dict, - ANSIBLE_ROOT, find_python, is_subdir, ) @@ -53,7 +53,7 @@ class Pep8Test(SanitySingleVersion): :type python_version: str :rtype: TestResult """ - current_ignore_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/pep8/current-ignore.txt') + current_ignore_file = os.path.join(SANITY_ROOT, 'pep8', 'current-ignore.txt') current_ignore = sorted(read_lines_without_comments(current_ignore_file, remove_blank_lines=True)) settings = self.load_processor(args) diff --git a/test/runner/lib/sanity/pslint.py b/test/runner/lib/sanity/pslint.py index e48bfb5def4..ac7a0b96a25 100644 --- a/test/runner/lib/sanity/pslint.py +++ b/test/runner/lib/sanity/pslint.py @@ -14,6 +14,7 @@ from lib.sanity import ( SanityFailure, SanitySuccess, SanitySkipped, + SANITY_ROOT, ) from lib.target import ( @@ -23,11 +24,11 @@ from lib.target import ( from lib.util import ( SubprocessError, find_executable, + ANSIBLE_TEST_DATA_ROOT, ) from lib.util_common import ( run_command, - ANSIBLE_ROOT, ) from lib.config import ( @@ -66,9 +67,9 @@ class PslintTest(SanityVersionNeutral): cmds = [] if args.requirements: - cmds.append([os.path.join(ANSIBLE_ROOT, 'test/runner/requirements/sanity.ps1')]) + cmds.append([os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'sanity.ps1')]) - cmds.append([os.path.join(ANSIBLE_ROOT, 'test/sanity/pslint/pslint.ps1')] + paths) + cmds.append([os.path.join(SANITY_ROOT, 'pslint', 'pslint.ps1')] + paths) stdout = '' diff --git a/test/runner/lib/sanity/pylint.py b/test/runner/lib/sanity/pylint.py index 212bfb8945a..6d2db308a7a 100644 --- a/test/runner/lib/sanity/pylint.py +++ b/test/runner/lib/sanity/pylint.py @@ -14,6 +14,7 @@ from lib.sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SANITY_ROOT, ) from lib.target import ( @@ -64,7 +65,7 @@ class PylintTest(SanitySingleVersion): :type python_version: str :rtype: TestResult """ - plugin_dir = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/plugins') + plugin_dir = os.path.join(SANITY_ROOT, 'pylint', 'plugins') plugin_names = sorted(p[0] for p in [ os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__') @@ -185,13 +186,13 @@ class PylintTest(SanitySingleVersion): python, # type: str ): # type: (...) -> t.List[t.Dict[str, str]] """Run pylint using the config specified by the context on the specified paths.""" - rcfile = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/config/%s' % context.split('/')[0]) + rcfile = os.path.join(SANITY_ROOT, 'pylint', 'config', context.split('/')[0]) if not os.path.exists(rcfile): if data_context().content.collection: - rcfile = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/config/collection') + rcfile = os.path.join(SANITY_ROOT, 'pylint', 'config', 'collection') else: - rcfile = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/config/default') + rcfile = os.path.join(SANITY_ROOT, 'pylint', 'config', 'default') parser = ConfigParser() parser.read(rcfile) diff --git a/test/runner/lib/sanity/rstcheck.py b/test/runner/lib/sanity/rstcheck.py index 759ca5e09f9..e1f26c92b3c 100644 --- a/test/runner/lib/sanity/rstcheck.py +++ b/test/runner/lib/sanity/rstcheck.py @@ -11,6 +11,7 @@ from lib.sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SANITY_ROOT, ) from lib.target import ( @@ -21,7 +22,6 @@ from lib.util import ( SubprocessError, parse_to_list_of_dict, read_lines_without_comments, - ANSIBLE_ROOT, find_python, ) @@ -47,7 +47,7 @@ class RstcheckTest(SanitySingleVersion): :type python_version: str :rtype: TestResult """ - ignore_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/rstcheck/ignore-substitutions.txt') + ignore_file = os.path.join(SANITY_ROOT, 'rstcheck', 'ignore-substitutions.txt') ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True))) settings = self.load_processor(args) diff --git a/test/runner/lib/sanity/shellcheck.py b/test/runner/lib/sanity/shellcheck.py index f3839810fe7..d6ec5e736ec 100644 --- a/test/runner/lib/sanity/shellcheck.py +++ b/test/runner/lib/sanity/shellcheck.py @@ -17,6 +17,7 @@ from lib.sanity import ( SanityFailure, SanitySuccess, SanitySkipped, + SANITY_ROOT, ) from lib.target import ( @@ -26,7 +27,6 @@ from lib.target import ( from lib.util import ( SubprocessError, read_lines_without_comments, - ANSIBLE_ROOT, find_executable, ) @@ -56,7 +56,7 @@ class ShellcheckTest(SanityVersionNeutral): :type targets: SanityTargets :rtype: TestResult """ - exclude_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/shellcheck/exclude.txt') + exclude_file = os.path.join(SANITY_ROOT, 'shellcheck', 'exclude.txt') exclude = set(read_lines_without_comments(exclude_file, remove_blank_lines=True, optional=True)) settings = self.load_processor(args) diff --git a/test/runner/lib/sanity/validate_modules.py b/test/runner/lib/sanity/validate_modules.py index d6dff148cdb..d33b345921b 100644 --- a/test/runner/lib/sanity/validate_modules.py +++ b/test/runner/lib/sanity/validate_modules.py @@ -12,6 +12,7 @@ from lib.sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SANITY_ROOT, ) from lib.target import ( @@ -21,7 +22,6 @@ from lib.target import ( from lib.util import ( SubprocessError, display, - ANSIBLE_ROOT, find_python, ) @@ -75,7 +75,7 @@ class ValidateModulesTest(SanitySingleVersion): cmd = [ find_python(python_version), - os.path.join(ANSIBLE_ROOT, 'test/sanity/validate-modules/validate-modules'), + os.path.join(SANITY_ROOT, 'validate-modules', 'validate-modules'), '--format', 'json', '--arg-spec', ] + paths diff --git a/test/runner/lib/sanity/yamllint.py b/test/runner/lib/sanity/yamllint.py index 24b3aa2e90f..5b67fe6ce07 100644 --- a/test/runner/lib/sanity/yamllint.py +++ b/test/runner/lib/sanity/yamllint.py @@ -12,6 +12,7 @@ from lib.sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SANITY_ROOT, ) from lib.target import ( @@ -21,7 +22,6 @@ from lib.target import ( from lib.util import ( SubprocessError, display, - ANSIBLE_ROOT, is_subdir, find_python, ) @@ -92,7 +92,7 @@ class YamllintTest(SanitySingleVersion): """ cmd = [ python, - os.path.join(ANSIBLE_ROOT, 'test/sanity/yamllint/yamllinter.py'), + os.path.join(SANITY_ROOT, 'yamllint', 'yamllinter.py'), ] data = '\n'.join(paths) diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index be7df05c59b..500194ecfc9 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -63,6 +63,7 @@ COVERAGE_CONFIG_PATH = '.coveragerc' COVERAGE_OUTPUT_PATH = 'coverage' ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +ANSIBLE_TEST_DATA_ROOT = os.path.join(ANSIBLE_ROOT, 'test', 'runner') # Modes are set to allow all users the same level of access. # This permits files to be used in tests that change users. @@ -136,7 +137,7 @@ def get_parameterized_completion(cache, name): :rtype: dict[str, dict[str, str]] """ if not cache: - images = read_lines_without_comments(os.path.join(ANSIBLE_ROOT, 'test/runner/completion/%s.txt' % name), remove_blank_lines=True) + images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', '%s.txt' % name), remove_blank_lines=True) cache.update(dict(kvp for kvp in [parse_parameterized_completion(i) for i in images] if kvp)) diff --git a/test/runner/lib/util_common.py b/test/runner/lib/util_common.py index baa056480e7..df498e4a9a0 100644 --- a/test/runner/lib/util_common.py +++ b/test/runner/lib/util_common.py @@ -22,6 +22,7 @@ from lib.util import ( PYTHON_PATHS, raw_command, to_bytes, + ANSIBLE_TEST_DATA_ROOT, ) @@ -210,7 +211,7 @@ def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd cmd = list(cmd) version = python_version or args.python_version interpreter = virtualenv or find_python(version) - inject_path = os.path.join(ANSIBLE_ROOT, 'test/runner/injector') + inject_path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'injector') if not virtualenv: # injection of python into the path is required when not activating a virtualenv