Prepare ansible-test for collections support.

Another round of changes to prepare ansible-test for supporting collections to help keep later PRs a manageable size.

test/runner/lib/delegation.py

@@ -7,9 +7,6 @@ import re
 import sys
 import tempfile
 
-import lib.pytar
-import lib.thread
-
 from lib.executor import (
     SUPPORTED_PYTHON_VERSIONS,
     HTTPTESTER_HOSTS,
@@ -49,6 +46,7 @@ from lib.util import (
 
 from lib.util_common import (
     run_command,
+    INSTALL_ROOT,
 )
 
 from lib.docker_util import (
@@ -71,6 +69,10 @@ from lib.target import (
     IntegrationTarget,
 )
 
+from lib.payload import (
+    create_payload,
+)
+
 
 def check_delegation_args(args):
     """
@@ -163,7 +165,7 @@ def delegate_tox(args, exclude, require, integration_targets):
 
         tox.append('--')
 
-        cmd = generate_command(args, None, os.path.abspath('bin/ansible-test'), options, exclude, require)
+        cmd = generate_command(args, None, INSTALL_ROOT, INSTALL_ROOT, options, exclude, require)
 
         if not args.python:
             cmd += ['--python', version]
@@ -224,7 +226,11 @@ def delegate_docker(args, exclude, require, integration_targets):
     }
 
     python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw)
-    cmd = generate_command(args, python_interpreter, '/root/ansible/bin/ansible-test', options, exclude, require)
+
+    install_root = '/root/ansible'
+    content_root = install_root
+
+    cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require)
 
     if isinstance(args, TestConfig):
         if args.coverage and not args.coverage_label:
@@ -243,13 +249,7 @@ def delegate_docker(args, exclude, require, integration_targets):
 
     with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
         try:
-            if not args.explain:
+            create_payload(args, local_source_fd.name)
-                if args.docker_keep_git:
-                    tar_filter = lib.pytar.AllowGitTarFilter()
-                else:
-                    tar_filter = lib.pytar.DefaultTarFilter()
-
-                lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter)
 
             if use_httptester:
                 httptester_id = run_httptester(args)
@@ -296,7 +296,7 @@ def delegate_docker(args, exclude, require, integration_targets):
                 test_id = test_id.strip()
 
             # write temporary files to /root since /tmp isn't ready immediately on container start
-            docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh')
+            docker_put(args, test_id, os.path.join(INSTALL_ROOT, 'test/runner/setup/docker.sh'), '/root/docker.sh')
             docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
             docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz')
             docker_exec(args, test_id, ['mkdir', '/root/ansible'])
@@ -310,16 +310,16 @@ def delegate_docker(args, exclude, require, integration_targets):
             # also disconnect from the network once requirements have been installed
             if isinstance(args, UnitsConfig):
                 writable_dirs = [
-                    '/root/ansible/.pytest_cache',
+                    os.path.join(content_root, '.pytest_cache'),
                 ]
 
                 docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs)
                 docker_exec(args, test_id, ['chmod', '777'] + writable_dirs)
 
-                docker_exec(args, test_id, ['find', '/root/ansible/test/results/', '-type', 'd', '-exec', 'chmod', '777', '{}', '+'])
+                docker_exec(args, test_id, ['find', os.path.join(content_root, 'test/results/'), '-type', 'd', '-exec', 'chmod', '777', '{}', '+'])
 
                 docker_exec(args, test_id, ['chmod', '755', '/root'])
-                docker_exec(args, test_id, ['chmod', '644', '/root/ansible/%s' % args.metadata_path])
+                docker_exec(args, test_id, ['chmod', '644', os.path.join(content_root, args.metadata_path)])
 
                 docker_exec(args, test_id, ['useradd', 'pytest', '--create-home'])
 
@@ -338,7 +338,7 @@ def delegate_docker(args, exclude, require, integration_targets):
                 docker_exec(args, test_id, cmd, options=cmd_options)
             finally:
                 with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd:
-                    docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results'])
+                    docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', os.path.join(content_root, 'test'), 'results'])
                     docker_get(args, test_id, '/root/results.tgz', local_result_fd.name)
                     run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test'])
         finally:
@@ -377,6 +377,7 @@ def delegate_remote(args, exclude, require, integration_targets):
 
     httptester_id = None
     ssh_options = []
+    content_root = None
 
     try:
         core_ci.start()
@@ -399,7 +400,11 @@ def delegate_remote(args, exclude, require, integration_targets):
             }
 
             python_interpreter = get_python_interpreter(args, get_remote_completion(), args.remote)
-            cmd = generate_command(args, python_interpreter, 'ansible/bin/ansible-test', options, exclude, require)
+
+            install_root = 'ansible'
+            content_root = install_root
+
+            cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require)
 
             if httptester_id:
                 cmd += ['--inject-httptester']
@@ -440,8 +445,8 @@ def delegate_remote(args, exclude, require, integration_targets):
                 if args.raw:
                     download = False
 
-            if download:
+            if download and content_root:
-                manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results')
+                manage.ssh('rm -rf /tmp/results && cp -a %s/test/results /tmp/results && chmod -R a+r /tmp/results' % content_root)
                 manage.download('/tmp/results', 'test')
     finally:
         if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success):
@@ -451,11 +456,12 @@ def delegate_remote(args, exclude, require, integration_targets):
             docker_rm(args, httptester_id)
 
 
-def generate_command(args, python_interpreter, path, options, exclude, require):
+def generate_command(args, python_interpreter, install_root, content_root, options, exclude, require):
     """
     :type args: EnvironmentConfig
     :type python_interpreter: str | None
-    :type path: str
+    :type install_root: str
+    :type content_root: str
     :type options: dict[str, int]
     :type exclude: list[str]
     :type require: list[str]
@@ -463,7 +469,7 @@ def generate_command(args, python_interpreter, path, options, exclude, require):
     """
     options['--color'] = 1
 
-    cmd = [path]
+    cmd = [os.path.join(install_root, 'bin/ansible-test')]
 
     if python_interpreter:
         cmd = [python_interpreter] + cmd
@@ -472,7 +478,14 @@ def generate_command(args, python_interpreter, path, options, exclude, require):
     # This is only needed because ansible-test relies on Python's file system encoding.
     # Environments that do not have the locale configured are thus unable to work with unicode file paths.
     # Examples include FreeBSD and some Linux containers.
-    cmd = ['/usr/bin/env', 'LC_ALL=en_US.UTF-8'] + cmd
+    env_vars = dict(
+        LC_ALL='en_US.UTF-8',
+        ANSIBLE_TEST_CONTENT_ROOT=content_root,
+    )
+
+    env_args = ['%s=%s' % (key, env_vars[key]) for key in sorted(env_vars)]
+
+    cmd = ['/usr/bin/env'] + env_args + cmd
 
     cmd += list(filter_options(args, sys.argv[1:], options, exclude, require))
     cmd += ['--color', 'yes' if args.color else 'no']

test/runner/lib/executor.py

@@ -20,7 +20,6 @@ import shutil
 
 import lib.types as t
 
-import lib.pytar
 import lib.thread
 
 from lib.core_ci import (
@@ -279,6 +278,9 @@ def generate_egg_info(args):
     """
     :type args: EnvironmentConfig
     """
+    if not os.path.exists(os.path.join(INSTALL_ROOT, 'setup.py')):
+        return
+
     if os.path.isdir(os.path.join(INSTALL_ROOT, 'lib/ansible.egg-info')):
         return
 
@@ -292,8 +294,8 @@ def generate_pip_install(pip, command, packages=None):
     :type packages: list[str] | None
     :rtype: list[str] | None
     """
-    constraints = 'test/runner/requirements/constraints.txt'
+    constraints = os.path.join(INSTALL_ROOT, 'test/runner/requirements/constraints.txt')
-    requirements = 'test/runner/requirements/%s.txt' % command
+    requirements = os.path.join(INSTALL_ROOT, 'test/runner/requirements/%s.txt' % command)
 
     options = []
 

test/runner/lib/import_analysis.py

@@ -122,10 +122,14 @@ def enumerate_module_utils():
     module_utils = []
     base_path = 'lib/ansible/module_utils'
 
+    paths = []
+
     for root, _dir_names, file_names in os.walk(base_path):
         for file_name in file_names:
-            path = os.path.join(root, file_name)
+            paths.append(os.path.join(root, file_name))
-            name, ext = os.path.splitext(file_name)
+
+    for path in paths:
+        name, ext = os.path.splitext(path)
 
         if path == 'lib/ansible/module_utils/__init__.py':
             continue
@@ -133,10 +137,10 @@ def enumerate_module_utils():
         if ext != '.py':
             continue
 
-            if name == '__init__':
+        if name.endswith('/__init__'):
-                module_util = root
+            module_util = os.path.dirname(name)
         else:
-                module_util = os.path.join(root, name)
+            module_util = name
 
         module_utils.append(module_util[4:].replace('/', '.'))
 

test/runner/lib/manage_ci.py

@@ -6,8 +6,6 @@ import os
 import tempfile
 import time
 
-import lib.pytar
-
 from lib.util import (
     SubprocessError,
     ApplicationError,
@@ -17,6 +15,7 @@ from lib.util import (
 from lib.util_common import (
     intercept_command,
     run_command,
+    INSTALL_ROOT,
 )
 
 from lib.core_ci import (
@@ -31,6 +30,10 @@ from lib.config import (
     ShellConfig,
 )
 
+from lib.payload import (
+    create_payload,
+)
+
 
 class ManageWindowsCI:
     """Manage access to a Windows instance provided by Ansible Core CI."""
@@ -237,7 +240,7 @@ class ManagePosixCI:
         """Configure remote host for testing.
         :type python_version: str
         """
-        self.upload('test/runner/setup/remote.sh', '/tmp')
+        self.upload(os.path.join(INSTALL_ROOT, 'test/runner/setup/remote.sh'), '/tmp')
         self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s' % (self.core_ci.platform, python_version))
 
     def upload_source(self):
@@ -246,8 +249,7 @@ class ManagePosixCI:
             remote_source_dir = '/tmp'
             remote_source_path = os.path.join(remote_source_dir, os.path.basename(local_source_fd.name))
 
-            if not self.core_ci.args.explain:
+            create_payload(self.core_ci.args, local_source_fd.name)
-                lib.pytar.create_tarfile(local_source_fd.name, '.', lib.pytar.DefaultTarFilter())
 
             self.upload(local_source_fd.name, remote_source_dir)
             self.ssh('rm -rf ~/ansible && mkdir ~/ansible && cd ~/ansible && tar oxzf %s' % remote_source_path)

test/runner/lib/payload.py

@@ -0,0 +1,27 @@
+"""Payload management for sending Ansible files and test content to other systems (VMs, containers)."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from lib.config import (
+    CommonConfig,
+    EnvironmentConfig,
+)
+
+from lib.pytar import (
+    AllowGitTarFilter,
+    create_tarfile,
+    DefaultTarFilter,
+)
+
+
+def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
+    """Create a payload for delegation."""
+    if args.explain:
+        return
+
+    if isinstance(args, EnvironmentConfig) and args.docker_keep_git:
+        tar_filter = AllowGitTarFilter()
+    else:
+        tar_filter = DefaultTarFilter()
+
+    create_tarfile(dst_path, '.', tar_filter)

test/runner/lib/sanity/import.py

@@ -20,11 +20,13 @@ from lib.util import (
     read_lines_without_comments,
     parse_to_list_of_dict,
     make_dirs,
+    is_subdir,
 )
 
 from lib.util_common import (
     intercept_command,
     run_command,
+    INSTALL_ROOT,
 )
 
 from lib.ansible_util import (
@@ -58,7 +60,7 @@ class ImportTest(SanityMultipleVersion):
             i.path
             for i in targets.include
             if os.path.splitext(i.path)[1] == '.py' and
-            (i.path.startswith('lib/ansible/modules/') or i.path.startswith('lib/ansible/module_utils/')) and
+            (is_subdir(i.path, 'lib/ansible/modules/') or is_subdir(i.path, 'lib/ansible/module_utils/')) and
             i.path not in skip_paths_set
         )
 
@@ -85,7 +87,7 @@ class ImportTest(SanityMultipleVersion):
         # add the importer to our virtual environment so it can be accessed through the coverage injector
         importer_path = os.path.join(virtual_environment_bin, 'importer.py')
         if not args.explain:
-            os.symlink(os.path.abspath('test/sanity/import/importer.py'), importer_path)
+            os.symlink(os.path.abspath(os.path.join(INSTALL_ROOT, 'test/sanity/import/importer.py')), importer_path)
 
         # create a minimal python library
         python_path = os.path.abspath('test/runner/.tox/import/lib')

test/runner/lib/sanity/pslint.py

@@ -25,6 +25,7 @@ from lib.util import (
 
 from lib.util_common import (
     run_command,
+    INSTALL_ROOT,
 )
 
 from lib.config import (
@@ -84,8 +85,8 @@ class PslintTest(SanitySingleVersion):
 
         # Make sure requirements are installed before running sanity checks
         cmds = [
-            ['test/runner/requirements/sanity.ps1'],
+            [os.path.join(INSTALL_ROOT, 'test/runner/requirements/sanity.ps1')],
-            ['test/sanity/pslint/pslint.ps1'] + paths
+            [os.path.join(INSTALL_ROOT, 'test/sanity/pslint/pslint.ps1')] + paths
         ]
 
         stdout = ''

test/runner/lib/sanity/pylint.py

@@ -24,6 +24,7 @@ from lib.util import (
     read_lines_without_comments,
     ConfigParser,
     INSTALL_ROOT,
+    is_subdir,
 )
 
 from lib.util_common import (
@@ -113,16 +114,17 @@ class PylintTest(SanitySingleVersion):
 
         skip_paths_set = set(skip_paths)
 
-        paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set)
+        paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/')) and i.path not in skip_paths_set)
 
-        module_paths = [p.split(os.path.sep) for p in paths if p.startswith('lib/ansible/modules/')]
+        module_paths = [p.split(os.path.sep) for p in paths if is_subdir(p, 'lib/ansible/modules/')]
         module_dirs = sorted(set([p[3] for p in module_paths if len(p) > 4]))
 
         large_module_group_threshold = 500
         large_module_groups = [key for key, value in
                                itertools.groupby(module_paths, lambda p: p[3] if len(p) > 4 else '') if len(list(value)) > large_module_group_threshold]
 
-        large_module_group_paths = [p.split(os.path.sep) for p in paths if any(p.startswith('lib/ansible/modules/%s/' % g) for g in large_module_groups)]
+        large_module_group_paths = [p.split(os.path.sep) for p in paths
+                                    if any(is_subdir(p, os.path.join('lib/ansible/modules/', g)) for g in large_module_groups)]
         large_module_group_dirs = sorted(set([os.path.sep.join(p[3:5]) for p in large_module_group_paths if len(p) > 5]))
 
         contexts = []
@@ -148,7 +150,7 @@ class PylintTest(SanitySingleVersion):
                 :type path_to_filter: str
                 :rtype: bool
                 """
-                return path_to_filter.startswith(path_filter)
+                return is_subdir(path_to_filter, path_filter)
 
             return context_filter
 

test/runner/lib/sanity/sanity_docs.py

@@ -4,6 +4,10 @@ __metaclass__ = type
 
 import os
 
+from lib.util import (
+    INSTALL_ROOT,
+)
+
 from lib.sanity import (
     SanitySingleVersion,
     SanityMessage,
@@ -26,7 +30,7 @@ class SanityDocsTest(SanitySingleVersion):
         :type targets: SanityTargets
         :rtype: TestResult
         """
-        sanity_dir = 'docs/docsite/rst/dev_guide/testing/sanity'
+        sanity_dir = os.path.join(INSTALL_ROOT, 'docs/docsite/rst/dev_guide/testing/sanity')
         sanity_docs = set(part[0] for part in (os.path.splitext(name) for name in os.listdir(sanity_dir)) if part[1] == '.rst')
         sanity_tests = set(sanity_test.name for sanity_test in sanity_get_tests())
 

test/runner/lib/test.py

@@ -8,6 +8,7 @@ import os
 
 from lib.util import (
     display,
+    make_dirs,
 )
 
 from lib.config import (
@@ -332,6 +333,8 @@ class TestFailure(TestResult):
         if args.explain:
             return
 
+        make_dirs(os.path.dirname(path))
+
         with open(path, 'w') as bot_fd:
             json.dump(bot_data, bot_fd, indent=4, sort_keys=True)
             bot_fd.write('\n')

test/runner/lib/types.py

@@ -5,6 +5,7 @@ __metaclass__ = type
 try:
     from typing import (
         Any,
+        Callable,
         Dict,
         FrozenSet,
         Iterable,