Split out module/module_utils unit test execution.

- Unit tests for `modules` and `module_utils` are now limited to importing only `ansible.module_utils` from the `ansible` module.
- Unit tests other than `modules` and `module_utils` are now run only on Python versions supported by the controller (Python 3.8+).
- Unit tests are now run in separate contexts (`controller`, `modules`, `module_utils`), each using separate invocations of `pytest`.

changelogs/fragments/ansible-test-split-unit-tests.yml

@@ -0,0 +1,5 @@
+breaking_changes:
+    - ansible-test - Unit tests for ``modules`` and ``module_utils`` are now limited to importing only ``ansible.module_utils`` from the ``ansible`` module.
+minor_changes:
+    - ansible-test - Unit tests other than ``modules`` and ``module_utils`` are now run only on Python versions supported by the controller (Python 3.8+).
+    - ansible-test - Unit tests are now run in separate contexts (``controller``, ``modules``, ``module_utils``), each using separate invocations of ``pytest``.

test/lib/ansible_test/_data/legacy_collection_loader/__init__.py

@@ -0,0 +1,23 @@
+# (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+# FIXME: decide what of this we want to actually be public/toplevel, put other stuff on a utility class?
+from ._collection_config import AnsibleCollectionConfig
+from ._collection_finder import AnsibleCollectionRef
+from ansible.module_utils.common.text.converters import to_text
+
+
+def resource_from_fqcr(ref):
+    """
+    Return resource from a fully-qualified collection reference,
+    or from a simple resource name.
+    For fully-qualified collection references, this is equivalent to
+    ``AnsibleCollectionRef.from_fqcr(ref).resource``.
+    :param ref: collection reference to parse
+    :return: the resource as a unicode string
+    """
+    ref = to_text(ref, errors='strict')
+    return ref.split(u'.')[-1]

test/lib/ansible_test/_data/legacy_collection_loader/_collection_config.py

@@ -0,0 +1,99 @@
+# (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.module_utils.common.text.converters import to_text
+from ansible.module_utils.six import with_metaclass
+
+
+class _EventSource:
+    def __init__(self):
+        self._handlers = set()
+
+    def __iadd__(self, handler):
+        if not callable(handler):
+            raise ValueError('handler must be callable')
+        self._handlers.add(handler)
+        return self
+
+    def __isub__(self, handler):
+        try:
+            self._handlers.remove(handler)
+        except KeyError:
+            pass
+
+        return self
+
+    def _on_exception(self, handler, exc, *args, **kwargs):
+        # if we return True, we want the caller to re-raise
+        return True
+
+    def fire(self, *args, **kwargs):
+        for h in self._handlers:
+            try:
+                h(*args, **kwargs)
+            except Exception as ex:
+                if self._on_exception(h, ex, *args, **kwargs):
+                    raise
+
+
+class _AnsibleCollectionConfig(type):
+    def __init__(cls, meta, name, bases):
+        cls._collection_finder = None
+        cls._default_collection = None
+        cls._on_collection_load = _EventSource()
+
+    @property
+    def collection_finder(cls):
+        return cls._collection_finder
+
+    @collection_finder.setter
+    def collection_finder(cls, value):
+        if cls._collection_finder:
+            raise ValueError('an AnsibleCollectionFinder has already been configured')
+
+        cls._collection_finder = value
+
+    @property
+    def collection_paths(cls):
+        cls._require_finder()
+        return [to_text(p) for p in cls._collection_finder._n_collection_paths]
+
+    @property
+    def default_collection(cls):
+        return cls._default_collection
+
+    @default_collection.setter
+    def default_collection(cls, value):
+
+        cls._default_collection = value
+
+    @property
+    def on_collection_load(cls):
+        return cls._on_collection_load
+
+    @on_collection_load.setter
+    def on_collection_load(cls, value):
+        if value is not cls._on_collection_load:
+            raise ValueError('on_collection_load is not directly settable (use +=)')
+
+    @property
+    def playbook_paths(cls):
+        cls._require_finder()
+        return [to_text(p) for p in cls._collection_finder._n_playbook_paths]
+
+    @playbook_paths.setter
+    def playbook_paths(cls, value):
+        cls._require_finder()
+        cls._collection_finder.set_playbook_paths(value)
+
+    def _require_finder(cls):
+        if not cls._collection_finder:
+            raise NotImplementedError('an AnsibleCollectionFinder has not been installed in this process')
+
+
+# concrete class of our metaclass type that defines the class properties we want
+class AnsibleCollectionConfig(with_metaclass(_AnsibleCollectionConfig)):
+    pass

test/lib/ansible_test/_data/legacy_collection_loader/_collection_meta.py

@@ -0,0 +1,33 @@
+# (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from yaml import load
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
+
+try:
+    from collections.abc import Mapping   # pylint: disable=ansible-bad-import-from
+except ImportError:
+    from collections import Mapping  # pylint: disable=ansible-bad-import-from
+
+
+def _meta_yml_to_dict(yaml_string_data, content_id):
+    """
+    Converts string YAML dictionary to a Python dictionary. This function may be monkeypatched to another implementation
+    by some tools (eg the import sanity test).
+    :param yaml_string_data: a bytes-ish YAML dictionary
+    :param content_id: a unique ID representing the content to allow other implementations to cache the output
+    :return: a Python dictionary representing the YAML dictionary content
+    """
+    # NB: content_id is passed in, but not used by this implementation
+    routing_dict = load(yaml_string_data, Loader=SafeLoader)
+    if not routing_dict:
+        routing_dict = {}
+    if not isinstance(routing_dict, Mapping):
+        raise ValueError('collection metadata must be an instance of Python Mapping')
+    return routing_dict

test/lib/ansible_test/_data/pytest/plugins/ansible_pytest_collections.py

@@ -3,10 +3,14 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
+import sys
 
 # set by ansible-test to a single directory, rather than a list of directories as supported by Ansible itself
 ANSIBLE_COLLECTIONS_PATH = os.path.join(os.environ['ANSIBLE_COLLECTIONS_PATH'], 'ansible_collections')
 
+# set by ansible-test to the minimum python version supported on the controller
+ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION = tuple(int(x) for x in os.environ['ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION'].split('.'))
+
 
 # this monkeypatch to _pytest.pathlib.resolve_package_path fixes PEP420 resolution for collections in pytest >= 6.0.0
 # NB: this code should never run under py2
@@ -37,8 +41,12 @@ def pytest_configure():
     except AttributeError:
         pytest_configure.executed = True
 
+    if sys.version_info >= ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION:
         # noinspection PyProtectedMember
         from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
+    else:
+        # noinspection PyProtectedMember
+        from ansible_test._internal.legacy_collection_loader._collection_finder import _AnsibleCollectionFinder
 
     # allow unit tests to import code from collections
 

test/lib/ansible_test/_internal/sanity/pylint.py

@@ -139,6 +139,7 @@ class PylintTest(SanitySingleVersion):
             add_context(remaining_paths, 'validate-modules', filter_path('test/lib/ansible_test/_data/sanity/validate-modules/'))
             add_context(remaining_paths, 'validate-modules-unit', filter_path('test/lib/ansible_test/tests/validate-modules-unit/'))
             add_context(remaining_paths, 'sanity', filter_path('test/lib/ansible_test/_data/sanity/'))
+            add_context(remaining_paths, 'legacy-collection-loader', filter_path('test/lib/ansible_test/_data/legacy_collection_loader/'))
             add_context(remaining_paths, 'ansible-test', filter_path('test/lib/'))
             add_context(remaining_paths, 'test', filter_path('test/'))
             add_context(remaining_paths, 'hacking', filter_path('hacking/'))

test/lib/ansible_test/_internal/units/__init__.py

@@ -5,24 +5,34 @@ __metaclass__ = type
 import os
 import sys
 
+from ..io import (
+    write_text_file,
+    make_dirs,
+)
+
 from ..util import (
     ANSIBLE_TEST_DATA_ROOT,
     display,
     get_available_python_versions,
     is_subdir,
     SubprocessError,
-    REMOTE_ONLY_PYTHON_VERSIONS,
+    SUPPORTED_PYTHON_VERSIONS,
+    ANSIBLE_LIB_ROOT,
+    str_to_version,
+    version_to_str,
 )
 
 from ..util_common import (
     intercept_command,
     ResultType,
     handle_layout_messages,
+    create_temp_dir,
 )
 
 from ..ansible_util import (
     ansible_environment,
     check_pyyaml,
+    get_ansible_python_path,
 )
 
 from ..target import (
@@ -47,10 +57,16 @@ from ..executor import (
     Delegate,
     get_changes_filter,
     install_command_requirements,
-    SUPPORTED_PYTHON_VERSIONS,
 )
 
 
+class TestContext:
+    """Contexts that unit tests run in based on the type of content."""
+    controller = 'controller'
+    modules = 'modules'
+    module_utils = 'module_utils'
+
+
 def command_units(args):
     """
     :type args: UnitsConfig
@@ -62,20 +78,33 @@ def command_units(args):
     include = walk_internal_targets(walk_units_targets(), args.include, args.exclude, require)
 
     paths = [target.path for target in include]
-    remote_paths = [path for path in paths
-                    if is_subdir(path, data_context().content.unit_module_path)
-                    or is_subdir(path, data_context().content.unit_module_utils_path)]
+
+    module_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_path)]
+    module_utils_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_utils_path)]
+    controller_paths = sorted(path for path in set(paths) - set(module_paths) - set(module_utils_paths))
+
+    remote_paths = module_paths or module_utils_paths
+
+    test_context_paths = {
+        TestContext.modules: module_paths,
+        TestContext.module_utils: module_utils_paths,
+        TestContext.controller: controller_paths,
+    }
+
+    # temporary definition of "remote" python versions until the full split controller/remote implementation is in place
+    controller_min_python_version = (3, 8)
+    remote_only_python_versions = tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) < controller_min_python_version)
 
     if not paths:
         raise AllTargetsSkipped()
 
-    if args.python and args.python in REMOTE_ONLY_PYTHON_VERSIONS and not remote_paths:
+    if args.python and args.python in remote_only_python_versions and not remote_paths:
         raise AllTargetsSkipped()
 
     if args.delegate:
         raise Delegate(require=changes, exclude=args.exclude)
 
-    version_commands = []
+    test_sets = []
 
     available_versions = sorted(get_available_python_versions(list(SUPPORTED_PYTHON_VERSIONS)).keys())
 
@@ -84,15 +113,41 @@ def command_units(args):
         if args.python and version != args.python_version:
             continue
 
+        test_candidates = []
+
+        for test_context, paths in test_context_paths.items():
+            if test_context == TestContext.controller and version in remote_only_python_versions:
+                continue
+
+            if not paths:
+                continue
+
+            env = ansible_environment(args)
+
+            env.update(
+                PYTHONPATH=get_units_ansible_python_path(args, test_context),
+                ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION=version_to_str(controller_min_python_version),
+            )
+
+            test_candidates.append((test_context, version, paths, env))
+
+        if not test_candidates:
+            continue
+
         if not args.python and version not in available_versions:
             display.warning("Skipping unit tests on Python %s due to missing interpreter." % version)
             continue
 
         if args.requirements_mode != 'skip':
             install_command_requirements(args, version)
+            check_pyyaml(args, version)
 
-        env = ansible_environment(args)
+        test_sets.extend(test_candidates)
 
+    if args.requirements_mode == 'only':
+        sys.exit()
+
+    for test_context, version, paths, env in test_sets:
         cmd = [
             'pytest',
             '--boxed',
@@ -102,7 +157,7 @@ def command_units(args):
             'yes' if args.color else 'no',
             '-p', 'no:cacheprovider',
             '-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'pytest.ini'),
-            '--junit-xml', os.path.join(ResultType.JUNIT.path, 'python%s-units.xml' % version),
+            '--junit-xml', os.path.join(ResultType.JUNIT.path, 'python%s-%s-units.xml' % (version, test_context)),
         ]
 
         if not data_context().content.collection:
@@ -130,30 +185,61 @@ def command_units(args):
         if args.verbosity:
             cmd.append('-' + ('v' * args.verbosity))
 
-        if version in REMOTE_ONLY_PYTHON_VERSIONS:
-            test_paths = remote_paths
-        else:
-            test_paths = paths
+        cmd.extend(paths)
 
-        if not test_paths:
-            continue
-
-        cmd.extend(test_paths)
-
-        version_commands.append((version, cmd, env))
-
-    if args.requirements_mode == 'only':
-        sys.exit()
-
-    for version, command, env in version_commands:
-        check_pyyaml(args, version)
-
-        display.info('Unit test with Python %s' % version)
+        display.info('Unit test %s with Python %s' % (test_context, version))
 
         try:
             with coverage_context(args):
-                intercept_command(args, command, target_name='units', env=env, python_version=version)
+                intercept_command(args, cmd, target_name=test_context, env=env, python_version=version)
         except SubprocessError as ex:
             # pytest exits with status code 5 when all tests are skipped, which isn't an error for our use case
             if ex.status != 5:
                 raise
+
+
+def get_units_ansible_python_path(args, test_context):  # type: (UnitsConfig, str) -> str
+    """
+    Return a directory usable for PYTHONPATH, containing only the modules and module_utils portion of the ansible package.
+    The temporary directory created will be cached for the lifetime of the process and cleaned up at exit.
+    """
+    if test_context == TestContext.controller:
+        return get_ansible_python_path(args)
+
+    try:
+        cache = get_units_ansible_python_path.cache
+    except AttributeError:
+        cache = get_units_ansible_python_path.cache = {}
+
+    python_path = cache.get(test_context)
+
+    if python_path:
+        return python_path
+
+    python_path = create_temp_dir(prefix='ansible-test-')
+    ansible_path = os.path.join(python_path, 'ansible')
+    ansible_test_path = os.path.join(python_path, 'ansible_test')
+
+    write_text_file(os.path.join(ansible_path, '__init__.py'), '', True)
+    os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'module_utils'), os.path.join(ansible_path, 'module_utils'))
+
+    if data_context().content.collection:
+        # built-in runtime configuration for the collection loader
+        make_dirs(os.path.join(ansible_path, 'config'))
+        os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'config', 'ansible_builtin_runtime.yml'), os.path.join(ansible_path, 'config', 'ansible_builtin_runtime.yml'))
+
+        # current collection loader required by all python versions supported by the controller
+        write_text_file(os.path.join(ansible_path, 'utils', '__init__.py'), '', True)
+        os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'collection_loader'), os.path.join(ansible_path, 'utils', 'collection_loader'))
+
+        # legacy collection loader required by all python versions not supported by the controller
+        write_text_file(os.path.join(ansible_test_path, '__init__.py'), '', True)
+        write_text_file(os.path.join(ansible_test_path, '_internal', '__init__.py'), '', True)
+        os.symlink(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'legacy_collection_loader'), os.path.join(ansible_test_path, '_internal', 'legacy_collection_loader'))
+    elif test_context == TestContext.modules:
+        # only non-collection ansible module tests should have access to ansible built-in modules
+        os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'modules'), os.path.join(ansible_path, 'modules'))
+
+    cache[test_context] = python_path
+
+    return python_path