Split out module/module_utils unit test execution.

- Unit tests for `modules` and `module_utils` are now limited to importing only `ansible.module_utils` from the `ansible` module.
- Unit tests other than `modules` and `module_utils` are now run only on Python versions supported by the controller (Python 3.8+).
- Unit tests are now run in separate contexts (`controller`, `modules`, `module_utils`), each using separate invocations of `pytest`.
This commit is contained in:
Matt Clay 2021-04-16 21:14:40 -07:00
parent 3740d7b028
commit 7786dbbdc1
8 changed files with 1346 additions and 30 deletions

View file

@ -0,0 +1,5 @@
breaking_changes:
- ansible-test - Unit tests for ``modules`` and ``module_utils`` are now limited to importing only ``ansible.module_utils`` from the ``ansible`` module.
minor_changes:
- ansible-test - Unit tests other than ``modules`` and ``module_utils`` are now run only on Python versions supported by the controller (Python 3.8+).
- ansible-test - Unit tests are now run in separate contexts (``controller``, ``modules``, ``module_utils``), each using separate invocations of ``pytest``.

View file

@ -0,0 +1,23 @@
# (c) 2019 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
# FIXME: decide what of this we want to actually be public/toplevel, put other stuff on a utility class?
from ._collection_config import AnsibleCollectionConfig
from ._collection_finder import AnsibleCollectionRef
from ansible.module_utils.common.text.converters import to_text
def resource_from_fqcr(ref):
"""
Return resource from a fully-qualified collection reference,
or from a simple resource name.
For fully-qualified collection references, this is equivalent to
``AnsibleCollectionRef.from_fqcr(ref).resource``.
:param ref: collection reference to parse
:return: the resource as a unicode string
"""
ref = to_text(ref, errors='strict')
return ref.split(u'.')[-1]

View file

@ -0,0 +1,99 @@
# (c) 2019 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.module_utils.common.text.converters import to_text
from ansible.module_utils.six import with_metaclass
class _EventSource:
def __init__(self):
self._handlers = set()
def __iadd__(self, handler):
if not callable(handler):
raise ValueError('handler must be callable')
self._handlers.add(handler)
return self
def __isub__(self, handler):
try:
self._handlers.remove(handler)
except KeyError:
pass
return self
def _on_exception(self, handler, exc, *args, **kwargs):
# if we return True, we want the caller to re-raise
return True
def fire(self, *args, **kwargs):
for h in self._handlers:
try:
h(*args, **kwargs)
except Exception as ex:
if self._on_exception(h, ex, *args, **kwargs):
raise
class _AnsibleCollectionConfig(type):
def __init__(cls, meta, name, bases):
cls._collection_finder = None
cls._default_collection = None
cls._on_collection_load = _EventSource()
@property
def collection_finder(cls):
return cls._collection_finder
@collection_finder.setter
def collection_finder(cls, value):
if cls._collection_finder:
raise ValueError('an AnsibleCollectionFinder has already been configured')
cls._collection_finder = value
@property
def collection_paths(cls):
cls._require_finder()
return [to_text(p) for p in cls._collection_finder._n_collection_paths]
@property
def default_collection(cls):
return cls._default_collection
@default_collection.setter
def default_collection(cls, value):
cls._default_collection = value
@property
def on_collection_load(cls):
return cls._on_collection_load
@on_collection_load.setter
def on_collection_load(cls, value):
if value is not cls._on_collection_load:
raise ValueError('on_collection_load is not directly settable (use +=)')
@property
def playbook_paths(cls):
cls._require_finder()
return [to_text(p) for p in cls._collection_finder._n_playbook_paths]
@playbook_paths.setter
def playbook_paths(cls, value):
cls._require_finder()
cls._collection_finder.set_playbook_paths(value)
def _require_finder(cls):
if not cls._collection_finder:
raise NotImplementedError('an AnsibleCollectionFinder has not been installed in this process')
# concrete class of our metaclass type that defines the class properties we want
class AnsibleCollectionConfig(with_metaclass(_AnsibleCollectionConfig)):
pass

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,33 @@
# (c) 2019 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from yaml import load
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
try:
from collections.abc import Mapping # pylint: disable=ansible-bad-import-from
except ImportError:
from collections import Mapping # pylint: disable=ansible-bad-import-from
def _meta_yml_to_dict(yaml_string_data, content_id):
"""
Converts string YAML dictionary to a Python dictionary. This function may be monkeypatched to another implementation
by some tools (eg the import sanity test).
:param yaml_string_data: a bytes-ish YAML dictionary
:param content_id: a unique ID representing the content to allow other implementations to cache the output
:return: a Python dictionary representing the YAML dictionary content
"""
# NB: content_id is passed in, but not used by this implementation
routing_dict = load(yaml_string_data, Loader=SafeLoader)
if not routing_dict:
routing_dict = {}
if not isinstance(routing_dict, Mapping):
raise ValueError('collection metadata must be an instance of Python Mapping')
return routing_dict

View file

@ -3,10 +3,14 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import os import os
import sys
# set by ansible-test to a single directory, rather than a list of directories as supported by Ansible itself # set by ansible-test to a single directory, rather than a list of directories as supported by Ansible itself
ANSIBLE_COLLECTIONS_PATH = os.path.join(os.environ['ANSIBLE_COLLECTIONS_PATH'], 'ansible_collections') ANSIBLE_COLLECTIONS_PATH = os.path.join(os.environ['ANSIBLE_COLLECTIONS_PATH'], 'ansible_collections')
# set by ansible-test to the minimum python version supported on the controller
ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION = tuple(int(x) for x in os.environ['ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION'].split('.'))
# this monkeypatch to _pytest.pathlib.resolve_package_path fixes PEP420 resolution for collections in pytest >= 6.0.0 # this monkeypatch to _pytest.pathlib.resolve_package_path fixes PEP420 resolution for collections in pytest >= 6.0.0
# NB: this code should never run under py2 # NB: this code should never run under py2
@ -37,8 +41,12 @@ def pytest_configure():
except AttributeError: except AttributeError:
pytest_configure.executed = True pytest_configure.executed = True
if sys.version_info >= ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION:
# noinspection PyProtectedMember # noinspection PyProtectedMember
from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
else:
# noinspection PyProtectedMember
from ansible_test._internal.legacy_collection_loader._collection_finder import _AnsibleCollectionFinder
# allow unit tests to import code from collections # allow unit tests to import code from collections

View file

@ -139,6 +139,7 @@ class PylintTest(SanitySingleVersion):
add_context(remaining_paths, 'validate-modules', filter_path('test/lib/ansible_test/_data/sanity/validate-modules/')) add_context(remaining_paths, 'validate-modules', filter_path('test/lib/ansible_test/_data/sanity/validate-modules/'))
add_context(remaining_paths, 'validate-modules-unit', filter_path('test/lib/ansible_test/tests/validate-modules-unit/')) add_context(remaining_paths, 'validate-modules-unit', filter_path('test/lib/ansible_test/tests/validate-modules-unit/'))
add_context(remaining_paths, 'sanity', filter_path('test/lib/ansible_test/_data/sanity/')) add_context(remaining_paths, 'sanity', filter_path('test/lib/ansible_test/_data/sanity/'))
add_context(remaining_paths, 'legacy-collection-loader', filter_path('test/lib/ansible_test/_data/legacy_collection_loader/'))
add_context(remaining_paths, 'ansible-test', filter_path('test/lib/')) add_context(remaining_paths, 'ansible-test', filter_path('test/lib/'))
add_context(remaining_paths, 'test', filter_path('test/')) add_context(remaining_paths, 'test', filter_path('test/'))
add_context(remaining_paths, 'hacking', filter_path('hacking/')) add_context(remaining_paths, 'hacking', filter_path('hacking/'))

View file

@ -5,24 +5,34 @@ __metaclass__ = type
import os import os
import sys import sys
from ..io import (
write_text_file,
make_dirs,
)
from ..util import ( from ..util import (
ANSIBLE_TEST_DATA_ROOT, ANSIBLE_TEST_DATA_ROOT,
display, display,
get_available_python_versions, get_available_python_versions,
is_subdir, is_subdir,
SubprocessError, SubprocessError,
REMOTE_ONLY_PYTHON_VERSIONS, SUPPORTED_PYTHON_VERSIONS,
ANSIBLE_LIB_ROOT,
str_to_version,
version_to_str,
) )
from ..util_common import ( from ..util_common import (
intercept_command, intercept_command,
ResultType, ResultType,
handle_layout_messages, handle_layout_messages,
create_temp_dir,
) )
from ..ansible_util import ( from ..ansible_util import (
ansible_environment, ansible_environment,
check_pyyaml, check_pyyaml,
get_ansible_python_path,
) )
from ..target import ( from ..target import (
@ -47,10 +57,16 @@ from ..executor import (
Delegate, Delegate,
get_changes_filter, get_changes_filter,
install_command_requirements, install_command_requirements,
SUPPORTED_PYTHON_VERSIONS,
) )
class TestContext:
"""Contexts that unit tests run in based on the type of content."""
controller = 'controller'
modules = 'modules'
module_utils = 'module_utils'
def command_units(args): def command_units(args):
""" """
:type args: UnitsConfig :type args: UnitsConfig
@ -62,20 +78,33 @@ def command_units(args):
include = walk_internal_targets(walk_units_targets(), args.include, args.exclude, require) include = walk_internal_targets(walk_units_targets(), args.include, args.exclude, require)
paths = [target.path for target in include] paths = [target.path for target in include]
remote_paths = [path for path in paths
if is_subdir(path, data_context().content.unit_module_path) module_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_path)]
or is_subdir(path, data_context().content.unit_module_utils_path)] module_utils_paths = [path for path in paths if is_subdir(path, data_context().content.unit_module_utils_path)]
controller_paths = sorted(path for path in set(paths) - set(module_paths) - set(module_utils_paths))
remote_paths = module_paths or module_utils_paths
test_context_paths = {
TestContext.modules: module_paths,
TestContext.module_utils: module_utils_paths,
TestContext.controller: controller_paths,
}
# temporary definition of "remote" python versions until the full split controller/remote implementation is in place
controller_min_python_version = (3, 8)
remote_only_python_versions = tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) < controller_min_python_version)
if not paths: if not paths:
raise AllTargetsSkipped() raise AllTargetsSkipped()
if args.python and args.python in REMOTE_ONLY_PYTHON_VERSIONS and not remote_paths: if args.python and args.python in remote_only_python_versions and not remote_paths:
raise AllTargetsSkipped() raise AllTargetsSkipped()
if args.delegate: if args.delegate:
raise Delegate(require=changes, exclude=args.exclude) raise Delegate(require=changes, exclude=args.exclude)
version_commands = [] test_sets = []
available_versions = sorted(get_available_python_versions(list(SUPPORTED_PYTHON_VERSIONS)).keys()) available_versions = sorted(get_available_python_versions(list(SUPPORTED_PYTHON_VERSIONS)).keys())
@ -84,15 +113,41 @@ def command_units(args):
if args.python and version != args.python_version: if args.python and version != args.python_version:
continue continue
test_candidates = []
for test_context, paths in test_context_paths.items():
if test_context == TestContext.controller and version in remote_only_python_versions:
continue
if not paths:
continue
env = ansible_environment(args)
env.update(
PYTHONPATH=get_units_ansible_python_path(args, test_context),
ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION=version_to_str(controller_min_python_version),
)
test_candidates.append((test_context, version, paths, env))
if not test_candidates:
continue
if not args.python and version not in available_versions: if not args.python and version not in available_versions:
display.warning("Skipping unit tests on Python %s due to missing interpreter." % version) display.warning("Skipping unit tests on Python %s due to missing interpreter." % version)
continue continue
if args.requirements_mode != 'skip': if args.requirements_mode != 'skip':
install_command_requirements(args, version) install_command_requirements(args, version)
check_pyyaml(args, version)
env = ansible_environment(args) test_sets.extend(test_candidates)
if args.requirements_mode == 'only':
sys.exit()
for test_context, version, paths, env in test_sets:
cmd = [ cmd = [
'pytest', 'pytest',
'--boxed', '--boxed',
@ -102,7 +157,7 @@ def command_units(args):
'yes' if args.color else 'no', 'yes' if args.color else 'no',
'-p', 'no:cacheprovider', '-p', 'no:cacheprovider',
'-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'pytest.ini'), '-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'pytest.ini'),
'--junit-xml', os.path.join(ResultType.JUNIT.path, 'python%s-units.xml' % version), '--junit-xml', os.path.join(ResultType.JUNIT.path, 'python%s-%s-units.xml' % (version, test_context)),
] ]
if not data_context().content.collection: if not data_context().content.collection:
@ -130,30 +185,61 @@ def command_units(args):
if args.verbosity: if args.verbosity:
cmd.append('-' + ('v' * args.verbosity)) cmd.append('-' + ('v' * args.verbosity))
if version in REMOTE_ONLY_PYTHON_VERSIONS: cmd.extend(paths)
test_paths = remote_paths
else:
test_paths = paths
if not test_paths: display.info('Unit test %s with Python %s' % (test_context, version))
continue
cmd.extend(test_paths)
version_commands.append((version, cmd, env))
if args.requirements_mode == 'only':
sys.exit()
for version, command, env in version_commands:
check_pyyaml(args, version)
display.info('Unit test with Python %s' % version)
try: try:
with coverage_context(args): with coverage_context(args):
intercept_command(args, command, target_name='units', env=env, python_version=version) intercept_command(args, cmd, target_name=test_context, env=env, python_version=version)
except SubprocessError as ex: except SubprocessError as ex:
# pytest exits with status code 5 when all tests are skipped, which isn't an error for our use case # pytest exits with status code 5 when all tests are skipped, which isn't an error for our use case
if ex.status != 5: if ex.status != 5:
raise raise
def get_units_ansible_python_path(args, test_context): # type: (UnitsConfig, str) -> str
"""
Return a directory usable for PYTHONPATH, containing only the modules and module_utils portion of the ansible package.
The temporary directory created will be cached for the lifetime of the process and cleaned up at exit.
"""
if test_context == TestContext.controller:
return get_ansible_python_path(args)
try:
cache = get_units_ansible_python_path.cache
except AttributeError:
cache = get_units_ansible_python_path.cache = {}
python_path = cache.get(test_context)
if python_path:
return python_path
python_path = create_temp_dir(prefix='ansible-test-')
ansible_path = os.path.join(python_path, 'ansible')
ansible_test_path = os.path.join(python_path, 'ansible_test')
write_text_file(os.path.join(ansible_path, '__init__.py'), '', True)
os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'module_utils'), os.path.join(ansible_path, 'module_utils'))
if data_context().content.collection:
# built-in runtime configuration for the collection loader
make_dirs(os.path.join(ansible_path, 'config'))
os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'config', 'ansible_builtin_runtime.yml'), os.path.join(ansible_path, 'config', 'ansible_builtin_runtime.yml'))
# current collection loader required by all python versions supported by the controller
write_text_file(os.path.join(ansible_path, 'utils', '__init__.py'), '', True)
os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'collection_loader'), os.path.join(ansible_path, 'utils', 'collection_loader'))
# legacy collection loader required by all python versions not supported by the controller
write_text_file(os.path.join(ansible_test_path, '__init__.py'), '', True)
write_text_file(os.path.join(ansible_test_path, '_internal', '__init__.py'), '', True)
os.symlink(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'legacy_collection_loader'), os.path.join(ansible_test_path, '_internal', 'legacy_collection_loader'))
elif test_context == TestContext.modules:
# only non-collection ansible module tests should have access to ansible built-in modules
os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'modules'), os.path.join(ansible_path, 'modules'))
cache[test_context] = python_path
return python_path