Import sanity test for plugins (#72497)

This commit is contained in:
Felix Fontein 2021-02-12 23:09:50 +01:00 committed by GitHub
parent 4059b37ab1
commit 1f3a90270b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 231 additions and 98 deletions

View file

@ -0,0 +1,2 @@
minor_changes:
- "ansible-test - the ``import`` sanity test now also tries to import all non-module and non-module_utils Python files in ``lib/ansible/`` resp. ``plugins/`` (https://github.com/ansible/ansible/pull/72497)."

View file

@ -0,0 +1,4 @@
ansible-requirements
====================
``test/lib/ansible_test/_data/requirements/sanity.import-plugins.txt`` must be an identical copy of ``requirements.txt`` found in the project's root.

View file

@ -0,0 +1,13 @@
# Note: this requirements.txt file is used to specify what dependencies are
# needed to make the package run rather than for deployment of a tested set of
# packages. Thus, this should be the loosest set possible (only required
# packages, not optional ones, and with the widest range of versions that could
# be suitable)
jinja2
PyYAML
cryptography
packaging
# NOTE: resolvelib 0.x version bumps should be considered major/breaking
# NOTE: and we should update the upper cap with care, at least until 1.0
# NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69
resolvelib >= 0.5.3, < 0.6.0 # dependency resolver used by ansible-galaxy

View file

@ -27,6 +27,7 @@ def main():
external_python = os.environ.get('SANITY_EXTERNAL_PYTHON') or sys.executable
collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME')
collection_root = os.environ.get('ANSIBLE_COLLECTIONS_PATH')
import_type = os.environ.get('SANITY_IMPORTER_TYPE')
try:
# noinspection PyCompatibility
@ -103,6 +104,7 @@ def main():
# remove all modules under the ansible package
list(map(sys.modules.pop, [m for m in sys.modules if m.partition('.')[0] == ansible.__name__]))
if import_type == 'module':
# pre-load an empty ansible package to prevent unwanted code in __init__.py from loading
# this more accurately reflects the environment that AnsiballZ runs modules under
# it also avoids issues with imports in the ansible package that are not allowed
@ -123,10 +125,11 @@ def main():
class RestrictedModuleLoader:
"""Python module loader that restricts inappropriate imports."""
def __init__(self, path, name):
def __init__(self, path, name, restrict_to_module_paths):
self.path = path
self.name = name
self.loaded_modules = set()
self.restrict_to_module_paths = restrict_to_module_paths
def find_module(self, fullname, path=None):
"""Return self if the given fullname is restricted, otherwise return None.
@ -138,6 +141,9 @@ def main():
return None # ignore modules that are already being loaded
if is_name_in_namepace(fullname, ['ansible']):
if not self.restrict_to_module_paths:
return None # for non-modules, everything in the ansible namespace is allowed
if fullname in ('ansible.module_utils.basic', 'ansible.module_utils.common.removed'):
return self # intercept loading so we can modify the result
@ -153,6 +159,9 @@ def main():
if not collection_loader:
return self # restrict access to collections when we are not testing a collection
if not self.restrict_to_module_paths:
return None # for non-modules, everything in the ansible namespace is allowed
if is_name_in_namepace(fullname, ['ansible_collections...plugins.module_utils', self.name]):
return None # module_utils and module under test are always allowed
@ -196,24 +205,25 @@ def main():
self.loaded_modules.add(fullname)
return import_module(fullname)
def run():
def run(restrict_to_module_paths):
"""Main program function."""
base_dir = os.getcwd()
messages = set()
for path in sys.argv[1:] or sys.stdin.read().splitlines():
name = convert_relative_path_to_name(path)
test_python_module(path, name, base_dir, messages)
test_python_module(path, name, base_dir, messages, restrict_to_module_paths)
if messages:
sys.exit(10)
def test_python_module(path, name, base_dir, messages):
def test_python_module(path, name, base_dir, messages, restrict_to_module_paths):
"""Test the given python module by importing it.
:type path: str
:type name: str
:type base_dir: str
:type messages: set[str]
:type restrict_to_module_paths: bool
"""
if name in sys.modules:
return # cannot be tested because it has already been loaded
@ -230,13 +240,13 @@ def main():
try:
with monitor_sys_modules(path, messages):
with restrict_imports(path, name, messages):
with restrict_imports(path, name, messages, restrict_to_module_paths):
with capture_output(capture_normal):
import_module(name)
if run_main:
with monitor_sys_modules(path, messages):
with restrict_imports(path, name, messages):
with restrict_imports(path, name, messages, restrict_to_module_paths):
with capture_output(capture_main):
runpy.run_module(name, run_name='__main__', alter_sys=True)
except ImporterAnsibleModuleException:
@ -398,13 +408,14 @@ def main():
print(message)
@contextlib.contextmanager
def restrict_imports(path, name, messages):
def restrict_imports(path, name, messages, restrict_to_module_paths):
"""Restrict available imports.
:type path: str
:type name: str
:type messages: set[str]
:type restrict_to_module_paths: bool
"""
restricted_loader = RestrictedModuleLoader(path, name)
restricted_loader = RestrictedModuleLoader(path, name, restrict_to_module_paths)
# noinspection PyTypeChecker
sys.meta_path.insert(0, restricted_loader)
@ -413,6 +424,10 @@ def main():
try:
yield
finally:
if import_type == 'plugin':
from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
_AnsibleCollectionFinder._remove() # pylint: disable=protected-access
if sys.meta_path[0] != restricted_loader:
report_message(path, 0, 0, 'metapath', 'changes to sys.meta_path[0] are not permitted', messages)
@ -457,6 +472,26 @@ def main():
with warnings.catch_warnings():
warnings.simplefilter('error')
if sys.version_info[0] == 2:
warnings.filterwarnings(
"ignore",
"Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography,"
" and will be removed in a future release.")
warnings.filterwarnings(
"ignore",
"Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography,"
" and will be removed in the next release.")
if sys.version_info[:2] == (3, 5):
warnings.filterwarnings(
"ignore",
"Python 3.5 support will be dropped in the next release ofcryptography. Please upgrade your Python.")
warnings.filterwarnings(
"ignore",
"Python 3.5 support will be dropped in the next release of cryptography. Please upgrade your Python.")
warnings.filterwarnings(
"ignore",
"The _yaml extension module is now located at yaml._yaml and its location is subject to change. To use the "
"LibYAML-based parser and emitter, import from `yaml`: `from yaml import CLoader as Loader, CDumper as Dumper`.")
try:
yield
@ -464,7 +499,7 @@ def main():
sys.stdout = old_stdout
sys.stderr = old_stderr
run()
run(import_type == 'module')
if __name__ == '__main__':

View file

@ -9,7 +9,6 @@ import re
import time
import textwrap
import functools
import hashlib
import difflib
import filecmp
import random
@ -44,7 +43,6 @@ from .cloud import (
from .io import (
make_dirs,
open_text_file,
read_binary_file,
read_text_file,
write_text_file,
)
@ -70,6 +68,7 @@ from .util import (
SUPPORTED_PYTHON_VERSIONS,
str_to_version,
version_to_str,
get_hash,
)
from .util_common import (
@ -2026,7 +2025,7 @@ class EnvironmentDescription:
pip_paths = dict((v, find_executable('pip%s' % v, required=False)) for v in sorted(versions))
program_versions = dict((v, self.get_version([python_paths[v], version_check], warnings)) for v in sorted(python_paths) if python_paths[v])
pip_interpreters = dict((v, self.get_shebang(pip_paths[v])) for v in sorted(pip_paths) if pip_paths[v])
known_hosts_hash = self.get_hash(os.path.expanduser('~/.ssh/known_hosts'))
known_hosts_hash = get_hash(os.path.expanduser('~/.ssh/known_hosts'))
for version in sorted(versions):
self.check_python_pip_association(version, python_paths, pip_paths, pip_interpreters, warnings)
@ -2166,21 +2165,6 @@ class EnvironmentDescription:
with open_text_file(path) as script_fd:
return script_fd.readline().strip()
@staticmethod
def get_hash(path):
"""
:type path: str
:rtype: str | None
"""
if not os.path.exists(path):
return None
file_hash = hashlib.sha256()
file_hash.update(read_binary_file(path))
return file_hash.hexdigest()
class NoChangesDetected(ApplicationWarning):
"""Exception when change detection was performed, but no changes were found."""

View file

@ -20,6 +20,7 @@ from ..target import (
)
from ..util import (
ANSIBLE_TEST_DATA_ROOT,
SubprocessError,
remove_tree,
display,
@ -27,6 +28,8 @@ from ..util import (
is_subdir,
generate_pip_command,
find_python,
get_hash,
REMOTE_ONLY_PYTHON_VERSIONS,
)
from ..util_common import (
@ -41,6 +44,7 @@ from ..ansible_util import (
from ..executor import (
generate_pip_install,
install_cryptography,
)
from ..config import (
@ -60,12 +64,21 @@ from ..data import (
)
def _get_module_test(module_restrictions): # type: (bool) -> t.Callable[[str], bool]
"""Create a predicate which tests whether a path can be used by modules or not."""
module_path = data_context().content.module_path
module_utils_path = data_context().content.module_utils_path
if module_restrictions:
return lambda path: is_subdir(path, module_path) or is_subdir(path, module_utils_path)
return lambda path: not (is_subdir(path, module_path) or is_subdir(path, module_utils_path))
class ImportTest(SanityMultipleVersion):
"""Sanity test for proper import exception handling."""
def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget]
"""Return the given list of test targets, filtered to include only those relevant for the test."""
return [target for target in targets if os.path.splitext(target.path)[1] == '.py' and
(is_subdir(target.path, data_context().content.module_path) or is_subdir(target.path, data_context().content.module_utils_path))]
any(is_subdir(target.path, path) for path in data_context().content.plugin_paths.values())]
def test(self, args, targets, python_version):
"""
@ -92,8 +105,27 @@ class ImportTest(SanityMultipleVersion):
temp_root = os.path.join(ResultType.TMP.path, 'sanity', 'import')
messages = []
for import_type, test, add_ansible_requirements in (
('module', _get_module_test(True), False),
('plugin', _get_module_test(False), True),
):
if import_type == 'plugin' and python_version in REMOTE_ONLY_PYTHON_VERSIONS:
continue
data = '\n'.join([path for path in paths if test(path)])
if not data:
continue
requirements_file = None
# create a clean virtual environment to minimize the available imports beyond the python standard library
virtual_environment_path = os.path.join(temp_root, 'minimal-py%s' % python_version.replace('.', ''))
virtual_environment_dirname = 'minimal-py%s' % python_version.replace('.', '')
if add_ansible_requirements:
requirements_file = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'sanity.import-plugins.txt')
virtual_environment_dirname += '-requirements-%s' % get_hash(requirements_file)
virtual_environment_path = os.path.join(temp_root, virtual_environment_dirname)
virtual_environment_bin = os.path.join(virtual_environment_path, 'bin')
remove_tree(virtual_environment_path)
@ -114,6 +146,7 @@ class ImportTest(SanityMultipleVersion):
env.update(
SANITY_TEMP_PATH=ResultType.TMP.path,
SANITY_IMPORTER_TYPE=import_type,
)
if data_context().content.collection:
@ -125,6 +158,11 @@ class ImportTest(SanityMultipleVersion):
virtualenv_python = os.path.join(virtual_environment_bin, 'python')
virtualenv_pip = generate_pip_command(virtualenv_python)
# make sure requirements are installed if needed
if requirements_file:
install_cryptography(args, virtualenv_python, python_version, virtualenv_pip)
run_command(args, generate_pip_install(virtualenv_pip, 'sanity', context='import-plugins'), env=env, capture=capture_pip)
# make sure coverage is available in the virtual environment if needed
if args.coverage:
run_command(args, generate_pip_install(virtualenv_pip, '', packages=['setuptools']), env=env, capture=capture_pip)
@ -144,14 +182,10 @@ class ImportTest(SanityMultipleVersion):
run_command(args, virtualenv_pip + ['uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env, capture=capture_pip)
run_command(args, virtualenv_pip + ['uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env, capture=capture_pip)
display.info(import_type + ': ' + data, verbosity=4)
cmd = ['importer.py']
data = '\n'.join(paths)
display.info(data, verbosity=4)
results = []
try:
with coverage_context(args):
stdout, stderr = intercept_command(args, cmd, self.name, env, capture=True, data=data, python_version=python_version,
@ -165,18 +199,18 @@ class ImportTest(SanityMultipleVersion):
pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<message>.*)$'
results = parse_to_list_of_dict(pattern, ex.stdout)
parsed = parse_to_list_of_dict(pattern, ex.stdout)
relative_temp_root = os.path.relpath(temp_root, data_context().content.root) + os.path.sep
results = [SanityMessage(
messages += [SanityMessage(
message=r['message'],
path=os.path.relpath(r['path'], relative_temp_root) if r['path'].startswith(relative_temp_root) else r['path'],
line=int(r['line']),
column=int(r['column']),
) for r in results]
) for r in parsed]
results = settings.process_errors(results, paths)
results = settings.process_errors(messages, paths)
if results:
return SanityFailure(self.name, messages=results, python_version=python_version)

View file

@ -5,6 +5,7 @@ __metaclass__ = type
import contextlib
import errno
import fcntl
import hashlib
import inspect
import os
import pkgutil
@ -53,6 +54,7 @@ from .encoding import (
from .io import (
open_binary_file,
read_binary_file,
read_text_file,
)
@ -857,4 +859,19 @@ def open_zipfile(path, mode='r'):
zib_obj.close()
def get_hash(path):
"""
:type path: str
:rtype: str | None
"""
if not os.path.exists(path):
return None
file_hash = hashlib.sha256()
file_hash.update(read_binary_file(path))
return file_hash.hexdigest()
display = Display() # pylint: disable=locally-disabled, invalid-name

View file

@ -0,0 +1,7 @@
{
"prefixes": [
"requirements.txt",
"test/lib/ansible_test/_data/requirements/sanity.import-plugins.txt"
],
"output": "path-line-column-message"
}

View file

@ -0,0 +1,31 @@
#!/usr/bin/env python
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import re
import sys
def read_file(path):
try:
with open(path, 'r') as f:
return f.read()
except Exception as ex: # pylint: disable=broad-except
print('%s:%d:%d: unable to read required file %s' % (path, 0, 0, re.sub(r'\s+', ' ', str(ex))))
return None
def main():
ORIGINAL_FILE = 'requirements.txt'
VENDORED_COPY = 'test/lib/ansible_test/_data/requirements/sanity.import-plugins.txt'
original_requirements = read_file(ORIGINAL_FILE)
vendored_requirements = read_file(VENDORED_COPY)
if original_requirements is not None and vendored_requirements is not None:
if original_requirements != vendored_requirements:
print('%s:%d:%d: must be identical to %s' % (VENDORED_COPY, 0, 0, ORIGINAL_FILE))
if __name__ == '__main__':
main()

View file

@ -12,6 +12,12 @@ def main():
requirements = {}
for path in sys.argv[1:] or sys.stdin.read().splitlines():
if path == 'test/lib/ansible_test/_data/requirements/sanity.import-plugins.txt':
# This file is an exact copy of requirements.txt that is used in the import
# sanity test. There is a code-smell test which ensures that the two files
# are identical, and it is only used inside an empty venv, so we can ignore
# it here.
continue
with open(path, 'r') as path_fd:
requirements[path] = parse_requirements(path_fd.read().splitlines())