Split out and install sanity test requirements. (#69971)

* Split out sanity test requirements.

* Run each --venv test separately.

This provides verification that the requirements for each test are properly specified.

* Use a separate requirements file per sanity test.

* Skip setuptools/cryptography setup for sanity.

* Eliminate pyyaml missing warning.

* Eliminate more pip noise.

* Fix conflicting generate_pip_install commands.

* Add changelog fragment.

changelogs/fragments/ansible-test-sanity-separate-requirements.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now installs sanity test requirements specific to each test instead of installing requirements for all sanity tests

test/integration/targets/ansible-test/collection-tests/venv.sh

@@ -6,12 +6,34 @@ cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
 cd "${WORK_DIR}/ansible_collections/ns/col"
 
 # common args for all tests
+# each test will be run in a separate venv to verify that requirements have been properly specified
 common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
 
-# prime the venv to work around issue with PyYAML detection in ansible-test
+# sanity tests
-ansible-test sanity "${common[@]}" --test ignores
 
-# tests
+tests=()
-ansible-test sanity "${common[@]}"
+
+set +x
+
+while IFS='' read -r line; do
+  tests+=("$line");
+done < <(
+  ansible-test sanity --list-tests
+)
+
+set -x
+
+for test in "${tests[@]}"; do
+  rm -rf "tests/output"
+  ansible-test sanity "${common[@]}" --test "${test}"
+done
+
+# unit tests
+
+rm -rf "tests/output"
 ansible-test units "${common[@]}"
+
+# integration tests
+
+rm -rf "tests/output"
 ansible-test integration "${common[@]}"

test/lib/ansible_test/_data/quiet_pip.py

@@ -10,11 +10,14 @@ import warnings
 BUILTIN_FILTERER_FILTER = logging.Filterer.filter
 
 LOGGING_MESSAGE_FILTER = re.compile("^("
+                                    "WARNING: Running pip install with root privileges is generally not a good idea. .*|"  # custom Fedora patch [1]
                                     "DEPRECATION: Python 2.7 will reach the end of its life .*|"  # pip 19.2.3
                                     "Ignoring .*: markers .* don't match your environment|"
                                     "Requirement already satisfied.*"
                                     ")$")
 
+# [1] https://src.fedoraproject.org/rpms/python-pip/blob/master/f/emit-a-warning-when-running-with-root-privileges.patch
+
 WARNING_MESSAGE_FILTERS = (
     # DEPRECATION: Python 2.6 is no longer supported by the Python core team, please upgrade your Python.
     # A future version of pip will drop support for Python 2.6

test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt

@@ -0,0 +1,2 @@
+jinja2  # ansible-base requirement
+pyyaml  # ansible-base requirement

test/lib/ansible_test/_data/requirements/sanity.import.txt

@@ -0,0 +1,2 @@
+pyyaml  # required for the collection loader to parse yaml for plugin routing
+virtualenv ; python_version <= '2.7'  # virtualenv required on Python 2.x, but on Python 3.x we can use the built-in venv instead

test/lib/ansible_test/_data/requirements/sanity.pep8.txt

@@ -0,0 +1 @@
+pycodestyle

test/lib/ansible_test/_data/requirements/sanity.pylint.txt

@@ -0,0 +1,2 @@
+pylint ; python_version < '3.9'  # installation fails on python 3.9.0b1
+pyyaml  # needed for collection_detail.py

test/lib/ansible_test/_data/requirements/sanity.rstcheck.txt

@@ -0,0 +1 @@
+rstcheck

test/lib/ansible_test/_data/requirements/sanity.txt

@@ -1,9 +0,0 @@
-cryptography
-jinja2
-pycodestyle ; python_version >= '3.5' # only used on python 3.5+
-pylint ; python_version >= '3.5' and python_version < '3.9' # only used on python 3.5+ and does not yet support python 3.9
-pyyaml
-rstcheck ; python_version >= '3.5' # only used on python 3.5+
-virtualenv
-voluptuous ; python_version >= '3.5' # only used on python 3.5+
-yamllint ; python_version >= '3.5' # only used on python 3.5+

test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt

@@ -0,0 +1,3 @@
+jinja2  # ansible-base requirement
+pyyaml  # needed for collection_detail.py
+voluptuous

test/lib/ansible_test/_data/requirements/sanity.yamllint.txt

@@ -0,0 +1 @@
+yamllint

test/lib/ansible_test/_internal/ansible_util.py

@@ -212,7 +212,10 @@ def check_pyyaml(args, version):
     cloader = result['cloader']
 
     if not yaml:
-        display.warning('PyYAML is not installed for interpreter: %s' % python)
+        # do not warn about missing pyyaml
+        # if it is required by tests they will fail with an error message that should be descriptive enough
+        # warning here assumes all tests require pyyaml, which is not the case for many sanity tests
+        pass
     elif not cloader:
         display.warning('PyYAML will be slow due to installation without libyaml support for interpreter: %s' % python)
 

test/lib/ansible_test/_internal/cli.py

@@ -205,7 +205,7 @@ def parse_args():
         # install argparse without using constraints since pip may be too old to support them
         # not using the ansible-test requirements file since this install is for sys.executable rather than the delegated python (which may be different)
         # argparse has no special requirements, so upgrading pip is not required here
-        raw_command(generate_pip_install(generate_pip_command(sys.executable), 'argparse', packages=['argparse'], use_constraints=False))
+        raw_command(generate_pip_install(generate_pip_command(sys.executable), '', packages=['argparse'], use_constraints=False))
         import argparse
 
     try:

test/lib/ansible_test/_internal/executor.py

@@ -216,10 +216,11 @@ def get_cryptography_requirement(args, python_version):  # type: (EnvironmentCon
     return cryptography
 
 
-def install_command_requirements(args, python_version=None):
+def install_command_requirements(args, python_version=None, context=None):
     """
     :type args: EnvironmentConfig
     :type python_version: str | None
+    :type context: str | None
     """
     if not args.explain:
         make_dirs(ResultType.COVERAGE.path)
@@ -254,19 +255,20 @@ def install_command_requirements(args, python_version=None):
     # virtualenvs created by older distributions may include very old pip versions, such as those created in the centos6 test container (pip 6.0.8)
     run_command(args, generate_pip_install(pip, 'ansible-test', use_constraints=False))
 
-    # make sure setuptools is available before trying to install cryptography
+    if args.command != 'sanity':
-    # the installed version of setuptools affects the version of cryptography to install
+        # make sure setuptools is available before trying to install cryptography
-    run_command(args, generate_pip_install(pip, 'setuptools', packages=['setuptools']))
+        # the installed version of setuptools affects the version of cryptography to install
+        run_command(args, generate_pip_install(pip, '', packages=['setuptools']))
 
-    # install the latest cryptography version that the current requirements can support
+        # install the latest cryptography version that the current requirements can support
-    # use a custom constraints file to avoid the normal constraints file overriding the chosen version of cryptography
+        # use a custom constraints file to avoid the normal constraints file overriding the chosen version of cryptography
-    # if not installed here later install commands may try to install an unsupported version due to the presence of older setuptools
+        # if not installed here later install commands may try to install an unsupported version due to the presence of older setuptools
-    # this is done instead of upgrading setuptools to allow tests to function with older distribution provided versions of setuptools
+        # this is done instead of upgrading setuptools to allow tests to function with older distribution provided versions of setuptools
-    run_command(args, generate_pip_install(pip, 'cryptography',
+        run_command(args, generate_pip_install(pip, '',
-                                           packages=[get_cryptography_requirement(args, python_version)],
+                                               packages=[get_cryptography_requirement(args, python_version)],
-                                           constraints=os.path.join(ANSIBLE_TEST_DATA_ROOT, 'cryptography-constraints.txt')))
+                                               constraints=os.path.join(ANSIBLE_TEST_DATA_ROOT, 'cryptography-constraints.txt')))
 
-    commands = [generate_pip_install(pip, args.command, packages=packages)]
+    commands = [generate_pip_install(pip, args.command, packages=packages, context=context)]
 
     if isinstance(args, IntegrationConfig):
         for cloud_platform in get_cloud_platforms(args):
@@ -377,26 +379,29 @@ License: GPLv3+
     write_text_file(pkg_info_path, pkg_info.lstrip(), create_directories=True)
 
 
-def generate_pip_install(pip, command, packages=None, constraints=None, use_constraints=True):
+def generate_pip_install(pip, command, packages=None, constraints=None, use_constraints=True, context=None):
     """
     :type pip: list[str]
     :type command: str
     :type packages: list[str] | None
     :type constraints: str | None
     :type use_constraints: bool
+    :type context: str | None
     :rtype: list[str] | None
     """
     constraints = constraints or os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt')
-    requirements = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', '%s.txt' % command)
+    requirements = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', '%s.txt' % ('%s.%s' % (command, context) if context else command))
 
     options = []
 
     if os.path.exists(requirements) and os.path.getsize(requirements):
         options += ['-r', requirements]
 
-    if data_context().content.is_ansible:
+    if command == 'sanity' and data_context().content.is_ansible:
-        if command == 'sanity':
+        requirements = os.path.join(data_context().content.sanity_path, 'code-smell', '%s.requirements.txt' % context)
-            options += ['-r', os.path.join(data_context().content.root, 'test', 'sanity', 'requirements.txt')]
+
+        if os.path.exists(requirements) and os.path.getsize(requirements):
+            options += ['-r', requirements]
 
     if command == 'units':
         requirements = os.path.join(data_context().content.unit_path, 'requirements.txt')

test/lib/ansible_test/_internal/sanity/__init__.py

@@ -109,8 +109,6 @@ def command_sanity(args):
     total = 0
     failed = []
 
-    requirements_installed = set()  # type: t.Set[str]
-
     for test in tests:
         if args.list_tests:
             display.info(test.name)
@@ -148,8 +146,6 @@ def command_sanity(args):
                 display.warning("Skipping sanity test '%s' on Python %s due to missing interpreter." % (test.name, version))
                 result = SanitySkipped(test.name, skip_version)
             else:
-                check_pyyaml(args, version)
-
                 if test.supported_python_versions:
                     display.info("Running sanity test '%s' with Python %s" % (test.name, version))
                 else:
@@ -183,9 +179,8 @@ def command_sanity(args):
                 sanity_targets = SanityTargets(tuple(all_targets), tuple(usable_targets))
 
                 if usable_targets or test.no_targets:
-                    if version not in requirements_installed:
+                    install_command_requirements(args, version, context=test.name)
-                        requirements_installed.add(version)
+                    check_pyyaml(args, version)
-                        install_command_requirements(args, version)
 
                     if isinstance(test, SanityCodeSmellTest):
                         result = test.test(args, sanity_targets, version)

test/lib/ansible_test/_internal/sanity/import.py

@@ -82,7 +82,7 @@ class ImportTest(SanityMultipleVersion):
             # hack to make sure that virtualenv is available under Python 2.x
             # on Python 3.x we can use the built-in venv
             pip = generate_pip_command(python)
-            run_command(args, generate_pip_install(pip, 'sanity.import', packages=['virtualenv']), capture=capture_pip)
+            run_command(args, generate_pip_install(pip, '', packages=['virtualenv']), capture=capture_pip)
 
         settings = self.load_processor(args, python_version)
 
@@ -127,8 +127,8 @@ class ImportTest(SanityMultipleVersion):
 
         # make sure coverage is available in the virtual environment if needed
         if args.coverage:
-            run_command(args, generate_pip_install(virtualenv_pip, 'sanity.import', packages=['setuptools']), env=env, capture=capture_pip)
+            run_command(args, generate_pip_install(virtualenv_pip, '', packages=['setuptools']), env=env, capture=capture_pip)
-            run_command(args, generate_pip_install(virtualenv_pip, 'sanity.import', packages=['coverage']), env=env, capture=capture_pip)
+            run_command(args, generate_pip_install(virtualenv_pip, '', packages=['coverage']), env=env, capture=capture_pip)
 
         try:
             # In some environments pkg_resources is installed as a separate pip package which needs to be removed.

test/sanity/code-smell/botmeta.requirements.txt

@@ -0,0 +1,2 @@
+pyyaml
+voluptuous

test/sanity/code-smell/changelog.requirements.txt

@@ -0,0 +1,5 @@
+docutils
+jinja2  # ansible-base requirement
+packaging
+pyyaml
+rstcheck

test/sanity/code-smell/deprecated-config.requirements.txt

@@ -0,0 +1,2 @@
+jinja2  # ansible-base requirement
+pyyaml

test/sanity/code-smell/docs-build.requirements.txt

@@ -0,0 +1,5 @@
+jinja2
+pyyaml
+sphinx
+sphinx-notfound-page
+straight.plugin

test/sanity/code-smell/package-data.requirements.txt

@@ -0,0 +1,7 @@
+docutils
+jinja2
+packaging
+pyyaml  # ansible-base requirement
+rstcheck
+setuptools > 39.2
+straight.plugin

test/sanity/code-smell/release-names.requirements.txt

@@ -0,0 +1 @@
+pyyaml

test/sanity/code-smell/update-bundled.requirements.txt

@@ -0,0 +1 @@
+packaging

test/sanity/requirements.txt

@@ -1,4 +0,0 @@
-packaging  # needed for update-bundled and changelog
-sphinx ; python_version >= '3.5' # docs build requires python 3+
-sphinx-notfound-page ; python_version >= '3.5' # docs build requires python 3+
-straight.plugin ; python_version >= '3.5' # needed for hacking/build-ansible.py which will host changelog generation and requires python 3+