Fix ansible-test constraints issues. (#73578)

* Silence Python 3.5 EOL notice in pip.
* Skip import test on compat/selinux.py utils.
* Improve Python version skip warning.
* Use Python 3.6 as minimum Python for sanity tests.
* Improve min Python handling for code-smell tests.
* Overhaul test-constraints sanity test.
* Merge sanity test constraints with requirements.
* Remove legacy content specific constraints.
* Add changelog fragment.

changelogs/fragments/ansible-test-updates.yml

@@ -0,0 +1,11 @@
+minor_changes:
+  - ansible-test - Most sanity tests are now skipped on Python 3.5 and earlier with a warning.
+                   Previously this was done for Python 2.7 and earlier.
+  - ansible-test - Removed ``pip`` constraints related to integration tests that have been moved to collections.
+                   This should reduce conflicts with ``pip`` requirements and constraints when testing collections.
+  - ansible-test - Most sanity test specific ``pip`` constraints are now used only when running sanity tests.
+                   This should reduce conflicts with ``pip`` requirements and constraints when testing collections.
+  - ansible-test - More sanity test requirements have been pinned to specific versions to provide consistent test results.
+  - ansible-test - Improved handling of minimum Python version requirements for sanity tests.
+                   Supported versions are now included in warning messages displayed when tests are skipped.
+  - ansible-test - Silence ``pip`` warnings about Python 3.5 being EOL when installing requirements.

test/lib/ansible_test/_data/quiet_pip.py

@@ -40,6 +40,10 @@ WARNING_MESSAGE_FILTERS = (
     # pip 21.0 will drop support for Python 2.7 in January 2021.
     # More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
     'DEPRECATION: Python 2.7 reached the end of its life ',
+
+    # DEPRECATION: Python 3.5 reached the end of its life on September 13th, 2020. Please upgrade your Python as Python 3.5 is no longer maintained.
+    # pip 21.0 will drop support for Python 3.5 in January 2021. pip 21.0 will remove support for this functionality.
+    'DEPRECATION: Python 3.5 reached the end of its life ',
 )
 
 

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -13,7 +13,6 @@ sphinx <= 2.1.2 ; python_version >= '2.7' # docs team hasn't  tested beyond 2.1.
 rstcheck >=3.3.1  # required for sphinx version >= 1.8
 pygments >= 2.4.0 # Pygments 2.4.0 includes bugfixes for YAML and YAML+Jinja lexers
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
-yamllint != 1.8.0, < 1.14.0 ; python_version < '2.7' # yamllint 1.8.0 and 1.14.0+ require python 2.7+
 pycrypto >= 2.6 # Need features found in 2.6 and greater
 ncclient >= 0.5.2 # Need features added in 0.5.2 and greater
 idna < 2.6, >= 2.5 # linode requires idna < 2.9, >= 2.5, requests requires idna < 2.6, but cryptography will cause the latest version to be installed instead
@@ -26,13 +25,11 @@ ntlm-auth >= 1.3.0 # message encryption support using cryptography
 requests < 2.20.0 ; python_version < '2.7' # requests 2.20.0 drops support for python 2.6
 requests-ntlm >= 1.1.0 # message encryption support
 requests-credssp >= 0.1.0 # message encryption support
-voluptuous >= 0.11.0 # Schema recursion via Self
 openshift >= 0.6.2, < 0.9.0 # merge_type support
 virtualenv < 16.0.0 ; python_version < '2.7' # virtualenv 16.0.0 and later require python 2.7 or later
 pathspec < 0.6.0 ; python_version < '2.7' # pathspec 0.6.0 and later require python 2.7 or later
 pyopenssl < 18.0.0 ; python_version < '2.7' # pyOpenSSL 18.0.0 and later require python 2.7 or later
 pyparsing < 3.0.0 ; python_version < '3.5' # pyparsing 3 and later require python 3.5 or later
-pyfmg == 0.6.1 # newer versions do not pass current unit tests
 pyyaml < 5.1 ; python_version < '2.7' # pyyaml 5.1 and later require python 2.7 or later
 pycparser < 2.19 ; python_version < '2.7' # pycparser 2.19 and later require python 2.7 or later
 mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...)
@@ -47,23 +44,3 @@ botocore >= 1.10.0 ; python_version >= '2.7' # adds support for the following AW
 setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require python 2.7 or later
 setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later
 gssapi < 1.6.0 ; python_version <= '2.7' # gssapi 1.6.0 and later require python 3 or later
-
-# freeze antsibull-changelog for consistent test results
-antsibull-changelog == 0.9.0
-
-# Make sure we have a new enough antsibull for the CLI args we use
-antsibull >= 0.21.0
-
-# freeze pylint and its requirements for consistent test results
-# NOTE: six is not frozen since it is a requirement for more than just pylint
-astroid == 2.4.2
-isort == 5.7.0
-lazy-object-proxy == 1.4.3
-mccabe == 0.6.1
-pylint == 2.6.0
-toml == 0.10.2
-typed-ast == 1.4.2
-wrapt == 1.12.1
-
-# freeze pycodestyle for consistent test results
-pycodestyle == 2.6.0

test/lib/ansible_test/_data/requirements/sanity.changelog.txt

@@ -1,2 +1 @@
-# changelog build requires python 3.6+
+antsibull-changelog == 0.9.0
-antsibull-changelog ; python_version >= '3.6'

test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt

@@ -1 +1 @@
-pyyaml
+pyyaml  # not frozen due to usage outside sanity tests

test/lib/ansible_test/_data/requirements/sanity.pep8.txt

@@ -1 +1 @@
-pycodestyle
+pycodestyle == 2.6.0

test/lib/ansible_test/_data/requirements/sanity.pylint.txt

@@ -1,3 +1,12 @@
-pylint
+pylint == 2.6.0
 pyyaml  # needed for collection_detail.py
-mccabe  # pylint complexity testing
+
+# dependencies
+astroid == 2.4.2
+isort == 5.7.0
+lazy-object-proxy == 1.4.3
+mccabe == 0.6.1
+six  # not frozen due to usage outside sanity tests
+toml == 0.10.2
+typed-ast == 1.4.2
+wrapt == 1.12.1

test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt

@@ -1,2 +1,2 @@
-pyyaml
+pyyaml  # not frozen due to usage outside sanity tests
-voluptuous
+voluptuous == 0.12.1

test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt

@@ -1,3 +1,3 @@
 jinja2  # ansible-core requirement
 pyyaml  # needed for collection_detail.py
-voluptuous
+voluptuous == 0.12.1

test/lib/ansible_test/_data/requirements/sanity.yamllint.txt

@@ -1 +1,5 @@
-yamllint
+yamllint == 1.26.0
+
+# dependencies
+pathspec  # not frozen since it should not impact test results
+pyyaml  # not frozen due to usage outside sanity tests

test/lib/ansible_test/_data/sanity/code-smell/changelog.json

@@ -1,6 +1,5 @@
 {
     "intercept": true,
-    "minimum_python_version": "3.6",
     "prefixes": [
         "changelogs/config.yaml",
         "changelogs/fragments/"

test/lib/ansible_test/_internal/sanity/__init__.py

@@ -141,7 +141,8 @@ def command_sanity(args):
             options = ''
 
             if test.supported_python_versions and version not in test.supported_python_versions:
-                display.warning("Skipping sanity test '%s' on unsupported Python %s." % (test.name, version))
+                display.warning("Skipping sanity test '%s' on Python %s. Supported Python versions: %s" % (
+                    test.name, version, ', '.join(test.supported_python_versions)))
                 result = SanitySkipped(test.name, skip_version)
             elif not args.python and version not in available_versions:
                 display.warning("Skipping sanity test '%s' on Python %s due to missing interpreter." % (test.name, version))
@@ -658,7 +659,7 @@ class SanityTest(ABC):
     @property
     def supported_python_versions(self):  # type: () -> t.Optional[t.Tuple[str, ...]]
         """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
-        return tuple(python_version for python_version in SUPPORTED_PYTHON_VERSIONS if python_version.startswith('3.'))
+        return tuple(python_version for python_version in SUPPORTED_PYTHON_VERSIONS if str_to_version(python_version) >= (3, 6))
 
     def filter_targets(self, targets):  # type: (t.List[TestTarget]) -> t.List[TestTarget]  # pylint: disable=unused-argument
         """Return the given list of test targets, filtered to include only those relevant for the test."""
@@ -751,6 +752,16 @@ class SanityCodeSmellTest(SanityTest):
         """True if the test targets should include symlinks."""
         return self.__include_symlinks
 
+    @property
+    def supported_python_versions(self):  # type: () -> t.Optional[t.Tuple[str, ...]]
+        """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
+        versions = super(SanityCodeSmellTest, self).supported_python_versions
+
+        if self.minimum_python_version:
+            versions = tuple(version for version in versions if str_to_version(version) >= str_to_version(self.minimum_python_version))
+
+        return versions
+
     def filter_targets(self, targets):  # type: (t.List[TestTarget]) -> t.List[TestTarget]
         """Return the given list of test targets, filtered to include only those relevant for the test."""
         if self.no_targets:
@@ -785,12 +796,6 @@ class SanityCodeSmellTest(SanityTest):
         :type python_version: str
         :rtype: TestResult
         """
-        if self.minimum_python_version:
-            if str_to_version(python_version) < str_to_version(self.minimum_python_version):
-                display.warning("Skipping sanity test '%s' on unsupported Python %s; requires Python %s or newer." % (
-                    self.name, python_version, self.minimum_python_version))
-                return SanitySkipped(self.name, 'Test requires Python %s or newer' % (self.minimum_python_version, ))
-
         cmd = [find_python(python_version), self.path]
 
         env = ansible_environment(args, color=False)

test/sanity/code-smell/botmeta.requirements.txt

@@ -1,2 +1,2 @@
 pyyaml
-voluptuous
+voluptuous == 0.12.1

test/sanity/code-smell/docs-build.requirements.txt

@@ -4,4 +4,4 @@ resolvelib
 sphinx
 sphinx-notfound-page
 straight.plugin
-antsibull
+antsibull == 0.26.0

test/sanity/code-smell/package-data.requirements.txt

@@ -4,8 +4,6 @@ packaging
 pyyaml  # ansible-core requirement
 resolvelib  # ansible-core requirement
 rstcheck
-setuptools > 39.2
+setuptools
 straight.plugin
-
+antsibull-changelog == 0.9.0
-# changelog build requires python 3.6+
-antsibull-changelog ; python_version >= '3.6'

test/sanity/code-smell/test-constraints.json

@@ -1,6 +1,8 @@
 {
+    "all_targets": true,
     "prefixes": [
-        "test/lib/ansible_test/_data/requirements/"
+        "test/lib/ansible_test/_data/requirements/",
+        "test/sanity/code-smell/"
     ],
     "extensions": [
         ".txt"

test/sanity/code-smell/test-constraints.py

@@ -7,14 +7,88 @@ import sys
 
 
 def main():
+    constraints_path = 'test/lib/ansible_test/_data/requirements/constraints.txt'
+
+    requirements = {}
+
     for path in sys.argv[1:] or sys.stdin.read().splitlines():
         with open(path, 'r') as path_fd:
-            for line, text in enumerate(path_fd.readlines()):
+            requirements[path] = parse_requirements(path_fd.read().splitlines())
-                match = re.search(r'^[^;#]*?([<>=])(?!.*sanity_ok.*)', text)
 
-                if match:
+    frozen_sanity = {}
-                    print('%s:%d:%d: put constraints in `test/lib/ansible_test/_data/requirements/constraints.txt`' % (
+    non_sanity_requirements = set()
-                        path, line + 1, match.start(1) + 1))
+
+    for path, requirements in requirements.items():
+        for lineno, line, requirement in requirements:
+            if not requirement:
+                print('%s:%d:%d: cannot parse requirement: %s' % (path, lineno, 1, line))
+                continue
+
+            name = requirement.group('name').lower()
+            raw_constraints = requirement.group('constraints')
+            raw_markers = requirement.group('markers')
+            constraints = raw_constraints.strip()
+            markers = raw_markers.strip()
+            comment = requirement.group('comment')
+
+            is_sanity = path.startswith('test/lib/ansible_test/_data/requirements/sanity.') or path.startswith('test/sanity/code-smell/')
+            is_pinned = re.search('^ *== *[0-9.]+$', constraints)
+            is_constraints = path == constraints_path
+
+            if is_sanity:
+                sanity = frozen_sanity.setdefault(name, [])
+                sanity.append((path, lineno, line, requirement))
+            elif not is_constraints:
+                non_sanity_requirements.add(name)
+
+            if constraints and not is_constraints:
+                allow_constraints = 'sanity_ok' in comment
+
+                if is_sanity and is_pinned and not markers:
+                    allow_constraints = True  # sanity tests can use frozen requirements without markers
+
+                if not allow_constraints:
+                    if is_sanity:
+                        # sanity test requirements which need constraints should be frozen to maintain consistent test results
+                        # use of anything other than frozen constraints will make evaluation of conflicts extremely difficult
+                        print('%s:%d:%d: sanity test constraint (%s%s) must be frozen (use `==`)' % (path, lineno, 1, name, raw_constraints))
+                    else:
+                        # keeping constraints for tests other than sanity tests in one file helps avoid conflicts
+                        print('%s:%d:%d: put the constraint (%s%s) in `%s`' % (path, lineno, 1, name, raw_constraints, constraints_path))
+
+    for name, requirements in frozen_sanity.items():
+        for req in requirements:
+            if name in non_sanity_requirements and req[3].group('constraints').strip():
+                print('%s:%d:%d: sanity constraint (%s) for package `%s` is not allowed because `%s` is used outside sanity tests' % (
+                    req[0], req[1], req[3].start('constraints') + 1, req[3].group('constraints'), name, name))
+
+        if len(set(req[3].group('constraints').strip() for req in requirements)) != 1:
+            for req in requirements:
+                print('%s:%d:%d: sanity constraint (%s) does not match others for package `%s`' % (
+                    req[0], req[1], req[3].start('constraints') + 1, req[3].group('constraints'), name))
+
+
+def parse_requirements(lines):
+    # see https://www.python.org/dev/peps/pep-0508/#names
+    pattern = re.compile(r'^(?P<name>[A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])(?P<extras> *\[[^]]*])?(?P<constraints>[^;#]*)(?P<markers>[^#]*)(?P<comment>.*)$',
+                         re.IGNORECASE)
+
+    matches = [(lineno, line, pattern.search(line)) for lineno, line in enumerate(lines, start=1)]
+    requirements = []
+
+    for lineno, line, match in matches:
+        if not line.strip():
+            continue
+
+        if line.strip().startswith('#'):
+            continue
+
+        if line.startswith('git+https://'):
+            continue  # hack to ignore git requirements
+
+        requirements.append((lineno, line, match))
+
+    return requirements
 
 
 if __name__ == '__main__':

test/sanity/ignore.txt

@@ -46,6 +46,13 @@ lib/ansible/galaxy/collection/__init__.py compile-2.6!skip # 'ansible-galaxy col
 lib/ansible/galaxy/collection/galaxy_api_proxy.py compile-2.6!skip  # 'ansible-galaxy collection' requires 2.7+
 lib/ansible/galaxy/dependency_resolution/dataclasses.py compile-2.6!skip  # 'ansible-galaxy collection' requires 2.7+
 lib/ansible/galaxy/dependency_resolution/providers.py compile-2.6!skip  # 'ansible-galaxy collection' requires 2.7+
+lib/ansible/module_utils/compat/selinux.py import-2.6!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-2.7!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-3.5!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-3.6!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-3.7!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-3.8!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-3.9!skip # pass/fail depends on presence of libselinux.so
 lib/ansible/module_utils/compat/_selectors2.py future-import-boilerplate # ignore bundled
 lib/ansible/module_utils/compat/_selectors2.py metaclass-boilerplate # ignore bundled
 lib/ansible/module_utils/compat/_selectors2.py pylint:blacklisted-name
@@ -196,7 +203,6 @@ test/integration/targets/win_script/files/test_script_removes_file.ps1 pslint:PS
 test/integration/targets/win_script/files/test_script_with_args.ps1 pslint:PSAvoidUsingWriteHost # Keep
 test/integration/targets/win_script/files/test_script_with_splatting.ps1 pslint:PSAvoidUsingWriteHost # Keep
 test/integration/targets/windows-minimal/library/win_ping_syntax_error.ps1 pslint!skip
-test/lib/ansible_test/_data/requirements/constraints.txt test-constraints
 test/lib/ansible_test/_data/requirements/integration.cloud.azure.txt test-constraints
 test/lib/ansible_test/_data/requirements/sanity.ps1 pslint:PSCustomUseLiteralPath # Uses wildcards on purpose
 test/lib/ansible_test/_data/sanity/pylint/plugins/string_format.py use-compat-six