diff --git a/changelogs/fragments/ansible-test-updates.yml b/changelogs/fragments/ansible-test-updates.yml new file mode 100644 index 00000000000..4d2318a8462 --- /dev/null +++ b/changelogs/fragments/ansible-test-updates.yml @@ -0,0 +1,11 @@ +minor_changes: + - ansible-test - Most sanity tests are now skipped on Python 3.5 and earlier with a warning. + Previously this was done for Python 2.7 and earlier. + - ansible-test - Removed ``pip`` constraints related to integration tests that have been moved to collections. + This should reduce conflicts with ``pip`` requirements and constraints when testing collections. + - ansible-test - Most sanity test specific ``pip`` constraints are now used only when running sanity tests. + This should reduce conflicts with ``pip`` requirements and constraints when testing collections. + - ansible-test - More sanity test requirements have been pinned to specific versions to provide consistent test results. + - ansible-test - Improved handling of minimum Python version requirements for sanity tests. + Supported versions are now included in warning messages displayed when tests are skipped. + - ansible-test - Silence ``pip`` warnings about Python 3.5 being EOL when installing requirements. diff --git a/test/lib/ansible_test/_data/quiet_pip.py b/test/lib/ansible_test/_data/quiet_pip.py index 7d2a6d16af1..5b5db823e22 100644 --- a/test/lib/ansible_test/_data/quiet_pip.py +++ b/test/lib/ansible_test/_data/quiet_pip.py @@ -40,6 +40,10 @@ WARNING_MESSAGE_FILTERS = ( # pip 21.0 will drop support for Python 2.7 in January 2021. # More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support 'DEPRECATION: Python 2.7 reached the end of its life ', + + # DEPRECATION: Python 3.5 reached the end of its life on September 13th, 2020. Please upgrade your Python as Python 3.5 is no longer maintained. + # pip 21.0 will drop support for Python 3.5 in January 2021. pip 21.0 will remove support for this functionality. + 'DEPRECATION: Python 3.5 reached the end of its life ', ) diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index dc3cad2bfd5..c14f4af55e1 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -13,7 +13,6 @@ sphinx <= 2.1.2 ; python_version >= '2.7' # docs team hasn't tested beyond 2.1. rstcheck >=3.3.1 # required for sphinx version >= 1.8 pygments >= 2.4.0 # Pygments 2.4.0 includes bugfixes for YAML and YAML+Jinja lexers wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later -yamllint != 1.8.0, < 1.14.0 ; python_version < '2.7' # yamllint 1.8.0 and 1.14.0+ require python 2.7+ pycrypto >= 2.6 # Need features found in 2.6 and greater ncclient >= 0.5.2 # Need features added in 0.5.2 and greater idna < 2.6, >= 2.5 # linode requires idna < 2.9, >= 2.5, requests requires idna < 2.6, but cryptography will cause the latest version to be installed instead @@ -26,13 +25,11 @@ ntlm-auth >= 1.3.0 # message encryption support using cryptography requests < 2.20.0 ; python_version < '2.7' # requests 2.20.0 drops support for python 2.6 requests-ntlm >= 1.1.0 # message encryption support requests-credssp >= 0.1.0 # message encryption support -voluptuous >= 0.11.0 # Schema recursion via Self openshift >= 0.6.2, < 0.9.0 # merge_type support virtualenv < 16.0.0 ; python_version < '2.7' # virtualenv 16.0.0 and later require python 2.7 or later pathspec < 0.6.0 ; python_version < '2.7' # pathspec 0.6.0 and later require python 2.7 or later pyopenssl < 18.0.0 ; python_version < '2.7' # pyOpenSSL 18.0.0 and later require python 2.7 or later pyparsing < 3.0.0 ; python_version < '3.5' # pyparsing 3 and later require python 3.5 or later -pyfmg == 0.6.1 # newer versions do not pass current unit tests pyyaml < 5.1 ; python_version < '2.7' # pyyaml 5.1 and later require python 2.7 or later pycparser < 2.19 ; python_version < '2.7' # pycparser 2.19 and later require python 2.7 or later mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...) @@ -47,23 +44,3 @@ botocore >= 1.10.0 ; python_version >= '2.7' # adds support for the following AW setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require python 2.7 or later setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later gssapi < 1.6.0 ; python_version <= '2.7' # gssapi 1.6.0 and later require python 3 or later - -# freeze antsibull-changelog for consistent test results -antsibull-changelog == 0.9.0 - -# Make sure we have a new enough antsibull for the CLI args we use -antsibull >= 0.21.0 - -# freeze pylint and its requirements for consistent test results -# NOTE: six is not frozen since it is a requirement for more than just pylint -astroid == 2.4.2 -isort == 5.7.0 -lazy-object-proxy == 1.4.3 -mccabe == 0.6.1 -pylint == 2.6.0 -toml == 0.10.2 -typed-ast == 1.4.2 -wrapt == 1.12.1 - -# freeze pycodestyle for consistent test results -pycodestyle == 2.6.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt index 8a98acc9d0f..a346a8d93eb 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt @@ -1,2 +1 @@ -# changelog build requires python 3.6+ -antsibull-changelog ; python_version >= '3.6' +antsibull-changelog == 0.9.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt b/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt index c3726e8bfee..7044777a158 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt @@ -1 +1 @@ -pyyaml +pyyaml # not frozen due to usage outside sanity tests diff --git a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt index 282a93fbd7c..86f73fba11e 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt @@ -1 +1 @@ -pycodestyle +pycodestyle == 2.6.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt index 438ca51dad4..ecdc2197d2a 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt @@ -1,3 +1,12 @@ -pylint +pylint == 2.6.0 pyyaml # needed for collection_detail.py -mccabe # pylint complexity testing + +# dependencies +astroid == 2.4.2 +isort == 5.7.0 +lazy-object-proxy == 1.4.3 +mccabe == 0.6.1 +six # not frozen due to usage outside sanity tests +toml == 0.10.2 +typed-ast == 1.4.2 +wrapt == 1.12.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt index edd96991dde..1eaef006e99 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt @@ -1,2 +1,2 @@ -pyyaml -voluptuous +pyyaml # not frozen due to usage outside sanity tests +voluptuous == 0.12.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt index efe940041c5..8288b14b73f 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt @@ -1,3 +1,3 @@ jinja2 # ansible-core requirement pyyaml # needed for collection_detail.py -voluptuous +voluptuous == 0.12.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt b/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt index b2c729ca4de..e0eac4e7902 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt @@ -1 +1,5 @@ -yamllint +yamllint == 1.26.0 + +# dependencies +pathspec # not frozen since it should not impact test results +pyyaml # not frozen due to usage outside sanity tests diff --git a/test/lib/ansible_test/_data/sanity/code-smell/changelog.json b/test/lib/ansible_test/_data/sanity/code-smell/changelog.json index 87f223b1802..7d19f101f29 100644 --- a/test/lib/ansible_test/_data/sanity/code-smell/changelog.json +++ b/test/lib/ansible_test/_data/sanity/code-smell/changelog.json @@ -1,6 +1,5 @@ { "intercept": true, - "minimum_python_version": "3.6", "prefixes": [ "changelogs/config.yaml", "changelogs/fragments/" diff --git a/test/lib/ansible_test/_internal/sanity/__init__.py b/test/lib/ansible_test/_internal/sanity/__init__.py index 976bbb2f09f..44cdd9e6277 100644 --- a/test/lib/ansible_test/_internal/sanity/__init__.py +++ b/test/lib/ansible_test/_internal/sanity/__init__.py @@ -141,7 +141,8 @@ def command_sanity(args): options = '' if test.supported_python_versions and version not in test.supported_python_versions: - display.warning("Skipping sanity test '%s' on unsupported Python %s." % (test.name, version)) + display.warning("Skipping sanity test '%s' on Python %s. Supported Python versions: %s" % ( + test.name, version, ', '.join(test.supported_python_versions))) result = SanitySkipped(test.name, skip_version) elif not args.python and version not in available_versions: display.warning("Skipping sanity test '%s' on Python %s due to missing interpreter." % (test.name, version)) @@ -658,7 +659,7 @@ class SanityTest(ABC): @property def supported_python_versions(self): # type: () -> t.Optional[t.Tuple[str, ...]] """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" - return tuple(python_version for python_version in SUPPORTED_PYTHON_VERSIONS if python_version.startswith('3.')) + return tuple(python_version for python_version in SUPPORTED_PYTHON_VERSIONS if str_to_version(python_version) >= (3, 6)) def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] # pylint: disable=unused-argument """Return the given list of test targets, filtered to include only those relevant for the test.""" @@ -751,6 +752,16 @@ class SanityCodeSmellTest(SanityTest): """True if the test targets should include symlinks.""" return self.__include_symlinks + @property + def supported_python_versions(self): # type: () -> t.Optional[t.Tuple[str, ...]] + """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" + versions = super(SanityCodeSmellTest, self).supported_python_versions + + if self.minimum_python_version: + versions = tuple(version for version in versions if str_to_version(version) >= str_to_version(self.minimum_python_version)) + + return versions + def filter_targets(self, targets): # type: (t.List[TestTarget]) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test.""" if self.no_targets: @@ -785,12 +796,6 @@ class SanityCodeSmellTest(SanityTest): :type python_version: str :rtype: TestResult """ - if self.minimum_python_version: - if str_to_version(python_version) < str_to_version(self.minimum_python_version): - display.warning("Skipping sanity test '%s' on unsupported Python %s; requires Python %s or newer." % ( - self.name, python_version, self.minimum_python_version)) - return SanitySkipped(self.name, 'Test requires Python %s or newer' % (self.minimum_python_version, )) - cmd = [find_python(python_version), self.path] env = ansible_environment(args, color=False) diff --git a/test/sanity/code-smell/botmeta.requirements.txt b/test/sanity/code-smell/botmeta.requirements.txt index edd96991dde..d0ed4f2d5bc 100644 --- a/test/sanity/code-smell/botmeta.requirements.txt +++ b/test/sanity/code-smell/botmeta.requirements.txt @@ -1,2 +1,2 @@ pyyaml -voluptuous +voluptuous == 0.12.1 diff --git a/test/sanity/code-smell/docs-build.requirements.txt b/test/sanity/code-smell/docs-build.requirements.txt index c857a5c2877..3d7943102b7 100644 --- a/test/sanity/code-smell/docs-build.requirements.txt +++ b/test/sanity/code-smell/docs-build.requirements.txt @@ -4,4 +4,4 @@ resolvelib sphinx sphinx-notfound-page straight.plugin -antsibull +antsibull == 0.26.0 diff --git a/test/sanity/code-smell/package-data.requirements.txt b/test/sanity/code-smell/package-data.requirements.txt index bd9d3b516b1..41b3b5772a5 100644 --- a/test/sanity/code-smell/package-data.requirements.txt +++ b/test/sanity/code-smell/package-data.requirements.txt @@ -4,8 +4,6 @@ packaging pyyaml # ansible-core requirement resolvelib # ansible-core requirement rstcheck -setuptools > 39.2 +setuptools straight.plugin - -# changelog build requires python 3.6+ -antsibull-changelog ; python_version >= '3.6' +antsibull-changelog == 0.9.0 diff --git a/test/sanity/code-smell/test-constraints.json b/test/sanity/code-smell/test-constraints.json index 69b07bf3bbb..8f47beb0109 100644 --- a/test/sanity/code-smell/test-constraints.json +++ b/test/sanity/code-smell/test-constraints.json @@ -1,6 +1,8 @@ { + "all_targets": true, "prefixes": [ - "test/lib/ansible_test/_data/requirements/" + "test/lib/ansible_test/_data/requirements/", + "test/sanity/code-smell/" ], "extensions": [ ".txt" diff --git a/test/sanity/code-smell/test-constraints.py b/test/sanity/code-smell/test-constraints.py index e8b9c7952a8..1c88995f5b8 100755 --- a/test/sanity/code-smell/test-constraints.py +++ b/test/sanity/code-smell/test-constraints.py @@ -7,14 +7,88 @@ import sys def main(): + constraints_path = 'test/lib/ansible_test/_data/requirements/constraints.txt' + + requirements = {} + for path in sys.argv[1:] or sys.stdin.read().splitlines(): with open(path, 'r') as path_fd: - for line, text in enumerate(path_fd.readlines()): - match = re.search(r'^[^;#]*?([<>=])(?!.*sanity_ok.*)', text) + requirements[path] = parse_requirements(path_fd.read().splitlines()) - if match: - print('%s:%d:%d: put constraints in `test/lib/ansible_test/_data/requirements/constraints.txt`' % ( - path, line + 1, match.start(1) + 1)) + frozen_sanity = {} + non_sanity_requirements = set() + + for path, requirements in requirements.items(): + for lineno, line, requirement in requirements: + if not requirement: + print('%s:%d:%d: cannot parse requirement: %s' % (path, lineno, 1, line)) + continue + + name = requirement.group('name').lower() + raw_constraints = requirement.group('constraints') + raw_markers = requirement.group('markers') + constraints = raw_constraints.strip() + markers = raw_markers.strip() + comment = requirement.group('comment') + + is_sanity = path.startswith('test/lib/ansible_test/_data/requirements/sanity.') or path.startswith('test/sanity/code-smell/') + is_pinned = re.search('^ *== *[0-9.]+$', constraints) + is_constraints = path == constraints_path + + if is_sanity: + sanity = frozen_sanity.setdefault(name, []) + sanity.append((path, lineno, line, requirement)) + elif not is_constraints: + non_sanity_requirements.add(name) + + if constraints and not is_constraints: + allow_constraints = 'sanity_ok' in comment + + if is_sanity and is_pinned and not markers: + allow_constraints = True # sanity tests can use frozen requirements without markers + + if not allow_constraints: + if is_sanity: + # sanity test requirements which need constraints should be frozen to maintain consistent test results + # use of anything other than frozen constraints will make evaluation of conflicts extremely difficult + print('%s:%d:%d: sanity test constraint (%s%s) must be frozen (use `==`)' % (path, lineno, 1, name, raw_constraints)) + else: + # keeping constraints for tests other than sanity tests in one file helps avoid conflicts + print('%s:%d:%d: put the constraint (%s%s) in `%s`' % (path, lineno, 1, name, raw_constraints, constraints_path)) + + for name, requirements in frozen_sanity.items(): + for req in requirements: + if name in non_sanity_requirements and req[3].group('constraints').strip(): + print('%s:%d:%d: sanity constraint (%s) for package `%s` is not allowed because `%s` is used outside sanity tests' % ( + req[0], req[1], req[3].start('constraints') + 1, req[3].group('constraints'), name, name)) + + if len(set(req[3].group('constraints').strip() for req in requirements)) != 1: + for req in requirements: + print('%s:%d:%d: sanity constraint (%s) does not match others for package `%s`' % ( + req[0], req[1], req[3].start('constraints') + 1, req[3].group('constraints'), name)) + + +def parse_requirements(lines): + # see https://www.python.org/dev/peps/pep-0508/#names + pattern = re.compile(r'^(?P[A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])(?P *\[[^]]*])?(?P[^;#]*)(?P[^#]*)(?P.*)$', + re.IGNORECASE) + + matches = [(lineno, line, pattern.search(line)) for lineno, line in enumerate(lines, start=1)] + requirements = [] + + for lineno, line, match in matches: + if not line.strip(): + continue + + if line.strip().startswith('#'): + continue + + if line.startswith('git+https://'): + continue # hack to ignore git requirements + + requirements.append((lineno, line, match)) + + return requirements if __name__ == '__main__': diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 6a0510e234d..8a30f046c59 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -46,6 +46,13 @@ lib/ansible/galaxy/collection/__init__.py compile-2.6!skip # 'ansible-galaxy col lib/ansible/galaxy/collection/galaxy_api_proxy.py compile-2.6!skip # 'ansible-galaxy collection' requires 2.7+ lib/ansible/galaxy/dependency_resolution/dataclasses.py compile-2.6!skip # 'ansible-galaxy collection' requires 2.7+ lib/ansible/galaxy/dependency_resolution/providers.py compile-2.6!skip # 'ansible-galaxy collection' requires 2.7+ +lib/ansible/module_utils/compat/selinux.py import-2.6!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-2.7!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.5!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.6!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.7!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.8!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.9!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/_selectors2.py future-import-boilerplate # ignore bundled lib/ansible/module_utils/compat/_selectors2.py metaclass-boilerplate # ignore bundled lib/ansible/module_utils/compat/_selectors2.py pylint:blacklisted-name @@ -196,7 +203,6 @@ test/integration/targets/win_script/files/test_script_removes_file.ps1 pslint:PS test/integration/targets/win_script/files/test_script_with_args.ps1 pslint:PSAvoidUsingWriteHost # Keep test/integration/targets/win_script/files/test_script_with_splatting.ps1 pslint:PSAvoidUsingWriteHost # Keep test/integration/targets/windows-minimal/library/win_ping_syntax_error.ps1 pslint!skip -test/lib/ansible_test/_data/requirements/constraints.txt test-constraints test/lib/ansible_test/_data/requirements/integration.cloud.azure.txt test-constraints test/lib/ansible_test/_data/requirements/sanity.ps1 pslint:PSCustomUseLiteralPath # Uses wildcards on purpose test/lib/ansible_test/_data/sanity/pylint/plugins/string_format.py use-compat-six