From 12b33c79ee8718c81518455a904aa64817cf32cb Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 9 Dec 2020 12:41:15 -0800 Subject: [PATCH] [stable-2.10] Fix cryptography constraints in ansible-test. (#72914) (#72922) (cherry picked from commit 36ab3d1189075000693cea928f32839bfabae36c) Co-authored-by: Matt Clay --- .../ansible-test-constraints-cryptography.yml | 2 + .../_data/cryptography-constraints.txt | 1 + .../_data/requirements/constraints.txt | 2 +- test/lib/ansible_test/_data/sslcheck.py | 24 ++++++++++ test/lib/ansible_test/_internal/executor.py | 44 ++++++++++++++++++- test/lib/ansible_test/_internal/util.py | 4 +- 6 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/ansible-test-constraints-cryptography.yml create mode 100755 test/lib/ansible_test/_data/sslcheck.py diff --git a/changelogs/fragments/ansible-test-constraints-cryptography.yml b/changelogs/fragments/ansible-test-constraints-cryptography.yml new file mode 100644 index 00000000000..6304a6ce79e --- /dev/null +++ b/changelogs/fragments/ansible-test-constraints-cryptography.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-test - ``cryptography`` is now limited to versions prior to 3.2 only when an incompatible OpenSSL version (earlier than 1.1.0) is detected diff --git a/test/lib/ansible_test/_data/cryptography-constraints.txt b/test/lib/ansible_test/_data/cryptography-constraints.txt index b057c83f9f2..8e3e99b4ae2 100644 --- a/test/lib/ansible_test/_data/cryptography-constraints.txt +++ b/test/lib/ansible_test/_data/cryptography-constraints.txt @@ -1,2 +1,3 @@ +# do not add a cryptography constraint here, see the get_cryptography_requirement function in executor.py for details idna < 2.8 ; python_version < '2.7' # idna 2.8+ requires python 2.7+ cffi != 1.14.4 # Fails on systems with older gcc. Should be fixed in the next release. https://foss.heptapod.net/pypy/cffi/-/issues/480 diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index 48d5636e8d3..f613ef0e603 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -2,7 +2,7 @@ coverage >= 4.5.1, < 5.0.0 ; python_version < '3.7' # coverage 4.4 required for coverage >= 4.5.2, < 5.0.0 ; python_version == '3.7' # coverage 4.5.2 fixes bugs in support for python 3.7, coverage 5.0+ incompatible coverage >= 4.5.4, < 5.0.0 ; python_version > '3.7' # coverage had a bug in < 4.5.4 that would cause unit tests to hang in Python 3.8, coverage 5.0+ incompatible cryptography < 2.2 ; python_version < '2.7' # cryptography 2.2 drops support for python 2.6 -cryptography < 3.2 ; python_version >= '2.7' # cryptography 3.2 drops support for OpenSSL 1.0.2 which some of our CI hosts (FreeBSD) still use +# do not add a cryptography constraint here unless it is for python version incompatibility, see the get_cryptography_requirement function in executor.py for details deepdiff < 4.0.0 ; python_version < '3' # deepdiff 4.0.0 and later require python 3 jinja2 < 2.11 ; python_version < '2.7' # jinja2 2.11 and later require python 2.7 or later urllib3 < 1.24 ; python_version < '2.7' # urllib3 1.24 and later require python 2.7 or later diff --git a/test/lib/ansible_test/_data/sslcheck.py b/test/lib/ansible_test/_data/sslcheck.py new file mode 100755 index 00000000000..37b8227936d --- /dev/null +++ b/test/lib/ansible_test/_data/sslcheck.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +"""Show openssl version.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json + +# noinspection PyBroadException +try: + from ssl import OPENSSL_VERSION_INFO + VERSION = list(OPENSSL_VERSION_INFO[:3]) +except Exception: # pylint: disable=broad-except + VERSION = None + + +def main(): + """Main program entry point.""" + print(json.dumps(dict( + version=VERSION, + ))) + + +if __name__ == '__main__': + main() diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index 1e32b1d3e24..881439ef945 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -68,6 +68,7 @@ from .util import ( open_zipfile, SUPPORTED_PYTHON_VERSIONS, str_to_version, + version_to_str, ) from .util_common import ( @@ -185,6 +186,42 @@ def create_shell_command(command): return cmd +def get_openssl_version(args, python, python_version): # type: (EnvironmentConfig, str, str) -> t.Optional[t.Tuple[int, ...]] + """Return the openssl version.""" + if not python_version.startswith('2.'): + # OpenSSL version checking only works on Python 3.x. + # This should be the most accurate, since it is the Python we will be using. + version = json.loads(run_command(args, [python, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'sslcheck.py')], capture=True, always=True)[0])['version'] + + if version: + display.info('Detected OpenSSL version %s under Python %s.' % (version_to_str(version), python_version), verbosity=1) + + return tuple(version) + + # Fall back to detecting the OpenSSL version from the CLI. + # This should provide an adequate solution on Python 2.x. + openssl_path = find_executable('openssl', required=False) + + if openssl_path: + try: + result = raw_command([openssl_path, 'version'], capture=True)[0] + except SubprocessError: + result = '' + + match = re.search(r'^OpenSSL (?P[0-9]+\.[0-9]+\.[0-9]+)', result) + + if match: + version = str_to_version(match.group('version')) + + display.info('Detected OpenSSL version %s using the openssl CLI.' % version_to_str(version), verbosity=1) + + return version + + display.info('Unable to detect OpenSSL version.', verbosity=1) + + return None + + def get_setuptools_version(args, python): # type: (EnvironmentConfig, str) -> t.Tuple[int] """Return the setuptools version for the given python.""" try: @@ -199,16 +236,21 @@ def get_setuptools_version(args, python): # type: (EnvironmentConfig, str) -> t def get_cryptography_requirement(args, python_version): # type: (EnvironmentConfig, str) -> str """ Return the correct cryptography requirement for the given python version. - The version of cryptograpy installed depends on the python version and setuptools version. + The version of cryptography installed depends on the python version, setuptools version and openssl version. """ python = find_python(python_version) setuptools_version = get_setuptools_version(args, python) + openssl_version = get_openssl_version(args, python, python_version) if setuptools_version >= (18, 5): if python_version == '2.6': # cryptography 2.2+ requires python 2.7+ # see https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#22---2018-03-19 cryptography = 'cryptography < 2.2' + elif openssl_version and openssl_version < (1, 1, 0): + # cryptography 3.2 requires openssl 1.1.x or later + # see https://cryptography.io/en/latest/changelog.html#v3-2 + cryptography = 'cryptography < 3.2' else: cryptography = 'cryptography' else: diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index f0c902a7148..005c3e053d0 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -770,12 +770,12 @@ def paths_to_dirs(paths): # type: (t.List[str]) -> t.List[str] return sorted(dir_names) -def str_to_version(version): # type: (str) -> t.Tuple[int] +def str_to_version(version): # type: (str) -> t.Tuple[int, ...] """Return a version tuple from a version string.""" return tuple(int(n) for n in version.split('.')) -def version_to_str(version): # type: (t.Tuple[int]) -> str +def version_to_str(version): # type: (t.Tuple[int, ...]) -> str """Return a version string from a version tuple.""" return '.'.join(str(n) for n in version)