From becf9416736dc911d3411b92f09512b4dae2955c Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 5 Apr 2021 16:48:59 -0700 Subject: [PATCH] Add PyPI proxy container for tests on Python 2.6. --- .../ansible-test-pypi-test-container.yml | 3 + test/lib/ansible_test/_data/quiet_pip.py | 1 + test/lib/ansible_test/_internal/cli.py | 11 ++ test/lib/ansible_test/_internal/config.py | 3 + test/lib/ansible_test/_internal/delegation.py | 9 ++ .../lib/ansible_test/_internal/docker_util.py | 2 +- test/lib/ansible_test/_internal/executor.py | 140 ++++++++++++++++++ 7 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 changelogs/fragments/ansible-test-pypi-test-container.yml diff --git a/changelogs/fragments/ansible-test-pypi-test-container.yml b/changelogs/fragments/ansible-test-pypi-test-container.yml new file mode 100644 index 00000000000..5c2f10c90ac --- /dev/null +++ b/changelogs/fragments/ansible-test-pypi-test-container.yml @@ -0,0 +1,3 @@ +major_changes: + - ansible-test - Tests run with the ``centos6`` and ``default`` test containers now use a PyPI proxy container to access PyPI when Python 2.6 is used. + This allows tests running under Python 2.6 to continue functioning even though PyPI is discontinuing support for non-SNI capable clients. diff --git a/test/lib/ansible_test/_data/quiet_pip.py b/test/lib/ansible_test/_data/quiet_pip.py index 5b5db823e22..e1bb8246464 100644 --- a/test/lib/ansible_test/_data/quiet_pip.py +++ b/test/lib/ansible_test/_data/quiet_pip.py @@ -13,6 +13,7 @@ LOGGING_MESSAGE_FILTER = re.compile("^(" ".*Running pip install with root privileges is generally not a good idea.*|" # custom Fedora patch [1] "DEPRECATION: Python 2.7 will reach the end of its life .*|" # pip 19.2.3 "Ignoring .*: markers .* don't match your environment|" + "Looking in indexes: .*|" # pypi-test-container "Requirement already satisfied.*" ")$") diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py index 7e2650c7026..15a235180bb 100644 --- a/test/lib/ansible_test/_internal/cli.py +++ b/test/lib/ansible_test/_internal/cli.py @@ -39,6 +39,7 @@ from .executor import ( Delegate, generate_pip_install, check_startup, + configure_pypi_proxy, ) from .config import ( @@ -170,6 +171,7 @@ def main(): display.info('MAXFD: %d' % MAXFD, verbosity=2) try: + configure_pypi_proxy(config) args.func(config) delegate_args = None except Delegate as ex: @@ -236,6 +238,15 @@ def parse_args(): default=0, help='display more output') + common.add_argument('--pypi-proxy', + action='store_true', + help=argparse.SUPPRESS) # internal use only + + common.add_argument('--pypi-endpoint', + metavar='URI', + default=None, + help=argparse.SUPPRESS) # internal use only + common.add_argument('--color', metavar='COLOR', nargs='?', diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index fcae6687271..eb9c1739a99 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -68,6 +68,9 @@ class EnvironmentConfig(CommonConfig): """ super(EnvironmentConfig, self).__init__(args, command) + self.pypi_endpoint = args.pypi_endpoint # type: str + self.pypi_proxy = args.pypi_proxy # type: bool + self.local = args.local is True self.venv = args.venv self.venv_system_site_packages = args.venv_system_site_packages diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index e2ab3217ad8..250b9114af7 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -19,6 +19,7 @@ from .executor import ( HTTPTESTER_HOSTS, create_shell_command, run_httptester, + run_pypi_proxy, start_httptester, get_python_interpreter, get_python_version, @@ -285,6 +286,11 @@ def delegate_docker(args, exclude, require, integration_targets): if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy): cmd_options.append('-it') + pypi_proxy_id, pypi_proxy_endpoint = run_pypi_proxy(args) + + if pypi_proxy_endpoint: + cmd += ['--pypi-endpoint', pypi_proxy_endpoint] + with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd: try: create_payload(args, local_source_fd.name) @@ -406,6 +412,9 @@ def delegate_docker(args, exclude, require, integration_targets): if httptester_id: docker_rm(args, httptester_id) + if pypi_proxy_id: + docker_rm(args, pypi_proxy_id) + if test_id: if args.docker_terminate == 'always' or (args.docker_terminate == 'success' and success): docker_rm(args, test_id) diff --git a/test/lib/ansible_test/_internal/docker_util.py b/test/lib/ansible_test/_internal/docker_util.py index c04a2a02f96..3ad771bd41c 100644 --- a/test/lib/ansible_test/_internal/docker_util.py +++ b/test/lib/ansible_test/_internal/docker_util.py @@ -112,7 +112,7 @@ def get_docker_container_ip(args, container_id): networks = network_settings.get('Networks') if networks: - network_name = get_docker_preferred_network_name(args) + network_name = get_docker_preferred_network_name(args) or 'bridge' ipaddress = networks[network_name]['IPAddress'] else: # podman doesn't provide Networks, fall back to using IPAddress diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index 421354e24fc..c3755a7113a 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import atexit import json import os import datetime @@ -81,6 +82,7 @@ from .util_common import ( write_json_test_results, ResultType, handle_layout_messages, + CommonConfig, ) from .docker_util import ( @@ -126,6 +128,8 @@ from .config import ( ShellConfig, WindowsIntegrationConfig, TIntegrationConfig, + UnitsConfig, + SanityConfig, ) from .metadata import ( @@ -145,6 +149,10 @@ from .data import ( data_context, ) +from .http import ( + urlparse, +) + HTTPTESTER_HOSTS = ( 'ansible.http.tests', 'sni1.ansible.http.tests', @@ -1404,6 +1412,138 @@ rdr pass inet proto tcp from any to any port 749 -> 127.0.0.1 port 8749 raise ApplicationError('No supported port forwarding mechanism detected.') +def run_pypi_proxy(args): # type: (EnvironmentConfig) -> t.Tuple[t.Optional[str], t.Optional[str]] + """Run a PyPI proxy container, returning the container ID and proxy endpoint.""" + use_proxy = False + + if args.docker_raw == 'centos6': + use_proxy = True # python 2.6 is the only version available + + if args.docker_raw == 'default': + if args.python == '2.6': + use_proxy = True # python 2.6 requested + elif not args.python and isinstance(args, (SanityConfig, UnitsConfig, ShellConfig)): + use_proxy = True # multiple versions (including python 2.6) can be used + + if args.docker_raw and args.pypi_proxy: + use_proxy = True # manual override to force proxy usage + + if not use_proxy: + return None, None + + proxy_image = 'quay.io/ansible/pypi-test-container:1.0.0' + port = 3141 + + options = [ + '--detach', + '-p', '%d:%d' % (port, port), + ] + + docker_pull(args, proxy_image) + + container_id = docker_run(args, proxy_image, options=options)[0] + + if args.explain: + container_id = 'pypi_id' + container_ip = '127.0.0.1' + else: + container_id = container_id.strip() + container_ip = get_docker_container_ip(args, container_id) + + endpoint = 'http://%s:%d/root/pypi/+simple/' % (container_ip, port) + + return container_id, endpoint + + +def configure_pypi_proxy(args): # type: (CommonConfig) -> None + """Configure the environment to use a PyPI proxy, if present.""" + if not isinstance(args, EnvironmentConfig): + return + + if args.pypi_endpoint: + configure_pypi_block_access() + configure_pypi_proxy_pip(args) + configure_pypi_proxy_easy_install(args) + + +def configure_pypi_block_access(): # type: () -> None + """Block direct access to PyPI to ensure proxy configurations are always used.""" + if os.getuid() != 0: + display.warning('Skipping custom hosts block for PyPI for non-root user.') + return + + hosts_path = '/etc/hosts' + hosts_block = ''' +127.0.0.1 pypi.org pypi.python.org files.pythonhosted.org +''' + + def hosts_cleanup(): + display.info('Removing custom PyPI hosts entries: %s' % hosts_path, verbosity=1) + + with open(hosts_path) as hosts_file_read: + content = hosts_file_read.read() + + content = content.replace(hosts_block, '') + + with open(hosts_path, 'w') as hosts_file_write: + hosts_file_write.write(content) + + display.info('Injecting custom PyPI hosts entries: %s' % hosts_path, verbosity=1) + display.info('Config: %s\n%s' % (hosts_path, hosts_block), verbosity=3) + + with open(hosts_path, 'a') as hosts_file: + hosts_file.write(hosts_block) + + atexit.register(hosts_cleanup) + + +def configure_pypi_proxy_pip(args): # type: (EnvironmentConfig) -> None + """Configure a custom index for pip based installs.""" + pypi_hostname = urlparse(args.pypi_endpoint)[1].split(':')[0] + + pip_conf_path = os.path.expanduser('~/.pip/pip.conf') + pip_conf = ''' +[global] +index-url = {0} +trusted-host = {1} +'''.format(args.pypi_endpoint, pypi_hostname).strip() + + def pip_conf_cleanup(): + display.info('Removing custom PyPI config: %s' % pip_conf_path, verbosity=1) + os.remove(pip_conf_path) + + if os.path.exists(pip_conf_path): + raise ApplicationError('Refusing to overwrite existing file: %s' % pip_conf_path) + + display.info('Injecting custom PyPI config: %s' % pip_conf_path, verbosity=1) + display.info('Config: %s\n%s' % (pip_conf_path, pip_conf), verbosity=3) + + write_text_file(pip_conf_path, pip_conf, True) + atexit.register(pip_conf_cleanup) + + +def configure_pypi_proxy_easy_install(args): # type: (EnvironmentConfig) -> None + """Configure a custom index for easy_install based installs.""" + pydistutils_cfg_path = os.path.expanduser('~/.pydistutils.cfg') + pydistutils_cfg = ''' +[easy_install] +index_url = {0} +'''.format(args.pypi_endpoint).strip() + + if os.path.exists(pydistutils_cfg_path): + raise ApplicationError('Refusing to overwrite existing file: %s' % pydistutils_cfg_path) + + def pydistutils_cfg_cleanup(): + display.info('Removing custom PyPI config: %s' % pydistutils_cfg_path, verbosity=1) + os.remove(pydistutils_cfg_path) + + display.info('Injecting custom PyPI config: %s' % pydistutils_cfg_path, verbosity=1) + display.info('Config: %s\n%s' % (pydistutils_cfg_path, pydistutils_cfg), verbosity=3) + + write_text_file(pydistutils_cfg_path, pydistutils_cfg, True) + atexit.register(pydistutils_cfg_cleanup) + + def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, inventory_path, temp_path, always): """ :type args: IntegrationConfig