[stable-2.10] Fix ansible-test handling of user-defined docker networks. (#72256)

* Fix ansible-test docker container detection.

* Attach test containers to the correct network.

* Do not assume `localhost` for accesing Docker.

* Look for containers on current network.

* Always map /var/run/docker.sock into containers.

This fixes issues when using a remote Docker host.

* Support container IP lookup from networks list.

* Fix container network attachment.

* Remove redundant container detection messages.

* Limit DOCKER_HOST parsing to TCP.

* Restore docker socket existence check.

The check is skipped if the docker hostname is not localhost.

* Correct changelog entry..
(cherry picked from commit 3c2e8b99be)

Co-authored-by: Matt Clay <mclay@redhat.com>

changelogs/fragments/ansible-test-container-ip-lookup.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Prefer container IP at ``.NetworkSettings.Networks.{NetworkName}.IPAddress`` over ``.NetworkSettings.IPAddress``.

changelogs/fragments/ansible-test-docker-default-network.yml

@@ -0,0 +1,4 @@
+bugfixes:
+  - ansible-test - Always connect additional Docker containers to the network used by the current container (if any).
+minor_changes:
+  - ansible-test - Add a ``--docker-network`` option to choose the network for running containers when using the ``--docker`` option.

changelogs/fragments/ansible-test-docker-detection-fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Correctly detect running in a Docker container on Azure Pipelines.

changelogs/fragments/ansible-test-docker-not-localhost.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Attempt to detect the Docker hostname instead of assuming ``localhost``.

changelogs/fragments/ansible-test-docker-socket.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Always map ``/var/run/docker.sock`` into test containers created by the ``--docker`` option if the docker host is not ``localhost``.

changelogs/fragments/ansible-test-network-container-search.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - The ``cs`` and ``openshift`` test plugins now search for containers on the current network instead of assuming the ``bridge`` network.

test/lib/ansible_test/_internal/cli.py

@@ -1057,6 +1057,9 @@ def add_extra_docker_options(parser, integration=True):
                         action='store_true',
                         help='run docker container in privileged mode')
 
+    docker.add_argument('--docker-network',
+                        help='run using the specified docker network')
+
     # noinspection PyTypeChecker
     docker.add_argument('--docker-memory',
                         help='memory limit for docker in bytes', type=int)

test/lib/ansible_test/_internal/cloud/acme.py

@@ -28,6 +28,10 @@ from ..docker_util import (
     docker_inspect,
     docker_pull,
     get_docker_container_id,
+    get_docker_hostname,
+    get_docker_container_ip,
+    get_docker_preferred_network_name,
+    is_docker_user_defined_network,
 )
 
 
@@ -100,7 +104,9 @@ class ACMEProvider(CloudProvider):
         """Get any additional options needed when delegating tests to a docker container.
         :rtype: list[str]
         """
-        if self.managed:
+        network = get_docker_preferred_network_name(self.args)
+
+        if self.managed and not is_docker_user_defined_network(network):
             return ['--link', self.DOCKER_SIMULATOR_NAME]
 
         return []
@@ -116,9 +122,6 @@ class ACMEProvider(CloudProvider):
         """Create a ACME test container using docker."""
         container_id = get_docker_container_id()
 
-        if container_id:
-            display.info('Running in docker container: %s' % container_id, verbosity=1)
-
         self.container_name = self.DOCKER_SIMULATOR_NAME
 
         results = docker_inspect(self.args, self.container_name)
@@ -158,7 +161,7 @@ class ACMEProvider(CloudProvider):
             acme_host_ip = acme_host
             display.info('Found ACME test container address: %s' % acme_host, verbosity=1)
         else:
-            acme_host = 'localhost'
+            acme_host = get_docker_hostname()
             acme_host_ip = acme_host
 
         self._set_cloud_config('acme_host', acme_host)
@@ -167,9 +170,7 @@ class ACMEProvider(CloudProvider):
         self._wait_for_service('https', acme_host_ip, 14000, 'dir', 'ACME CA endpoint')
 
     def _get_simulator_address(self):
-        results = docker_inspect(self.args, self.container_name)
+        return get_docker_container_ip(self.args, self.container_name)
-        ipaddress = results[0]['NetworkSettings']['IPAddress']
-        return ipaddress
 
     def _setup_static(self):
         raise NotImplementedError()

test/lib/ansible_test/_internal/cloud/cs.py

@@ -35,6 +35,9 @@ from ..docker_util import (
     docker_network_inspect,
     docker_exec,
     get_docker_container_id,
+    get_docker_preferred_network_name,
+    get_docker_hostname,
+    is_docker_user_defined_network,
 )
 
 
@@ -90,7 +93,7 @@ class CsCloudProvider(CloudProvider):
         :rtype: list[str]
         """
         if self.managed:
-            return ['-R', '8888:localhost:8888']
+            return ['-R', '8888:%s:8888' % get_docker_hostname()]
 
         return []
 
@@ -98,7 +101,9 @@ class CsCloudProvider(CloudProvider):
         """Get any additional options needed when delegating tests to a docker container.
         :rtype: list[str]
         """
-        if self.managed:
+        network = get_docker_preferred_network_name(self.args)
+
+        if self.managed and not is_docker_user_defined_network(network):
             return ['--link', self.DOCKER_SIMULATOR_NAME]
 
         return []
@@ -169,11 +174,10 @@ class CsCloudProvider(CloudProvider):
         container_id = get_docker_container_id()
 
         if container_id:
-            display.info('Running in docker container: %s' % container_id, verbosity=1)
             self.host = self._get_simulator_address()
             display.info('Found CloudStack simulator container address: %s' % self.host, verbosity=1)
         else:
-            self.host = 'localhost'
+            self.host = get_docker_hostname()
 
         self.port = 8888
         self.endpoint = 'http://%s:%d' % (self.host, self.port)
@@ -190,6 +194,8 @@ class CsCloudProvider(CloudProvider):
 
             if self.args.docker:
                 host = self.DOCKER_SIMULATOR_NAME
+            elif self.args.remote:
+                host = 'localhost'
             else:
                 host = self.host
 
@@ -207,11 +213,12 @@ class CsCloudProvider(CloudProvider):
         self._write_config(config)
 
     def _get_simulator_address(self):
-        networks = docker_network_inspect(self.args, 'bridge')
+        current_network = get_docker_preferred_network_name(self.args)
+        networks = docker_network_inspect(self.args, current_network)
 
         try:
-            bridge = [network for network in networks if network['Name'] == 'bridge'][0]
+            network = [network for network in networks if network['Name'] == current_network][0]
-            containers = bridge['Containers']
+            containers = network['Containers']
             container = [containers[container] for container in containers if containers[container]['Name'] == self.DOCKER_SIMULATOR_NAME][0]
             return re.sub(r'/[0-9]+$', '', container['IPv4Address'])
         except Exception:

test/lib/ansible_test/_internal/cloud/foreman.py

@@ -21,6 +21,10 @@ from ..docker_util import (
     docker_inspect,
     docker_pull,
     get_docker_container_id,
+    get_docker_hostname,
+    get_docker_container_ip,
+    get_docker_preferred_network_name,
+    is_docker_user_defined_network,
 )
 
 
@@ -96,7 +100,12 @@ class ForemanProvider(CloudProvider):
 
         :rtype: list[str]
         """
-        return ['--link', self.DOCKER_SIMULATOR_NAME] if self.managed else []
+        network = get_docker_preferred_network_name(self.args)
+
+        if self.managed and not is_docker_user_defined_network(network):
+            return ['--link', self.DOCKER_SIMULATOR_NAME]
+
+        return []
 
     def cleanup(self):
         """Clean up the resource and temporary configs files after tests."""
@@ -110,12 +119,6 @@ class ForemanProvider(CloudProvider):
         foreman_port = 8080
         container_id = get_docker_container_id()
 
-        if container_id:
-            display.info(
-                'Running in docker container: %s' % container_id,
-                verbosity=1,
-            )
-
         self.container_name = self.DOCKER_SIMULATOR_NAME
 
         results = docker_inspect(self.args, self.container_name)
@@ -157,15 +160,13 @@ class ForemanProvider(CloudProvider):
                 % foreman_host, verbosity=1
             )
         else:
-            foreman_host = 'localhost'
+            foreman_host = get_docker_hostname()
 
         self._set_cloud_config('FOREMAN_HOST', foreman_host)
         self._set_cloud_config('FOREMAN_PORT', str(foreman_port))
 
     def _get_simulator_address(self):
-        results = docker_inspect(self.args, self.container_name)
+        return get_docker_container_ip(self.args, self.container_name)
-        ip_address = results[0]['NetworkSettings']['IPAddress']
-        return ip_address
 
     def _setup_static(self):
         raise NotImplementedError

test/lib/ansible_test/_internal/cloud/nios.py

@@ -21,6 +21,10 @@ from ..docker_util import (
     docker_inspect,
     docker_pull,
     get_docker_container_id,
+    get_docker_hostname,
+    get_docker_container_ip,
+    get_docker_preferred_network_name,
+    is_docker_user_defined_network,
 )
 
 
@@ -96,7 +100,12 @@ class NiosProvider(CloudProvider):
 
         :rtype: list[str]
         """
-        return ['--link', self.DOCKER_SIMULATOR_NAME] if self.managed else []
+        network = get_docker_preferred_network_name(self.args)
+
+        if self.managed and not is_docker_user_defined_network(network):
+            return ['--link', self.DOCKER_SIMULATOR_NAME]
+
+        return []
 
     def cleanup(self):
         """Clean up the resource and temporary configs files after tests."""
@@ -110,12 +119,6 @@ class NiosProvider(CloudProvider):
         nios_port = 443
         container_id = get_docker_container_id()
 
-        if container_id:
-            display.info(
-                'Running in docker container: %s' % container_id,
-                verbosity=1,
-            )
-
         self.container_name = self.DOCKER_SIMULATOR_NAME
 
         results = docker_inspect(self.args, self.container_name)
@@ -157,14 +160,12 @@ class NiosProvider(CloudProvider):
                 % nios_host, verbosity=1
             )
         else:
-            nios_host = 'localhost'
+            nios_host = get_docker_hostname()
 
         self._set_cloud_config('NIOS_HOST', nios_host)
 
     def _get_simulator_address(self):
-        results = docker_inspect(self.args, self.container_name)
+        return get_docker_container_ip(self.args, self.container_name)
-        ip_address = results[0]['NetworkSettings']['IPAddress']
-        return ip_address
 
     def _setup_static(self):
         raise NotImplementedError

test/lib/ansible_test/_internal/cloud/openshift.py

@@ -36,6 +36,9 @@ from ..docker_util import (
     docker_pull,
     docker_network_inspect,
     get_docker_container_id,
+    get_docker_preferred_network_name,
+    get_docker_hostname,
+    is_docker_user_defined_network,
 )
 
 
@@ -88,7 +91,7 @@ class OpenShiftCloudProvider(CloudProvider):
         :rtype: list[str]
         """
         if self.managed:
-            return ['-R', '8443:localhost:8443']
+            return ['-R', '8443:%s:8443' % get_docker_hostname()]
 
         return []
 
@@ -96,7 +99,9 @@ class OpenShiftCloudProvider(CloudProvider):
         """Get any additional options needed when delegating tests to a docker container.
         :rtype: list[str]
         """
-        if self.managed:
+        network = get_docker_preferred_network_name(self.args)
+
+        if self.managed and not is_docker_user_defined_network(network):
             return ['--link', self.DOCKER_CONTAINER_NAME]
 
         return []
@@ -141,11 +146,10 @@ class OpenShiftCloudProvider(CloudProvider):
         container_id = get_docker_container_id()
 
         if container_id:
-            display.info('Running in docker container: %s' % container_id, verbosity=1)
             host = self._get_container_address()
             display.info('Found OpenShift container address: %s' % host, verbosity=1)
         else:
-            host = 'localhost'
+            host = get_docker_hostname()
 
         port = 8443
         endpoint = 'https://%s:%s/' % (host, port)
@@ -157,6 +161,8 @@ class OpenShiftCloudProvider(CloudProvider):
         else:
             if self.args.docker:
                 host = self.DOCKER_CONTAINER_NAME
+            elif self.args.remote:
+                host = 'localhost'
 
             server = 'https://%s:%s' % (host, port)
             config = self._get_config(server)
@@ -164,11 +170,12 @@ class OpenShiftCloudProvider(CloudProvider):
         self._write_config(config)
 
     def _get_container_address(self):
-        networks = docker_network_inspect(self.args, 'bridge')
+        current_network = get_docker_preferred_network_name(self.args)
+        networks = docker_network_inspect(self.args, current_network)
 
         try:
-            bridge = [network for network in networks if network['Name'] == 'bridge'][0]
+            network = [network for network in networks if network['Name'] == current_network][0]
-            containers = bridge['Containers']
+            containers = network['Containers']
             container = [containers[container] for container in containers if containers[container]['Name'] == self.DOCKER_CONTAINER_NAME][0]
             return re.sub(r'/[0-9]+$', '', container['IPv4Address'])
         except Exception:

test/lib/ansible_test/_internal/cloud/vcenter.py

@@ -23,6 +23,10 @@ from ..docker_util import (
     docker_inspect,
     docker_pull,
     get_docker_container_id,
+    get_docker_hostname,
+    get_docker_container_ip,
+    get_docker_preferred_network_name,
+    is_docker_user_defined_network,
 )
 
 
@@ -92,7 +96,9 @@ class VcenterProvider(CloudProvider):
         """Get any additional options needed when delegating tests to a docker container.
         :rtype: list[str]
         """
-        if self.managed:
+        network = get_docker_preferred_network_name(self.args)
+
+        if self.managed and not is_docker_user_defined_network(network):
             return ['--link', self.DOCKER_SIMULATOR_NAME]
 
         return []
@@ -108,9 +114,6 @@ class VcenterProvider(CloudProvider):
         """Create a vcenter simulator using docker."""
         container_id = get_docker_container_id()
 
-        if container_id:
-            display.info('Running in docker container: %s' % container_id, verbosity=1)
-
         self.container_name = self.DOCKER_SIMULATOR_NAME
 
         results = docker_inspect(self.args, self.container_name)
@@ -150,14 +153,12 @@ class VcenterProvider(CloudProvider):
             vcenter_hostname = self._get_simulator_address()
             display.info('Found vCenter simulator container address: %s' % vcenter_hostname, verbosity=1)
         else:
-            vcenter_hostname = 'localhost'
+            vcenter_hostname = get_docker_hostname()
 
         self._set_cloud_config('vcenter_hostname', vcenter_hostname)
 
     def _get_simulator_address(self):
-        results = docker_inspect(self.args, self.container_name)
+        return get_docker_container_ip(self.args, self.container_name)
-        ipaddress = results[0]['NetworkSettings']['IPAddress']
-        return ipaddress
 
     def _setup_static(self):
         if not os.path.exists(self.config_static_path):

test/lib/ansible_test/_internal/config.py

@@ -90,6 +90,7 @@ class EnvironmentConfig(CommonConfig):
         self.docker_seccomp = args.docker_seccomp if 'docker_seccomp' in args else None  # type: str
         self.docker_memory = args.docker_memory if 'docker_memory' in args else None
         self.docker_terminate = args.docker_terminate if 'docker_terminate' in args else None  # type: str
+        self.docker_network = args.docker_network if 'docker_network' in args else None  # type: str
 
         if self.docker_seccomp is None:
             self.docker_seccomp = get_docker_completion().get(self.docker_raw, {}).get('seccomp', 'default')

test/lib/ansible_test/_internal/delegation.py

@@ -72,6 +72,9 @@ from .docker_util import (
     docker_available,
     docker_network_disconnect,
     get_docker_networks,
+    get_docker_preferred_network_name,
+    get_docker_hostname,
+    is_docker_user_defined_network,
 )
 
 from .cloud import (
@@ -305,14 +308,18 @@ def delegate_docker(args, exclude, require, integration_targets):
             if args.docker_seccomp != 'default':
                 test_options += ['--security-opt', 'seccomp=%s' % args.docker_seccomp]
 
-            if os.path.exists(docker_socket):
+            if get_docker_hostname() != 'localhost' or os.path.exists(docker_socket):
                 test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)]
 
             if httptester_id:
                 test_options += ['--env', 'HTTPTESTER=1']
 
-                for host in HTTPTESTER_HOSTS:
+                network = get_docker_preferred_network_name(args)
-                    test_options += ['--link', '%s:%s' % (httptester_id, host)]
+
+                if not is_docker_user_defined_network(network):
+                    # legacy links are required when using the default bridge network instead of user-defined networks
+                    for host in HTTPTESTER_HOSTS:
+                        test_options += ['--link', '%s:%s' % (httptester_id, host)]
 
             if isinstance(args, IntegrationConfig):
                 cloud_platforms = get_cloud_providers(args)

test/lib/ansible_test/_internal/docker_util.py

@@ -19,6 +19,10 @@ from .util import (
     SubprocessError,
 )
 
+from .http import (
+    urlparse,
+)
+
 from .util_common import (
     run_command,
 )
@@ -37,27 +41,62 @@ def docker_available():
     return find_executable('docker', required=False)
 
 
+def get_docker_hostname():  # type: () -> str
+    """Return the hostname of the Docker service."""
+    try:
+        return get_docker_hostname.hostname
+    except AttributeError:
+        pass
+
+    docker_host = os.environ.get('DOCKER_HOST')
+
+    if docker_host and docker_host.startswith('tcp://'):
+        try:
+            hostname = urlparse(docker_host)[1].split(':')[0]
+            display.info('Detected Docker host: %s' % hostname, verbosity=1)
+        except ValueError:
+            hostname = 'localhost'
+            display.warning('Could not parse DOCKER_HOST environment variable "%s", falling back to localhost.' % docker_host)
+    else:
+        hostname = 'localhost'
+        display.info('Assuming Docker is available on localhost.', verbosity=1)
+
+    get_docker_hostname.hostname = hostname
+
+    return hostname
+
+
 def get_docker_container_id():
     """
     :rtype: str | None
     """
-    path = '/proc/self/cgroup'
+    try:
+        return get_docker_container_id.container_id
+    except AttributeError:
+        pass
 
-    if not os.path.exists(path):
+    path = '/proc/self/cpuset'
-        return None
+    container_id = None
 
-    contents = read_text_file(path)
+    if os.path.exists(path):
+        # File content varies based on the environment:
+        #   No Container: /
+        #   Docker: /docker/c86f3732b5ba3d28bb83b6e14af767ab96abbc52de31313dcb1176a62d91a507
+        #   Azure Pipelines (Docker): /azpl_job/0f2edfed602dd6ec9f2e42c867f4d5ee640ebf4c058e6d3196d4393bb8fd0891
+        #   Podman: /../../../../../..
+        contents = read_text_file(path)
 
-    paths = [line.split(':')[2] for line in contents.splitlines()]
+        cgroup_path, cgroup_name = os.path.split(contents.strip())
-    container_ids = set(path.split('/')[2] for path in paths if path.startswith('/docker/'))
 
-    if not container_ids:
+        if cgroup_path in ('/docker', '/azpl_job'):
-        return None
+            container_id = cgroup_name
 
-    if len(container_ids) == 1:
+    get_docker_container_id.container_id = container_id
-        return container_ids.pop()
 
-    raise ApplicationError('Found multiple container_id candidates: %s\n%s' % (sorted(container_ids), contents))
+    if container_id:
+        display.info('Detected execution in Docker container: %s' % container_id, verbosity=1)
+
+    return container_id
 
 
 def get_docker_container_ip(args, container_id):
@@ -67,10 +106,65 @@ def get_docker_container_ip(args, container_id):
     :rtype: str
     """
     results = docker_inspect(args, container_id)
-    ipaddress = results[0]['NetworkSettings']['IPAddress']
+    network_settings = results[0]['NetworkSettings']
+    networks = network_settings.get('Networks')
+
+    if networks:
+        network_name = get_docker_preferred_network_name(args)
+        ipaddress = networks[network_name]['IPAddress']
+    else:
+        # podman doesn't provide Networks, fall back to using IPAddress
+        ipaddress = network_settings['IPAddress']
+
+    if not ipaddress:
+        raise ApplicationError('Cannot retrieve IP address for container: %s' % container_id)
+
     return ipaddress
 
 
+def get_docker_network_name(args, container_id):  # type: (EnvironmentConfig, str) -> str
+    """
+    Return the network name of the specified container.
+    Raises an exception if zero or more than one network is found.
+    """
+    networks = get_docker_networks(args, container_id)
+
+    if not networks:
+        raise ApplicationError('No network found for Docker container: %s.' % container_id)
+
+    if len(networks) > 1:
+        raise ApplicationError('Found multiple networks for Docker container %s instead of only one: %s' % (container_id, ', '.join(networks)))
+
+    return networks[0]
+
+
+def get_docker_preferred_network_name(args):  # type: (EnvironmentConfig) -> str
+    """
+    Return the preferred network name for use with Docker. The selection logic is:
+    - the network selected by the user with `--docker-network`
+    - the network of the currently running docker container (if any)
+    - the default docker network (returns None)
+    """
+    network = None
+
+    if args.docker_network:
+        network = args.docker_network
+    else:
+        current_container_id = get_docker_container_id()
+
+        if current_container_id:
+            # Make sure any additional containers we launch use the same network as the current container we're running in.
+            # This is needed when ansible-test is running in a container that is not connected to Docker's default network.
+            network = get_docker_network_name(args, current_container_id)
+
+    return network
+
+
+def is_docker_user_defined_network(network):  # type: (str) -> bool
+    """Return True if the network being used is a user-defined network."""
+    return network and network != 'bridge'
+
+
 def get_docker_networks(args, container_id):
     """
     :param args: EnvironmentConfig
@@ -149,6 +243,13 @@ def docker_run(args, image, options, cmd=None):
     if not cmd:
         cmd = []
 
+    network = get_docker_preferred_network_name(args)
+
+    if is_docker_user_defined_network(network):
+        # Only when the network is not the default bridge network.
+        # Using this with the default bridge network results in an error when using --link: links are only supported for user-defined networks
+        options.extend(['--network', network])
+
     for _iteration in range(1, 3):
         try:
             return docker_command(args, ['run'] + options + [image] + cmd, capture=True)

test/lib/ansible_test/_internal/executor.py

@@ -90,6 +90,9 @@ from .docker_util import (
     docker_rm,
     get_docker_container_id,
     get_docker_container_ip,
+    get_docker_hostname,
+    get_docker_preferred_network_name,
+    is_docker_user_defined_network,
 )
 
 from .ansible_util import (
@@ -1255,9 +1258,7 @@ def start_httptester(args):
 
     container_id = get_docker_container_id()
 
-    if container_id:
+    if not container_id:
-        display.info('Running in docker container: %s' % container_id, verbosity=1)
-    else:
         for item in ports:
             item['localhost'] = get_available_port()
 
@@ -1269,7 +1270,7 @@ def start_httptester(args):
         container_host = get_docker_container_ip(args, httptester_id)
         display.info('Found httptester container address: %s' % container_host, verbosity=1)
     else:
-        container_host = 'localhost'
+        container_host = get_docker_hostname()
 
     ssh_options = []
 
@@ -1293,6 +1294,13 @@ def run_httptester(args, ports=None):
         for localhost_port, container_port in ports.items():
             options += ['-p', '%d:%d' % (localhost_port, container_port)]
 
+    network = get_docker_preferred_network_name(args)
+
+    if is_docker_user_defined_network(network):
+        # network-scoped aliases are only supported for containers in user defined networks
+        for alias in HTTPTESTER_HOSTS:
+            options.extend(['--network-alias', alias])
+
     httptester_id = docker_run(args, args.httptester, options=options)[0]
 
     if args.explain: