Overhaul ansible-test SSH key management. (#73451)

* Pass remote.sh to shell over stdin.
* Pass docker.sh to shell over stdin.
* Standardize SSH key management.
* Update docker containers.

changelogs/fragments/ansible-test-ssh-key-management.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Unified SSH key management for all instances created with the ``--remote`` or ``--docker`` options.

test/lib/ansible_test/_data/completion/docker.txt

@@ -1,15 +1,15 @@
-default name=quay.io/ansible/default-test-container:2.11.0 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=collection
-default name=quay.io/ansible/ansible-core-test-container:1.9.0 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=ansible-core
-alpine3 name=quay.io/ansible/alpine3-test-container:1.29.0 python=3.8
-centos6 name=quay.io/ansible/centos6-test-container:1.30.0 python=2.6 seccomp=unconfined
-centos7 name=quay.io/ansible/centos7-test-container:1.29.0 python=2.7 seccomp=unconfined
-centos8 name=quay.io/ansible/centos8-test-container:1.29.0 python=3.6 seccomp=unconfined
-fedora30 name=quay.io/ansible/fedora30-test-container:1.17.0 python=3.7
-fedora31 name=quay.io/ansible/fedora31-test-container:1.17.0 python=3.7
-fedora32 name=quay.io/ansible/fedora32-test-container:1.29.0 python=3.8
-fedora33 name=quay.io/ansible/fedora33-test-container:1.29.0 python=3.9
-opensuse15py2 name=quay.io/ansible/opensuse15py2-test-container:1.29.0 python=2.7
-opensuse15 name=quay.io/ansible/opensuse15-test-container:1.29.0 python=3.6
-ubuntu1604 name=quay.io/ansible/ubuntu1604-test-container:1.29.0 python=2.7 seccomp=unconfined
-ubuntu1804 name=quay.io/ansible/ubuntu1804-test-container:1.29.0 python=3.6 seccomp=unconfined
-ubuntu2004 name=quay.io/ansible/ubuntu2004-test-container:1.29.0 python=3.8 seccomp=unconfined
+default name=quay.io/ansible/default-test-container:3.0.0 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=collection
+default name=quay.io/ansible/ansible-core-test-container:3.0.0 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=ansible-core
+alpine3 name=quay.io/ansible/alpine3-test-container:2.0.1 python=3.8
+centos6 name=quay.io/ansible/centos6-test-container:2.0.1 python=2.6 seccomp=unconfined
+centos7 name=quay.io/ansible/centos7-test-container:2.0.1 python=2.7 seccomp=unconfined
+centos8 name=quay.io/ansible/centos8-test-container:2.0.1 python=3.6 seccomp=unconfined
+fedora30 name=quay.io/ansible/fedora30-test-container:2.0.1 python=3.7
+fedora31 name=quay.io/ansible/fedora31-test-container:2.0.1 python=3.7
+fedora32 name=quay.io/ansible/fedora32-test-container:2.0.1 python=3.8
+fedora33 name=quay.io/ansible/fedora33-test-container:2.0.1 python=3.9
+opensuse15py2 name=quay.io/ansible/opensuse15py2-test-container:2.0.1 python=2.7
+opensuse15 name=quay.io/ansible/opensuse15-test-container:2.0.1 python=3.6
+ubuntu1604 name=quay.io/ansible/ubuntu1604-test-container:2.0.1 python=2.7 seccomp=unconfined
+ubuntu1804 name=quay.io/ansible/ubuntu1804-test-container:2.0.1 python=3.6 seccomp=unconfined
+ubuntu2004 name=quay.io/ansible/ubuntu2004-test-container:2.0.1 python=3.8 seccomp=unconfined

test/lib/ansible_test/_data/setup/remote.sh

@@ -2,9 +2,10 @@
 
 set -eu
 
-platform="$1"
-platform_version="$2"
-python_version="$3"
+platform=#{platform}
+platform_version=#{platform_version}
+python_version=#{python_version}
+
 python_interpreter="python${python_version}"
 
 cd ~/
@@ -164,24 +165,6 @@ elif [ "${platform}" = "aix" ]; then
     done
 fi
 
-# Generate our ssh key and add it to our authorized_keys file.
-# We also need to add localhost's server keys to known_hosts.
-
-if [ ! -f "${HOME}/.ssh/id_rsa.pub" ]; then
-    ssh-keygen -m PEM -q -t rsa -N '' -f "${HOME}/.ssh/id_rsa"
-    # newer ssh-keygen PEM output (such as on RHEL 8.1) is not recognized by paramiko
-    touch "${HOME}/.ssh/id_rsa.new"
-    chmod 0600 "${HOME}/.ssh/id_rsa.new"
-    sed 's/\(BEGIN\|END\) PRIVATE KEY/\1 RSA PRIVATE KEY/' "${HOME}/.ssh/id_rsa" > "${HOME}/.ssh/id_rsa.new"
-    mv "${HOME}/.ssh/id_rsa.new" "${HOME}/.ssh/id_rsa"
-    cat "${HOME}/.ssh/id_rsa.pub" >> "${HOME}/.ssh/authorized_keys"
-    chmod 0600 "${HOME}/.ssh/authorized_keys"
-    for key in /etc/ssh/ssh_host_*_key.pub; do
-        pk=$(cat "${key}")
-        echo "localhost ${pk}" >> "${HOME}/.ssh/known_hosts"
-    done
-fi
-
 # Improve prompts on remote host for interactive use.
 # shellcheck disable=SC1117
 cat << EOF > ~/.bashrc

test/lib/ansible_test/_data/setup/ssh-keys.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Configure SSH keys.
+
+ssh_public_key=#{ssh_public_key}
+ssh_private_key=#{ssh_private_key}
+ssh_key_type=#{ssh_key_type}
+
+ssh_path="${HOME}/.ssh"
+private_key_path="${ssh_path}/id_${ssh_key_type}"
+
+if [ ! -f "${private_key_path}" ]; then
+    # write public/private ssh key pair
+    public_key_path="${private_key_path}.pub"
+
+    # shellcheck disable=SC2174
+    mkdir -m 0700 -p "${ssh_path}"
+    touch "${public_key_path}" "${private_key_path}"
+    chmod 0600 "${public_key_path}" "${private_key_path}"
+    echo "${ssh_public_key}" > "${public_key_path}"
+    echo "${ssh_private_key}" > "${private_key_path}"
+
+    # add public key to authorized_keys
+    authoried_keys_path="${HOME}/.ssh/authorized_keys"
+
+    # the existing file is overwritten to avoid conflicts (ex: RHEL on EC2 blocks root login)
+    cat "${public_key_path}" > "${authoried_keys_path}"
+    chmod 0600 "${authoried_keys_path}"
+
+    # add localhost's server keys to known_hosts
+    known_hosts_path="${HOME}/.ssh/known_hosts"
+
+    for key in /etc/ssh/ssh_host_*_key.pub; do
+        echo "localhost $(cat "${key}")" >> "${known_hosts_path}"
+    done
+fi

test/lib/ansible_test/_internal/core_ci.py

@@ -490,8 +490,9 @@ class CoreHttpError(HttpError):
 
 class SshKey:
     """Container for SSH key used to connect to remote instances."""
-    KEY_NAME = 'id_rsa'
-    PUB_NAME = 'id_rsa.pub'
+    KEY_TYPE = 'rsa'  # RSA is used to maintain compatibility with paramiko and EC2
+    KEY_NAME = 'id_%s' % KEY_TYPE
+    PUB_NAME = '%s.pub' % KEY_NAME
 
     def __init__(self, args):
         """
@@ -519,8 +520,10 @@ class SshKey:
 
         if args.explain:
             self.pub_contents = None
+            self.key_contents = None
         else:
             self.pub_contents = read_text_file(self.pub).strip()
+            self.key_contents = read_text_file(self.key).strip()
 
     def get_in_tree_key_pair_paths(self):  # type: () -> t.Optional[t.Tuple[str, str]]
         """Return the ansible-test SSH key pair paths from the content tree."""
@@ -562,7 +565,7 @@ class SshKey:
             make_dirs(os.path.dirname(key))
 
         if not os.path.isfile(key) or not os.path.isfile(pub):
-            run_command(args, ['ssh-keygen', '-m', 'PEM', '-q', '-t', 'rsa', '-N', '', '-f', key])
+            run_command(args, ['ssh-keygen', '-m', 'PEM', '-q', '-t', self.KEY_TYPE, '-N', '', '-f', key])
 
             # newer ssh-keygen PEM output (such as on RHEL 8.1) is not recognized by paramiko
             key_contents = read_text_file(key)

test/lib/ansible_test/_internal/delegation.py

@@ -11,6 +11,7 @@ from . import types as t
 
 from .io import (
     make_dirs,
+    read_text_file,
 )
 
 from .executor import (
@@ -36,11 +37,13 @@ from .config import (
 
 from .core_ci import (
     AnsibleCoreCI,
+    SshKey,
 )
 
 from .manage_ci import (
     ManagePosixCI,
     ManageWindowsCI,
+    get_ssh_key_setup,
 )
 
 from .util import (
@@ -334,9 +337,16 @@ def delegate_docker(args, exclude, require, integration_targets):
             else:
                 test_id = test_id.strip()
 
+            setup_sh = read_text_file(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'docker.sh'))
+
+            ssh_keys_sh = get_ssh_key_setup(SshKey(args))
+
+            setup_sh += ssh_keys_sh
+            shell = setup_sh.splitlines()[0][2:]
+
+            docker_exec(args, test_id, [shell], data=setup_sh)
+
             # write temporary files to /root since /tmp isn't ready immediately on container start
-            docker_put(args, test_id, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'docker.sh'), '/root/docker.sh')
-            docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
             docker_put(args, test_id, local_source_fd.name, '/root/test.tgz')
             docker_exec(args, test_id, ['tar', 'oxzf', '/root/test.tgz', '-C', '/root'])
 

test/lib/ansible_test/_internal/docker_util.py

@@ -377,7 +377,7 @@ def docker_network_inspect(args, network):
             raise ex
 
 
-def docker_exec(args, container_id, cmd, options=None, capture=False, stdin=None, stdout=None):
+def docker_exec(args, container_id, cmd, options=None, capture=False, stdin=None, stdout=None, data=None):
     """
     :type args: EnvironmentConfig
     :type container_id: str
@@ -386,12 +386,16 @@ def docker_exec(args, container_id, cmd, options=None, capture=False, stdin=None
     :type capture: bool
     :type stdin: BinaryIO | None
     :type stdout: BinaryIO | None
+    :type data: str | None
     :rtype: str | None, str | None
     """
     if not options:
         options = []
 
-    return docker_command(args, ['exec'] + options + [container_id] + cmd, capture=capture, stdin=stdin, stdout=stdout)
+    if data:
+        options.append('-i')
+
+    return docker_command(args, ['exec'] + options + [container_id] + cmd, capture=capture, stdin=stdin, stdout=stdout, data=data)
 
 
 def docker_info(args):

test/lib/ansible_test/_internal/manage_ci.py

@@ -8,6 +8,10 @@ import time
 
 from . import types as t
 
+from .io import (
+    read_text_file,
+)
+
 from .util import (
     SubprocessError,
     ApplicationError,
@@ -20,10 +24,12 @@ from .util_common import (
     intercept_command,
     get_network_completion,
     run_command,
+    ShellScriptTemplate,
 )
 
 from .core_ci import (
     AnsibleCoreCI,
+    SshKey,
 )
 
 from .ansible_util import (
@@ -268,8 +274,19 @@ class ManagePosixCI:
         """Configure remote host for testing.
         :type python_version: str
         """
-        self.upload(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'remote.sh'), '/tmp')
-        self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s %s' % (self.core_ci.platform, self.core_ci.version, python_version))
+        template = ShellScriptTemplate(read_text_file(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'remote.sh')))
+        setup_sh = template.substitute(
+            platform=self.core_ci.platform,
+            platform_version=self.core_ci.version,
+            python_version=python_version,
+        )
+
+        ssh_keys_sh = get_ssh_key_setup(self.core_ci.ssh_key)
+
+        setup_sh += ssh_keys_sh
+        shell = setup_sh.splitlines()[0][2:]
+
+        self.ssh(shell, data=setup_sh)
 
     def upload_source(self):
         """Upload and extract source."""
@@ -302,11 +319,12 @@ class ManagePosixCI:
         """
         self.scp(local, '%s@%s:%s' % (self.core_ci.connection.username, self.core_ci.connection.hostname, remote))
 
-    def ssh(self, command, options=None, capture=False):
+    def ssh(self, command, options=None, capture=False, data=None):
         """
         :type command: str | list[str]
         :type options: list[str] | None
         :type capture: bool
+        :type data: str | None
         :rtype: str | None, str | None
         """
         if not options:
@@ -316,12 +334,18 @@ class ManagePosixCI:
             command = ' '.join(cmd_quote(c) for c in command)
 
         command = cmd_quote(command) if self.become else command
+
+        options.append('-q')
+
+        if not data:
+            options.append('-tt')
+
         return run_command(self.core_ci.args,
-                           ['ssh', '-tt', '-q'] + self.ssh_args +
+                           ['ssh'] + self.ssh_args +
                            options +
                            ['-p', str(self.core_ci.connection.port),
                             '%s@%s' % (self.core_ci.connection.username, self.core_ci.connection.hostname)] +
-                           self.become + [command], capture=capture)
+                           self.become + [command], capture=capture, data=data)
 
     def scp(self, src, dst):
         """
@@ -340,6 +364,19 @@ class ManagePosixCI:
         raise ApplicationError('Failed transfer: %s -> %s' % (src, dst))
 
 
+def get_ssh_key_setup(ssh_key):  # type: (SshKey) -> str
+    """Generate and return a script to configure SSH keys on a host."""
+    template = ShellScriptTemplate(read_text_file(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'ssh-keys.sh')))
+
+    ssh_keys_sh = template.substitute(
+        ssh_public_key=ssh_key.pub_contents,
+        ssh_private_key=ssh_key.key_contents,
+        ssh_key_type=ssh_key.KEY_TYPE,
+    )
+
+    return ssh_keys_sh
+
+
 def get_network_settings(args, platform, version):  # type: (NetworkIntegrationConfig, str, str) -> NetworkPlatformSettings
     """Returns settings for the given network platform and version."""
     platform_version = '%s/%s' % (platform, version)

test/lib/ansible_test/_internal/util_common.py

@@ -5,6 +5,7 @@ __metaclass__ = type
 import atexit
 import contextlib
 import os
+import re
 import shutil
 import sys
 import tempfile
@@ -29,6 +30,7 @@ from .util import (
     read_lines_without_comments,
     ANSIBLE_TEST_DATA_ROOT,
     ApplicationError,
+    cmd_quote,
 )
 
 from .io import (
@@ -49,6 +51,19 @@ REMOTE_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
 NETWORK_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
 
 
+class ShellScriptTemplate:
+    """A simple substition template for shell scripts."""
+    def __init__(self, template):  # type: (str) -> None
+        self.template = template
+
+    def substitute(self, **kwargs):
+        """Return a string templated with the given arguments."""
+        kvp = dict((k, cmd_quote(v)) for k, v in kwargs.items())
+        pattern = re.compile(r'#{(?P<name>[^}]+)}')
+        value = pattern.sub(lambda match: kvp[match.group('name')], self.template)
+        return value
+
+
 class ResultType:
     """Test result type."""
     BOT = None  # type: ResultType