Add FreeBSD to Shippable CI. (#16883)

shippable.yml

@@ -8,6 +8,8 @@ matrix:
   exclude:
     - env: TEST=none
   include:
+    - env: TEST=remote TARGET=all PLATFORM=freebsd VERSION=10.3-STABLE
+
     - env: TEST=remote TARGET=ci_win1 PLATFORM=windows VERSION=2012-R2_RTM
     - env: TEST=remote TARGET=ci_win2 PLATFORM=windows VERSION=2012-R2_RTM
     - env: TEST=remote TARGET=ci_win3 PLATFORM=windows VERSION=2012-R2_RTM

test/integration/Makefile

@@ -98,6 +98,8 @@ endif
 # Otherwise use the test_default inventory group, which runs fewer tests, but should work on any system.
 ifeq ($(container),docker)
 TEST_CONNECTION_FILTER := 'test_docker'
+else ifeq ($(container),freebsd)
+TEST_CONNECTION_FILTER := 'test_freebsd'
 else
 TEST_CONNECTION_FILTER := 'test_default'
 endif

test/integration/inventory.remote.template

@@ -0,0 +1,8 @@
+[posix]
+remote
+
+[posix:vars]
+ansible_connection=ssh
+ansible_host=@ansible_host
+ansible_user=@ansible_user
+ansible_python_interpreter=/usr/local/bin/python2

test/integration/test_connection.inventory

@@ -81,3 +81,8 @@ chroot
 test_default
 ssh
 paramiko_ssh
+
+[test_freebsd:children]
+test_default
+ssh
+paramiko_ssh

test/utils/shippable/ansible-core-ci

@@ -74,6 +74,12 @@ def main():
                               default=None,
                               help='path to ssh public key for authentication')
 
+    start_parser.add_argument('--query',
+                              dest='query',
+                              action='store_true',
+                              default=False,
+                              help='query only, do not start instance')
+
     shippable = start_subparsers.add_parser('shippable', help='start instance for shippable testing')
 
     shippable.add_argument('--run-id',
@@ -164,6 +170,7 @@ def start_instance(args):
             platform=args.platform,
             version=args.version,
             public_key=public_key,
+            query=args.query,
         )
     )
 
@@ -204,6 +211,9 @@ def start_instance(args):
 
     print(args.instance_id)
 
+    if args.query:
+        print_stderr(json.dumps(response.json(), indent=4, sort_keys=True))
+
 
 def stop_instance(args):
     if args.verbose:

test/utils/shippable/remote-integration.sh

@@ -0,0 +1,130 @@
+#!/bin/sh -e
+
+# TODO: add support for other posix environments
+container=freebsd
+build_dir="${HOME}/ansible"
+
+test_target="${TARGET:-}"
+test_flags="${TEST_FLAGS:-}"
+
+# Force ansible color output by default.
+# To disable color force mode use FORCE_COLOR=0
+force_color="${FORCE_COLOR:-1}"
+
+# FIXME: these tests fail
+skip_tags='test_copy,test_template,test_unarchive,test_command_shell,test_sudo,test_become,test_service,test_postgresql,test_mysql_db,test_mysql_user,test_mysql_variables,test_uri,test_get_url'
+
+cd ~/
+
+# ssl certificate errors using fetch, so install curl
+pkg install -y curl
+
+if [ ! -f bootstrap.sh ]; then
+    curl "https://raw.githubusercontent.com/mattclay/ansible-hacking/master/bootstrap.sh" -o bootstrap.sh
+fi
+
+chmod +x bootstrap.sh
+./bootstrap.sh pip -y -q
+
+# tests require these packages
+# TODO: bootstrap.sh should be capable of installing these
+pkg install -y \
+    bash \
+    devel/ruby-gems \
+    mercurial \
+    rsync \
+    ruby \
+    subversion \
+    sudo \
+    zip
+
+# TODO: bootstrap.sh should install these
+pip install \
+    junit-xml \
+    virtualenv
+
+# FIXME: tests assume bash is in /bin/bash
+if [ ! -f /bin/bash ]; then
+    ln -s /usr/local/bin/bash /bin/bash
+fi
+
+# FIXME: tests assume true is in /bin/true
+if [ ! -f /bin/true ]; then
+    ln -s /usr/bin/true /bin/true
+fi
+
+# FIXME: async doesn't work with ansible_python_interpreter, see: https://github.com/ansible/ansible/issues/14101
+if [ ! -f /usr/bin/python ]; then
+    ln -s /usr/local/bin/python /usr/bin/python
+fi
+
+# Tests assume loopback addresses other than 127.0.0.1 will work.
+# Add aliases for loopback addresses used by tests.
+
+for i in 3 4 254; do
+    ifconfig lo0 alias "127.0.0.${i}" up
+done
+
+ifconfig lo0
+
+# Since tests run as root, we also need to be able to ssh to localhost as root.
+sed -i '' 's/^# *PermitRootLogin.*$/PermitRootLogin yes/;' /etc/ssh/sshd_config
+
+# Restart sshd for configuration changes and loopback aliases to work.
+service sshd restart
+
+# Generate our ssh key and add it to our authorized_keys file.
+# We also need to add localhost's server keys to known_hosts.
+
+if [ ! -f "${HOME}/.ssh/id_rsa.pub" ]; then
+    ssh-keygen -q -t rsa -N '' -f "${HOME}/.ssh/id_rsa"
+    cp "${HOME}/.ssh/id_rsa.pub" "${HOME}/.ssh/authorized_keys"
+    for key in /etc/ssh/ssh_host_*_key.pub; do
+        pk=$(cat "${key}")
+        echo "localhost ${pk}" >> "${HOME}/.ssh/known_hosts"
+    done
+fi
+
+if [ -d "${build_dir}" ]; then
+    cd "${build_dir}"
+else
+    git clone "${REPOSITORY_URL:-https://github.com/ansible/ansible.git}" "${build_dir}"
+    cd "${build_dir}"
+
+    if [ "${PULL_REQUEST:-false}" = "false" ]; then
+        git checkout -f "${BRANCH:-devel}" --
+        git reset --hard "${COMMIT:-HEAD}"
+    else
+        git fetch origin "pull/${PULL_REQUEST}/head"
+        git checkout -f FETCH_HEAD
+        git merge "origin/${BRANCH}"
+    fi
+fi
+
+git submodule init
+git submodule sync
+git submodule update
+
+. hacking/env-setup
+
+cd test/integration
+
+# FIXME: these test targets fail
+sed -i '' 's/ blocks / /;' Makefile
+sed -i '' 's/ pull / /;' Makefile
+sed -i '' 's/ test_handlers / /;' Makefile
+sed -i '' 's/ no_log / /;' Makefile
+
+# TODO: support httptester via reverse ssh tunnel
+
+rm -rf "/tmp/shippable"
+mkdir -p "/tmp/shippable/testresults"
+
+# TODO: enable jail test
+# shellcheck disable=SC2086
+JUNIT_OUTPUT_DIR="/tmp/shippable/testresults" \
+    ANSIBLE_FORCE_COLOR="${force_color}" \
+    ANSIBLE_CALLBACK_WHITELIST=junit \
+    TEST_FLAGS="-e ansible_python_interpreter=/usr/local/bin/python2 --skip-tags '${skip_tags}' ${test_flags}" \
+    container="${container}" \
+    gmake ${test_target}

test/utils/shippable/remote.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -eux
 
+set -o pipefail
+
 source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))")
 
 test_flags="${TEST_FLAGS:-}"
@@ -8,50 +10,152 @@ test_version="${VERSION}"
 
 test_target=(${TARGET})
 
+instance_id="${INSTANCE_ID:-}"
+start_instance=
+
+if [ "${instance_id}" == "" ]; then
+    instance_id=$(python -c 'import uuid; print(uuid.uuid4())')
+    start_instance=1
+fi
+
+# Set this to a non-empty value to skip immediate termination of the remote instance after tests finish.
+# The remote instance will still be auto-terminated when the remote time limit is reached.
+keep_instance="${KEEP_INSTANCE:-}"
+
 # Force ansible color output by default.
 # To disable color force mode use FORCE_COLOR=0
 force_color="${FORCE_COLOR:-1}"
 
+if [ "${SHIPPABLE:-}" = "true" ]; then
+    test_auth="shippable"
+else
+    test_auth="remote"
+fi
+
 env
 
-instance_id=$("${source_root}/test/utils/shippable/ansible-core-ci" -v \
-    start shippable "${test_platform}" "${test_version}")
+case "${test_platform}" in
+    "windows")
+        args=""
+        ;;
+    *)
+        ssh_key="${HOME}/.ssh/id_rsa"
+        args="--public-key=${ssh_key}.pub"
+        if [ ! -f "${ssh_key}.pub" ]; then
+            ssh-keygen -q -t rsa -N '' -f "${ssh_key}"
+        fi
+        ;;
+esac
 
-pip install -r "${source_root}/test/utils/shippable/remote-requirements.txt" --upgrade
-pip list
+pre_cleanup=
 
 function cleanup
 {
+    if [ "${pre_cleanup}" != '' ]; then
+        "${pre_cleanup}"
+    fi
+
+    if [ "${keep_instance}" = '' ]; then
         "${source_root}/test/utils/shippable/ansible-core-ci" -v stop "${instance_id}"
+    fi
+
+    echo "instance_id: ${instance_id}"
 }
 
 trap cleanup EXIT INT TERM
 
+if [ ${start_instance} ]; then
+    # shellcheck disable=SC2086
+    "${source_root}/test/utils/shippable/ansible-core-ci" -v \
+        start --id "${instance_id}" "${test_auth}" "${test_platform}" "${test_version}" ${args}
+fi
+
+pip install -r "${source_root}/test/utils/shippable/remote-requirements.txt" --upgrade
+pip list
+
 cd "${source_root}"
 source hacking/env-setup
 cd test/integration
 
+case "${test_platform}" in
+    "windows")
         inventory_template="${source_root}/test/integration/inventory.winrm.template"
         inventory_file="${source_root}/test/integration/inventory.winrm"
+        ping_module="win_ping"
+        ping_host="windows"
+        test_function="test_windows"
+        ;;
+    *)
+        inventory_template="${source_root}/test/integration/inventory.remote.template"
+        inventory_file="${source_root}/test/integration/inventory.remote"
+        ping_module="ping"
+        ping_host="remote"
+        test_function="test_remote"
+        ;;
+esac
 
 "${source_root}/test/utils/shippable/ansible-core-ci" -v \
     get "${instance_id}" \
     --template "${inventory_template}" \
     > "${inventory_file}" \
 
-# hack to make sure windows instance is responding before beginning tests
-n=20
+# hack to make sure instance is responding before beginning tests
+n=60
 for i in $(seq 1 ${n}); do
     echo "Verifying host is responding ($i of $n)"
-    if ANSIBLE_FORCE_COLOR="${force_color}" ansible -m win_ping -i "${inventory_file}" windows; then
+    if \
+        ANSIBLE_SSH_ARGS='' \
+        ANSIBLE_HOST_KEY_CHECKING=False \
+        ANSIBLE_FORCE_COLOR="${force_color}" \
+        ansible -m "${ping_module}" -i "${inventory_file}" "${ping_host}"; then
         break
     fi
-    sleep 3
+    sleep 5
 done
 
+test_windows() {
     JUNIT_OUTPUT_DIR="${source_root}/shippable/testresults" \
         ANSIBLE_FORCE_COLOR="${force_color}" \
         ANSIBLE_CALLBACK_WHITELIST=junit \
         TEST_FLAGS="${test_flags}" \
         LC_ALL=en_US.utf-8 \
         make "${test_target[@]}"
+}
+
+test_remote() {
+    endpoint=$("${source_root}/test/utils/shippable/ansible-core-ci" get \
+        "${instance_id}" \
+        --template <(echo "@ansible_user@@ansible_host"))
+
+(
+cat <<EOF
+env \
+REPOSITORY_URL='${REPOSITORY_URL:-}' \
+PULL_REQUEST='${PULL_REQUEST:-}' \
+BRANCH='${BRANCH:-}' \
+COMMIT='${COMMIT:-}' \
+FORCE_COLOR='${force_color}' \
+TARGET='${test_target[*]}' \
+TEST_FLAGS='${test_flags}' \
+/bin/sh -e /tmp/remote-integration.sh
+EOF
+) > /tmp/remote-script.sh
+
+(
+cat <<EOF
+put "${source_root}/test/utils/shippable/remote-integration.sh" "/tmp/remote-integration.sh"
+put "/tmp/remote-script.sh" "/tmp/remote-script.sh"
+EOF
+) | sftp -b - -o StrictHostKeyChecking=no "${endpoint}"
+
+    pre_cleanup=test_remote_cleanup
+
+    ssh "${endpoint}" \
+        "su -l root -c 'chmod +x /tmp/remote-script.sh; /tmp/remote-script.sh'"
+}
+
+test_remote_cleanup() {
+    scp -r "${endpoint}:/tmp/shippable" "${source_root}"
+}
+
+"${test_function}"