[stable-2.10] Add macOS 11 to CI (#72622) (#73180)

* [stable-2.10] Add macOS 11 to CI (#72622)

* Fix connection_paramiko_ssh test for macOS 11
* Update Azure Pipelines config
* Add changelog
(cherry picked from commit a7e834071c)

Co-authored-by: Sam Doran <sdoran@redhat.com>

* Prefer venv for tests

* Update pip integration test to use venv on py3.

(cherry picked from commit 456e9b7a33)

Co-authored-by: Matt Clay <matt@mystile.com>

.azure-pipelines/azure-pipelines.yml

@@ -87,8 +87,8 @@ stages:
       - template: templates/matrix.yml
         parameters:
           targets:
-            - name: macOS 10.15
+            - name: macOS 11.1
-              test: macos/10.15
+              test: macos/11.1
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.2

changelogs/fragments/ci-add-macos-11.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - add macOS 11.1 as a remote target (https://github.com/ansible/ansible/pull/72622)

test/integration/targets/collections_runtime_pythonpath/runme.sh

@@ -5,7 +5,7 @@ set -eux -o pipefail
 
 export PIP_DISABLE_PIP_VERSION_CHECK=1
 
-
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 

test/integration/targets/delegate_to/runme.sh

@@ -63,6 +63,7 @@ ansible-playbook has_hostvars.yml -i inventory -v "$@"
 
 # test ansible_x_interpreter
 # python
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 (
 cd "${OUTPUT_DIR}"/venv/bin

test/integration/targets/filter_urls/runme.sh

@@ -6,6 +6,7 @@ export ANSIBLE_ROLES_PATH=../
 
 ansible-playbook runme.yml "$@"
 
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 # This is necessary for installing Jinja 2.6. We need this because Jinja 2.6

test/integration/targets/groupby_filter/runme.sh

@@ -2,6 +2,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 pip install -U jinja2==2.9.4

test/integration/targets/lookup_password/runme.sh

@@ -2,6 +2,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 # Requirements have to be installed prior to running ansible-playbook

test/integration/targets/old_style_cache_plugins/runme.sh

@@ -2,6 +2,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 # Run test if dependencies are installed

test/integration/targets/pip/tasks/main.yml

@@ -1,20 +1,30 @@
 # Current pip unconditionally uses md5.
 # We can re-enable if pip switches to a different hash or allows us to not check md5.
 
-- name: find virtualenv command
+- name: Python 2
-  command: "which virtualenv virtualenv-{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
+  when: ansible_python.version.major == 2
-  register: command
+  block:
-  ignore_errors: true
+    - name: find virtualenv command
+      command: "which virtualenv virtualenv-{{ ansible_python.version.major }}.{{ ansible_python.version.minor }}"
+      register: command
+      ignore_errors: true
 
-- name: is virtualenv available to python -m
+    - name: is virtualenv available to python -m
-  command: '{{ ansible_python_interpreter }} -m virtualenv'
+      command: '{{ ansible_python_interpreter }} -m virtualenv'
-  register: python_m
+      register: python_m
-  when: not command.stdout_lines
+      when: not command.stdout_lines
-  failed_when: python_m.rc != 2
+      failed_when: python_m.rc != 2
 
-- name: remember selected virtualenv command
+    - name: remember selected virtualenv command
-  set_fact:
+      set_fact:
-    virtualenv: "{{ command.stdout_lines[0] if command is successful else ansible_python_interpreter ~ ' -m virtualenv' }}"
+        virtualenv: "{{ command.stdout_lines[0] if command is successful else ansible_python_interpreter ~ ' -m virtualenv' }}"
+
+- name: Python 3+
+  when: ansible_python.version.major > 2
+  block:
+    - name: remember selected virtualenv command
+      set_fact:
+        virtualenv: "{{ ansible_python_interpreter ~ ' -m venv' }}"
 
 - block:
     - name: install git, needed for repo installs

test/integration/targets/pip/tasks/pip.yml

@@ -521,7 +521,7 @@
 ### test virtualenv_command begin ###
 
 - name: Test virtualenv command with arguments
-  when: "ansible_system == 'Linux'"
+  when: ansible_python.version.major == 2
   block:
     - name: make sure the virtualenv does not exist
       file:
@@ -533,7 +533,7 @@
       pip:
         name: "{{ pip_test_package }}"
         virtualenv: "{{ output_dir }}/pipenv"
-        virtualenv_command: "virtualenv --verbose"
+        virtualenv_command: "{{ command.stdout_lines[0] | basename }} --verbose"
         state: present
       register: version13
 

test/integration/targets/setup_paramiko/install.yml

@@ -13,5 +13,6 @@
       with_first_found:
         - "install-{{ ansible_distribution }}-{{ ansible_distribution_major_version }}-python-{{ ansible_python.version.major }}.yml"
         - "install-{{ ansible_os_family }}-{{ ansible_distribution_major_version }}-python-{{ ansible_python.version.major }}.yml"
+        - "install-{{ ansible_os_family }}-python-{{ ansible_python.version.major }}.yml"
         - "install-python-{{ ansible_python.version.major }}.yml"
         - "install-fail.yml"

test/integration/targets/setup_paramiko/setup.sh

@@ -3,6 +3,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh  # for pip installs, if needed, otherwise unused
 ansible-playbook ../setup_paramiko/install.yml -i ../setup_paramiko/inventory "$@"
 trap 'ansible-playbook ../setup_paramiko/uninstall.yml -i ../setup_paramiko/inventory "$@"' EXIT

test/integration/targets/setup_paramiko/uninstall.yml

@@ -10,6 +10,7 @@
           with_first_found:
             - "uninstall-{{ ansible_distribution }}-{{ ansible_distribution_major_version }}-python-{{ ansible_python.version.major }}.yml"
             - "uninstall-{{ ansible_os_family }}-{{ ansible_distribution_major_version }}-python-{{ ansible_python.version.major }}.yml"
+            - "uninstall-{{ ansible_os_family }}-python-{{ ansible_python.version.major }}.yml"
             - "uninstall-{{ ansible_pkg_mgr }}-python-{{ ansible_python.version.major }}.yml"
             - "uninstall-{{ ansible_pkg_mgr }}.yml"
             - "uninstall-fail.yml"

test/integration/targets/template_jinja2_latest/runme.sh

@@ -2,6 +2,7 @@
 
 set -eux
 
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 pip install -U -r requirements.txt

test/integration/targets/vault/runme.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
 set -euvx
+
+export ANSIBLE_TEST_PREFER_VENV=1
 source virtualenv.sh
 
 
@@ -521,4 +523,4 @@ ansible-playbook -i ../../inventory -v "$@" --vault-password-file vault-password
 # Ensure we don't leave unencrypted temp files dangling
 ansible-playbook -v "$@" --vault-password-file vault-password test_dangling_temp.yml
 
-ansible-playbook "$@" --vault-password-file vault-password single_vault_as_string.yml
+ansible-playbook "$@" --vault-password-file vault-password single_vault_as_string.yml

test/lib/ansible_test/_data/completion/remote.txt

@@ -2,6 +2,7 @@ freebsd/11.1 python=2.7,3.6 python_dir=/usr/local/bin
 freebsd/12.1 python=3.6,2.7 python_dir=/usr/local/bin
 osx/10.11 python=2.7 python_dir=/usr/local/bin
 macos/10.15 python=3.8 python_dir=/usr/local/bin
+macos/11.1 python=3.9 python_dir=/usr/local/bin
 rhel/7.6 python=2.7
 rhel/7.8 python=2.7
 rhel/7.9 python=2.7