Increase python version coverage for tests. (#24762)
* Improve ansible-test inventory handling. * Fix python 3 re-raise of exception from thread. * Fix python 3 encoding for windows-integration. * Run network tests on multiple python versions. * Run windows tests on multiple python versions. * Support Shippable delegation using --tox. * Skip vyos_command on python 3 tests until fixed. * Add python 3 filtering to local and tox. * Fix tests to support back to back runs. * Temporarily test networking with python 2.7 only. Running the tests back to back causes intermittent test failures which need to be addressed before we can test multiple versions in a single test run.
This commit is contained in:
parent
c99c3b2b5d
commit
5babe2daea
13 changed files with 140 additions and 51 deletions
14
test/integration/inventory.networking.template
Normal file
14
test/integration/inventory.networking.template
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
# This is the configuration template for ansible-test network-integration tests.
|
||||||
|
#
|
||||||
|
# You do not need this template if you are:
|
||||||
|
#
|
||||||
|
# 1) Running integration tests without using ansible-test.
|
||||||
|
# 2) Using the `--platform` option to provision temporary network instances on EC2.
|
||||||
|
#
|
||||||
|
# If you do not want to use the automatically provisioned temporary network instances,
|
||||||
|
# fill in the @VAR placeholders below and save this file without the .template extension.
|
||||||
|
#
|
||||||
|
# NOTE: Automatic provisioning of network instances on EC2 requires an ansible-core-ci API key.
|
||||||
|
|
||||||
|
[@OS]
|
||||||
|
@NAME ansible_connection="local" ansible_host=@HOST ansible_network_os="@OS" ansible_user="@USER" ansible_ssh_private_key_file="@KEY_FILE"
|
|
@ -1,25 +1,28 @@
|
||||||
[windows_hosts]
|
# This is the configuration template for ansible-test windows-integration tests.
|
||||||
|
# It can also be used with the legacy `make` based method of running tests.
|
||||||
|
#
|
||||||
|
# You do not need this template if you are:
|
||||||
|
#
|
||||||
|
# 1) Using the `--windows` option to provision temporary Windows instances on EC2.
|
||||||
|
#
|
||||||
|
# If you do not want to use the automatically provisioned temporary Windows instances,
|
||||||
|
# fill in the @VAR placeholders below and save this file without the .template extension.
|
||||||
|
#
|
||||||
|
# NOTE: Automatic provisioning of Windows instances on EC2 requires an ansible-core-ci API key.
|
||||||
|
#
|
||||||
|
# REMINDER: Standard ports for winrm are 5985 (HTTP) and 5986 (HTTPS).
|
||||||
|
|
||||||
|
[windows]
|
||||||
|
@NAME ansible_host=@HOST ansible_user=@USER ansible_password=@PASSWORD ansible_port=@PORT
|
||||||
|
|
||||||
|
[windows:vars]
|
||||||
|
ansible_connection=winrm
|
||||||
|
ansible_winrm_server_cert_validation=ignore
|
||||||
|
|
||||||
|
# support winrm connection tests (temporary solution, does not support testing enable/disable of pipelining)
|
||||||
|
[winrm:children]
|
||||||
windows
|
windows
|
||||||
testhost_binary_modules
|
|
||||||
|
|
||||||
[windows_hosts:vars]
|
# support winrm binary module tests (temporary solution)
|
||||||
ansible_connection=winrm
|
[testhost_binary_modules:children]
|
||||||
ansible_host=@ansible_host
|
windows
|
||||||
ansible_user=@ansible_user
|
|
||||||
ansible_password=@ansible_password
|
|
||||||
# HTTPS uses 5986, HTTP uses 5985
|
|
||||||
ansible_port=5986
|
|
||||||
ansible_winrm_server_cert_validation=ignore
|
|
||||||
|
|
||||||
[winrm]
|
|
||||||
winrm-pipelining ansible_ssh_pipelining=true
|
|
||||||
winrm-no-pipelining ansible_ssh_pipelining=false
|
|
||||||
|
|
||||||
[winrm:vars]
|
|
||||||
ansible_connection=winrm
|
|
||||||
ansible_host=@ansible_host
|
|
||||||
ansible_user=@ansible_user
|
|
||||||
ansible_password=@ansible_password
|
|
||||||
# HTTPS uses 5986, HTTP uses 5985
|
|
||||||
ansible_port=5986
|
|
||||||
ansible_winrm_server_cert_validation=ignore
|
|
||||||
|
|
|
@ -1 +1,2 @@
|
||||||
network/ci
|
network/ci
|
||||||
|
skip/python3
|
||||||
|
|
|
@ -35,8 +35,6 @@
|
||||||
- show hardware cpu detail
|
- show hardware cpu detail
|
||||||
- show hardware mem
|
- show hardware mem
|
||||||
- show license
|
- show license
|
||||||
- show log
|
|
||||||
- show system boot-messages
|
|
||||||
provider: "{{ cli }}"
|
provider: "{{ cli }}"
|
||||||
register: result
|
register: result
|
||||||
|
|
||||||
|
|
|
@ -1 +1,2 @@
|
||||||
windows/ci/group1
|
windows/ci/group1
|
||||||
|
windows/ci/minimal
|
||||||
|
|
|
@ -265,8 +265,8 @@ class AnsibleCoreCI(object):
|
||||||
display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1)
|
display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1)
|
||||||
|
|
||||||
if self.platform == 'windows':
|
if self.platform == 'windows':
|
||||||
with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd:
|
with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'rb') as winrm_config_fd:
|
||||||
winrm_config = winrm_config_fd.read()
|
winrm_config = winrm_config_fd.read().decode('utf-8')
|
||||||
else:
|
else:
|
||||||
winrm_config = None
|
winrm_config = None
|
||||||
|
|
||||||
|
|
|
@ -35,6 +35,8 @@ from lib.util import (
|
||||||
ApplicationError,
|
ApplicationError,
|
||||||
EnvironmentConfig,
|
EnvironmentConfig,
|
||||||
run_command,
|
run_command,
|
||||||
|
common_environment,
|
||||||
|
pass_vars,
|
||||||
)
|
)
|
||||||
|
|
||||||
from lib.docker_util import (
|
from lib.docker_util import (
|
||||||
|
@ -129,7 +131,18 @@ def delegate_tox(args, exclude, require):
|
||||||
if args.coverage and not args.coverage_label:
|
if args.coverage and not args.coverage_label:
|
||||||
cmd += ['--coverage-label', 'tox-%s' % version]
|
cmd += ['--coverage-label', 'tox-%s' % version]
|
||||||
|
|
||||||
run_command(args, tox + cmd)
|
env = common_environment()
|
||||||
|
|
||||||
|
# temporary solution to permit ansible-test delegated to tox to provision remote resources
|
||||||
|
optional = (
|
||||||
|
'SHIPPABLE',
|
||||||
|
'SHIPPABLE_BUILD_ID',
|
||||||
|
'SHIPPABLE_JOB_NUMBER',
|
||||||
|
)
|
||||||
|
|
||||||
|
env.update(pass_vars(required=[], optional=optional))
|
||||||
|
|
||||||
|
run_command(args, tox + cmd, env=env)
|
||||||
|
|
||||||
|
|
||||||
def delegate_docker(args, exclude, require):
|
def delegate_docker(args, exclude, require):
|
||||||
|
|
|
@ -245,6 +245,11 @@ def command_network_integration(args):
|
||||||
"""
|
"""
|
||||||
:type args: NetworkIntegrationConfig
|
:type args: NetworkIntegrationConfig
|
||||||
"""
|
"""
|
||||||
|
filename = 'test/integration/inventory.networking'
|
||||||
|
|
||||||
|
if not args.explain and not args.platform and not os.path.isfile(filename):
|
||||||
|
raise ApplicationError('Use the --platform option or provide an inventory file (see %s.template).' % filename)
|
||||||
|
|
||||||
internal_targets = command_integration_filter(args, walk_network_integration_targets())
|
internal_targets = command_integration_filter(args, walk_network_integration_targets())
|
||||||
platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
|
platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
|
||||||
|
|
||||||
|
@ -276,8 +281,6 @@ def command_network_integration(args):
|
||||||
remotes = [instance.wait_for_result() for instance in instances]
|
remotes = [instance.wait_for_result() for instance in instances]
|
||||||
inventory = network_inventory(remotes)
|
inventory = network_inventory(remotes)
|
||||||
|
|
||||||
filename = 'test/integration/inventory.networking'
|
|
||||||
|
|
||||||
display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
|
display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
|
||||||
|
|
||||||
if not args.explain:
|
if not args.explain:
|
||||||
|
@ -349,6 +352,11 @@ def command_windows_integration(args):
|
||||||
"""
|
"""
|
||||||
:type args: WindowsIntegrationConfig
|
:type args: WindowsIntegrationConfig
|
||||||
"""
|
"""
|
||||||
|
filename = 'test/integration/inventory.winrm'
|
||||||
|
|
||||||
|
if not args.explain and not args.windows and not os.path.isfile(filename):
|
||||||
|
raise ApplicationError('Use the --windows option or provide an inventory file (see %s.template).' % filename)
|
||||||
|
|
||||||
internal_targets = command_integration_filter(args, walk_windows_integration_targets())
|
internal_targets = command_integration_filter(args, walk_windows_integration_targets())
|
||||||
|
|
||||||
if args.windows:
|
if args.windows:
|
||||||
|
@ -368,8 +376,6 @@ def command_windows_integration(args):
|
||||||
remotes = [instance.wait_for_result() for instance in instances]
|
remotes = [instance.wait_for_result() for instance in instances]
|
||||||
inventory = windows_inventory(remotes)
|
inventory = windows_inventory(remotes)
|
||||||
|
|
||||||
filename = 'test/integration/inventory.winrm'
|
|
||||||
|
|
||||||
display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
|
display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
|
||||||
|
|
||||||
if not args.explain:
|
if not args.explain:
|
||||||
|
@ -1111,6 +1117,14 @@ def get_integration_local_filter(args, targets):
|
||||||
display.warning('Excluding tests marked "%s" which require --allow-destructive to run locally: %s'
|
display.warning('Excluding tests marked "%s" which require --allow-destructive to run locally: %s'
|
||||||
% (skip.rstrip('/'), ', '.join(skipped)))
|
% (skip.rstrip('/'), ', '.join(skipped)))
|
||||||
|
|
||||||
|
if args.python_version.startswith('3'):
|
||||||
|
skip = 'skip/python3/'
|
||||||
|
skipped = [target.name for target in targets if skip in target.aliases]
|
||||||
|
if skipped:
|
||||||
|
exclude.append(skip)
|
||||||
|
display.warning('Excluding tests marked "%s" which are not yet supported on python 3: %s'
|
||||||
|
% (skip.rstrip('/'), ', '.join(skipped)))
|
||||||
|
|
||||||
return exclude
|
return exclude
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -43,7 +43,7 @@ class WrappedThread(threading.Thread):
|
||||||
result, exception = self._result.get()
|
result, exception = self._result.get()
|
||||||
if exception:
|
if exception:
|
||||||
if sys.version_info[0] > 2:
|
if sys.version_info[0] > 2:
|
||||||
raise exception[0](exception[1]).with_traceback(exception[2])
|
raise exception[1].with_traceback(exception[2])
|
||||||
# noinspection PyRedundantParentheses
|
# noinspection PyRedundantParentheses
|
||||||
exec('raise exception[0], exception[1], exception[2]') # pylint: disable=locally-disabled, exec-used
|
exec('raise exception[0], exception[1], exception[2]') # pylint: disable=locally-disabled, exec-used
|
||||||
return result
|
return result
|
||||||
|
|
|
@ -208,7 +208,7 @@ def common_environment():
|
||||||
return env
|
return env
|
||||||
|
|
||||||
|
|
||||||
def pass_vars(required=None, optional=None):
|
def pass_vars(required, optional):
|
||||||
"""
|
"""
|
||||||
:type required: collections.Iterable[str]
|
:type required: collections.Iterable[str]
|
||||||
:type optional: collections.Iterable[str]
|
:type optional: collections.Iterable[str]
|
||||||
|
|
|
@ -5,5 +5,5 @@ minversion = 2.5.0
|
||||||
[testenv]
|
[testenv]
|
||||||
changedir = {toxinidir}/../../
|
changedir = {toxinidir}/../../
|
||||||
commands = {posargs}
|
commands = {posargs}
|
||||||
passenv = HOME
|
passenv = HOME SHIPPABLE*
|
||||||
args_are_paths = False
|
args_are_paths = False
|
||||||
|
|
|
@ -12,22 +12,38 @@ fi
|
||||||
|
|
||||||
target="network/ci/"
|
target="network/ci/"
|
||||||
|
|
||||||
|
# python versions to test in order
|
||||||
|
# all versions run full tests
|
||||||
|
python_versions=(
|
||||||
|
2.7
|
||||||
|
)
|
||||||
|
|
||||||
if [ -s /tmp/network.txt ]; then
|
if [ -s /tmp/network.txt ]; then
|
||||||
echo "Detected changes requiring integration tests specific to networking:"
|
echo "Detected changes requiring integration tests specific to networking:"
|
||||||
cat /tmp/network.txt
|
cat /tmp/network.txt
|
||||||
|
|
||||||
echo "Running network integration tests for multiple platforms concurrently."
|
echo "Running network integration tests for multiple platforms concurrently."
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
platforms=(
|
||||||
ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
|
--platform vyos/1.1.0
|
||||||
--platform vyos/1.1.0 \
|
--platform ios/csr1000v
|
||||||
--platform ios/csr1000v \
|
)
|
||||||
|
|
||||||
else
|
else
|
||||||
echo "No changes requiring integration tests specific to networking were detected."
|
echo "No changes requiring integration tests specific to networking were detected."
|
||||||
echo "Running network integration tests for a single platform only."
|
echo "Running network integration tests for a single platform only."
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
platforms=(
|
||||||
ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
|
|
||||||
--platform vyos/1.1.0
|
--platform vyos/1.1.0
|
||||||
|
)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
retry.py pip install tox --disable-pip-version-check
|
||||||
|
|
||||||
|
for version in "${python_versions[@]}"; do
|
||||||
|
# clean up between test runs until we switch from --tox to --docker
|
||||||
|
rm -rf ~/.ansible/{cp,pc,tmp}/
|
||||||
|
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
ansible-test network-integration --color -v --retry-on-error "${target}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
|
||||||
|
"${platforms[@]}"
|
||||||
|
done
|
||||||
|
|
|
@ -7,6 +7,15 @@ IFS='/:' read -ra args <<< "${TEST}"
|
||||||
|
|
||||||
job="${args[1]}"
|
job="${args[1]}"
|
||||||
|
|
||||||
|
# python versions to test in order
|
||||||
|
# python 2.7 runs full tests while other versions run minimal tests
|
||||||
|
python_versions=(
|
||||||
|
2.6
|
||||||
|
3.5
|
||||||
|
3.6
|
||||||
|
2.7
|
||||||
|
)
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
|
ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
|
||||||
|
|
||||||
|
@ -23,20 +32,40 @@ if [ -s /tmp/windows.txt ]; then
|
||||||
|
|
||||||
target="windows/ci/"
|
target="windows/ci/"
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
platforms=(
|
||||||
ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
|
--windows 2008-SP2
|
||||||
--windows 2008-SP2 \
|
--windows 2008-R2_SP1
|
||||||
--windows 2008-R2_SP1 \
|
--windows 2012-RTM
|
||||||
--windows 2012-RTM \
|
--windows 2012-R2_RTM
|
||||||
--windows 2012-R2_RTM \
|
)
|
||||||
|
|
||||||
else
|
else
|
||||||
echo "No changes requiring integration tests specific to Windows were detected."
|
echo "No changes requiring integration tests specific to Windows were detected."
|
||||||
echo "Running Windows integration tests for a single version only."
|
echo "Running Windows integration tests for a single version only."
|
||||||
|
|
||||||
target="windows/ci/group${job}/"
|
target="windows/ci/group${job}/"
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
platforms=(
|
||||||
ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
|
|
||||||
--windows 2012-R2_RTM
|
--windows 2012-R2_RTM
|
||||||
|
)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
retry.py pip install tox --disable-pip-version-check
|
||||||
|
|
||||||
|
for version in "${python_versions[@]}"; do
|
||||||
|
# clean up between test runs until we switch from --tox to --docker
|
||||||
|
rm -rf ~/.ansible/{cp,pc,tmp}/
|
||||||
|
|
||||||
|
if [ "${job}" == "1" ] || [ "${version}" == "2.7" ]; then
|
||||||
|
if [ "${version}" == "2.7" ]; then
|
||||||
|
# full tests for python 2.7
|
||||||
|
ci="${target}"
|
||||||
|
else
|
||||||
|
# minimal tests for other python versions
|
||||||
|
ci="windows/ci/minimal/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
ansible-test windows-integration --color -v --retry-on-error "${ci}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
|
||||||
|
"${platforms[@]}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
Loading…
Reference in a new issue