cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Prepare ansible-test for collections support.

Browse source 
Another round of changes to prepare ansible-test for supporting collections to help keep later PRs a manageable size.
This commit is contained in:
Matt Clay 2019-07-15 16:20:03 -07:00
parent f81238012b
commit 4255ee8e5a
11 changed files with 113 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
test/runner/lib/delegation.py
View file

@ -7,9 +7,6 @@ import re
import sys
import tempfile
import lib.pytar
import lib.thread
from lib.executor import (
SUPPORTED_PYTHON_VERSIONS,
HTTPTESTER_HOSTS,
@ -49,6 +46,7 @@ from lib.util import (
from lib.util_common import (
run_command,
INSTALL_ROOT,
)
from lib.docker_util import (
@ -71,6 +69,10 @@ from lib.target import (
IntegrationTarget,
)
from lib.payload import (
create_payload,
)
def check_delegation_args(args):
"""
@ -163,7 +165,7 @@ def delegate_tox(args, exclude, require, integration_targets):
tox.append('--')
cmd = generate_command(args, None, os.path.abspath('bin/ansible-test'), options, exclude, require)
cmd = generate_command(args, None, INSTALL_ROOT, INSTALL_ROOT, options, exclude, require)
if not args.python:
cmd += ['--python', version]
@ -224,7 +226,11 @@ def delegate_docker(args, exclude, require, integration_targets):
}
python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw)
cmd = generate_command(args, python_interpreter, '/root/ansible/bin/ansible-test', options, exclude, require)
install_root = '/root/ansible'
content_root = install_root
cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require)
if isinstance(args, TestConfig):
if args.coverage and not args.coverage_label:
@ -243,13 +249,7 @@ def delegate_docker(args, exclude, require, integration_targets):
with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
try:
if not args.explain:
if args.docker_keep_git:
tar_filter = lib.pytar.AllowGitTarFilter()
else:
tar_filter = lib.pytar.DefaultTarFilter()
lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter)
create_payload(args, local_source_fd.name)
if use_httptester:
httptester_id = run_httptester(args)
@ -296,7 +296,7 @@ def delegate_docker(args, exclude, require, integration_targets):
test_id = test_id.strip()
# write temporary files to /root since /tmp isn't ready immediately on container start
docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh')
docker_put(args, test_id, os.path.join(INSTALL_ROOT, 'test/runner/setup/docker.sh'), '/root/docker.sh')
docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz')
docker_exec(args, test_id, ['mkdir', '/root/ansible'])
@ -310,16 +310,16 @@ def delegate_docker(args, exclude, require, integration_targets):
# also disconnect from the network once requirements have been installed
if isinstance(args, UnitsConfig):
writable_dirs = [
'/root/ansible/.pytest_cache',
os.path.join(content_root, '.pytest_cache'),
]
docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs)
docker_exec(args, test_id, ['chmod', '777'] + writable_dirs)
docker_exec(args, test_id, ['find', '/root/ansible/test/results/', '-type', 'd', '-exec', 'chmod', '777', '{}', '+'])
docker_exec(args, test_id, ['find', os.path.join(content_root, 'test/results/'), '-type', 'd', '-exec', 'chmod', '777', '{}', '+'])
docker_exec(args, test_id, ['chmod', '755', '/root'])
docker_exec(args, test_id, ['chmod', '644', '/root/ansible/%s' % args.metadata_path])
docker_exec(args, test_id, ['chmod', '644', os.path.join(content_root, args.metadata_path)])
docker_exec(args, test_id, ['useradd', 'pytest', '--create-home'])
@ -338,7 +338,7 @@ def delegate_docker(args, exclude, require, integration_targets):
docker_exec(args, test_id, cmd, options=cmd_options)
finally:
with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd:
docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results'])
docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', os.path.join(content_root, 'test'), 'results'])
docker_get(args, test_id, '/root/results.tgz', local_result_fd.name)
run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test'])
finally:
@ -377,6 +377,7 @@ def delegate_remote(args, exclude, require, integration_targets):
httptester_id = None
ssh_options = []
content_root = None
try:
core_ci.start()
@ -399,7 +400,11 @@ def delegate_remote(args, exclude, require, integration_targets):
}
python_interpreter = get_python_interpreter(args, get_remote_completion(), args.remote)
cmd = generate_command(args, python_interpreter, 'ansible/bin/ansible-test', options, exclude, require)
install_root = 'ansible'
content_root = install_root
cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require)
if httptester_id:
cmd += ['--inject-httptester']
@ -440,8 +445,8 @@ def delegate_remote(args, exclude, require, integration_targets):
if args.raw:
download = False
if download:
manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results')
if download and content_root:
manage.ssh('rm -rf /tmp/results && cp -a %s/test/results /tmp/results && chmod -R a+r /tmp/results' % content_root)
manage.download('/tmp/results', 'test')
finally:
if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success):
@ -451,11 +456,12 @@ def delegate_remote(args, exclude, require, integration_targets):
docker_rm(args, httptester_id)
def generate_command(args, python_interpreter, path, options, exclude, require):
def generate_command(args, python_interpreter, install_root, content_root, options, exclude, require):
"""
:type args: EnvironmentConfig
:type python_interpreter: str | None
:type path: str
:type install_root: str
:type content_root: str
:type options: dict[str, int]
:type exclude: list[str]
:type require: list[str]
@ -463,7 +469,7 @@ def generate_command(args, python_interpreter, path, options, exclude, require):
"""
options['--color'] = 1
cmd = [path]
cmd = [os.path.join(install_root, 'bin/ansible-test')]
if python_interpreter:
cmd = [python_interpreter] + cmd
@ -472,7 +478,14 @@ def generate_command(args, python_interpreter, path, options, exclude, require):
# This is only needed because ansible-test relies on Python's file system encoding.
# Environments that do not have the locale configured are thus unable to work with unicode file paths.
# Examples include FreeBSD and some Linux containers.
cmd = ['/usr/bin/env', 'LC_ALL=en_US.UTF-8'] + cmd
env_vars = dict(
LC_ALL='en_US.UTF-8',
ANSIBLE_TEST_CONTENT_ROOT=content_root,
)
env_args = ['%s=%s' % (key, env_vars[key]) for key in sorted(env_vars)]
cmd = ['/usr/bin/env'] + env_args + cmd
cmd += list(filter_options(args, sys.argv[1:], options, exclude, require))
cmd += ['--color', 'yes' if args.color else 'no']

8
test/runner/lib/executor.py
View file

@ -20,7 +20,6 @@ import shutil
import lib.types as t
import lib.pytar
import lib.thread
from lib.core_ci import (
@ -279,6 +278,9 @@ def generate_egg_info(args):
"""
:type args: EnvironmentConfig
"""
if not os.path.exists(os.path.join(INSTALL_ROOT, 'setup.py')):
return
if os.path.isdir(os.path.join(INSTALL_ROOT, 'lib/ansible.egg-info')):
return
@ -292,8 +294,8 @@ def generate_pip_install(pip, command, packages=None):
:type packages: list[str] | None
:rtype: list[str] | None
"""
constraints = 'test/runner/requirements/constraints.txt'
requirements = 'test/runner/requirements/%s.txt' % command
constraints = os.path.join(INSTALL_ROOT, 'test/runner/requirements/constraints.txt')
requirements = os.path.join(INSTALL_ROOT, 'test/runner/requirements/%s.txt' % command)
options = []

14
test/runner/lib/import_analysis.py
View file

@ -122,10 +122,14 @@ def enumerate_module_utils():
module_utils = []
base_path = 'lib/ansible/module_utils'
paths = []
for root, _dir_names, file_names in os.walk(base_path):
for file_name in file_names:
path = os.path.join(root, file_name)
name, ext = os.path.splitext(file_name)
paths.append(os.path.join(root, file_name))
for path in paths:
name, ext = os.path.splitext(path)
if path == 'lib/ansible/module_utils/__init__.py':
continue
@ -133,10 +137,10 @@ def enumerate_module_utils():
if ext != '.py':
continue
if name == '__init__':
module_util = root
if name.endswith('/__init__'):
module_util = os.path.dirname(name)
else:
module_util = os.path.join(root, name)
module_util = name
module_utils.append(module_util[4:].replace('/', '.'))

12
test/runner/lib/manage_ci.py
View file

@ -6,8 +6,6 @@ import os
import tempfile
import time
import lib.pytar
from lib.util import (
SubprocessError,
ApplicationError,
@ -17,6 +15,7 @@ from lib.util import (
from lib.util_common import (
intercept_command,
run_command,
INSTALL_ROOT,
)
from lib.core_ci import (
@ -31,6 +30,10 @@ from lib.config import (
ShellConfig,
)
from lib.payload import (
create_payload,
)
class ManageWindowsCI:
"""Manage access to a Windows instance provided by Ansible Core CI."""
@ -237,7 +240,7 @@ class ManagePosixCI:
"""Configure remote host for testing.
:type python_version: str
"""
self.upload('test/runner/setup/remote.sh', '/tmp')
self.upload(os.path.join(INSTALL_ROOT, 'test/runner/setup/remote.sh'), '/tmp')
self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s' % (self.core_ci.platform, python_version))
def upload_source(self):
@ -246,8 +249,7 @@ class ManagePosixCI:
remote_source_dir = '/tmp'
remote_source_path = os.path.join(remote_source_dir, os.path.basename(local_source_fd.name))
if not self.core_ci.args.explain:
lib.pytar.create_tarfile(local_source_fd.name, '.', lib.pytar.DefaultTarFilter())
create_payload(self.core_ci.args, local_source_fd.name)
self.upload(local_source_fd.name, remote_source_dir)
self.ssh('rm -rf ~/ansible && mkdir ~/ansible && cd ~/ansible && tar oxzf %s' % remote_source_path)

27
test/runner/lib/payload.py Normal file
View file

@ -0,0 +1,27 @@
"""Payload management for sending Ansible files and test content to other systems (VMs, containers)."""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from lib.config import (
CommonConfig,
EnvironmentConfig,
)
from lib.pytar import (
AllowGitTarFilter,
create_tarfile,
DefaultTarFilter,
)
def create_payload(args, dst_path): # type: (CommonConfig, str) -> None
"""Create a payload for delegation."""
if args.explain:
return
if isinstance(args, EnvironmentConfig) and args.docker_keep_git:
tar_filter = AllowGitTarFilter()
else:
tar_filter = DefaultTarFilter()
create_tarfile(dst_path, '.', tar_filter)

6
test/runner/lib/sanity/import.py
View file

@ -20,11 +20,13 @@ from lib.util import (
read_lines_without_comments,
parse_to_list_of_dict,
make_dirs,
is_subdir,
)
from lib.util_common import (
intercept_command,
run_command,
INSTALL_ROOT,
)
from lib.ansible_util import (
@ -58,7 +60,7 @@ class ImportTest(SanityMultipleVersion):
i.path
for i in targets.include
if os.path.splitext(i.path)[1] == '.py' and
(i.path.startswith('lib/ansible/modules/') or i.path.startswith('lib/ansible/module_utils/')) and
(is_subdir(i.path, 'lib/ansible/modules/') or is_subdir(i.path, 'lib/ansible/module_utils/')) and
i.path not in skip_paths_set
)
@ -85,7 +87,7 @@ class ImportTest(SanityMultipleVersion):
# add the importer to our virtual environment so it can be accessed through the coverage injector
importer_path = os.path.join(virtual_environment_bin, 'importer.py')
if not args.explain:
os.symlink(os.path.abspath('test/sanity/import/importer.py'), importer_path)
os.symlink(os.path.abspath(os.path.join(INSTALL_ROOT, 'test/sanity/import/importer.py')), importer_path)
# create a minimal python library
python_path = os.path.abspath('test/runner/.tox/import/lib')

5
test/runner/lib/sanity/pslint.py
View file

@ -25,6 +25,7 @@ from lib.util import (
from lib.util_common import (
run_command,
INSTALL_ROOT,
)
from lib.config import (
@ -84,8 +85,8 @@ class PslintTest(SanitySingleVersion):
# Make sure requirements are installed before running sanity checks
cmds = [
['test/runner/requirements/sanity.ps1'],
['test/sanity/pslint/pslint.ps1'] + paths
[os.path.join(INSTALL_ROOT, 'test/runner/requirements/sanity.ps1')],
[os.path.join(INSTALL_ROOT, 'test/sanity/pslint/pslint.ps1')] + paths
]
stdout = ''

10
test/runner/lib/sanity/pylint.py
View file

@ -24,6 +24,7 @@ from lib.util import (
read_lines_without_comments,
ConfigParser,
INSTALL_ROOT,
is_subdir,
)
from lib.util_common import (
@ -113,16 +114,17 @@ class PylintTest(SanitySingleVersion):
skip_paths_set = set(skip_paths)
paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set)
paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/')) and i.path not in skip_paths_set)
module_paths = [p.split(os.path.sep) for p in paths if p.startswith('lib/ansible/modules/')]
module_paths = [p.split(os.path.sep) for p in paths if is_subdir(p, 'lib/ansible/modules/')]
module_dirs = sorted(set([p[3] for p in module_paths if len(p) > 4]))
large_module_group_threshold = 500
large_module_groups = [key for key, value in
itertools.groupby(module_paths, lambda p: p[3] if len(p) > 4 else '') if len(list(value)) > large_module_group_threshold]
large_module_group_paths = [p.split(os.path.sep) for p in paths if any(p.startswith('lib/ansible/modules/%s/' % g) for g in large_module_groups)]
large_module_group_paths = [p.split(os.path.sep) for p in paths
if any(is_subdir(p, os.path.join('lib/ansible/modules/', g)) for g in large_module_groups)]
large_module_group_dirs = sorted(set([os.path.sep.join(p[3:5]) for p in large_module_group_paths if len(p) > 5]))
contexts = []
@ -148,7 +150,7 @@ class PylintTest(SanitySingleVersion):
:type path_to_filter: str
:rtype: bool
"""
return path_to_filter.startswith(path_filter)
return is_subdir(path_to_filter, path_filter)
return context_filter

6
test/runner/lib/sanity/sanity_docs.py
View file

@ -4,6 +4,10 @@ __metaclass__ = type
import os
from lib.util import (
INSTALL_ROOT,
)
from lib.sanity import (
SanitySingleVersion,
SanityMessage,
@ -26,7 +30,7 @@ class SanityDocsTest(SanitySingleVersion):
:type targets: SanityTargets
:rtype: TestResult
"""
sanity_dir = 'docs/docsite/rst/dev_guide/testing/sanity'
sanity_dir = os.path.join(INSTALL_ROOT, 'docs/docsite/rst/dev_guide/testing/sanity')
sanity_docs = set(part[0] for part in (os.path.splitext(name) for name in os.listdir(sanity_dir)) if part[1] == '.rst')
sanity_tests = set(sanity_test.name for sanity_test in sanity_get_tests())

3
test/runner/lib/test.py
View file

@ -8,6 +8,7 @@ import os
from lib.util import (
display,
make_dirs,
)
from lib.config import (
@ -332,6 +333,8 @@ class TestFailure(TestResult):
if args.explain:
return
make_dirs(os.path.dirname(path))
with open(path, 'w') as bot_fd:
json.dump(bot_data, bot_fd, indent=4, sort_keys=True)
bot_fd.write('\n')

1
test/runner/lib/types.py
View file

@ -5,6 +5,7 @@ __metaclass__ = type
try:
from typing import (
Any,
Callable,
Dict,
FrozenSet,
Iterable,