cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Reorganize util code in ansible-test.

Browse source 
Code in util.py that depends on CommonConfig is now in util_common.py.
This commit is contained in:
Matt Clay 2019-07-10 22:00:34 -07:00
parent 17c88724ab
commit 86ad4c99ba
25 changed files with 351 additions and 264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
test/runner/lib/ansible_util.py
View file

@ -13,11 +13,14 @@ from lib.util import (
common_environment,
display,
find_python,
run_command,
ApplicationError,
INSTALL_ROOT,
)
from lib.util_common import (
run_command,
)
from lib.config import (
IntegrationConfig,
EnvironmentConfig,

5
test/runner/lib/changes.py
View file

@ -9,10 +9,13 @@ from lib.util import (
ApplicationError,
SubprocessError,
MissingEnvironmentVariable,
CommonConfig,
display,
)
from lib.util_common import (
CommonConfig,
)
from lib.http import (
HttpClient,
urlencode,

5
test/runner/lib/cloud/tower.py
View file

@ -8,11 +8,14 @@ from lib.util import (
display,
ApplicationError,
is_shippable,
run_command,
SubprocessError,
ConfigParser,
)
from lib.util_common import (
run_command,
)
from lib.cloud import (
CloudProvider,
CloudEnvironment,

5
test/runner/lib/config.py
View file

@ -8,7 +8,6 @@ import sys
import lib.types as t
from lib.util import (
CommonConfig,
is_shippable,
docker_qualify_image,
find_python,
@ -17,6 +16,10 @@ from lib.util import (
ApplicationError,
)
from lib.util_common import (
CommonConfig,
)
from lib.metadata import (
Metadata,
)

5
test/runner/lib/core_ci.py
View file

@ -18,12 +18,15 @@ from lib.http import (
from lib.util import (
ApplicationError,
run_command,
make_dirs,
display,
is_shippable,
)
from lib.util_common import (
run_command,
)
from lib.config import (
EnvironmentConfig,
)

5
test/runner/lib/cover.py
View file

@ -13,10 +13,13 @@ from lib.target import (
from lib.util import (
display,
ApplicationError,
run_command,
common_environment,
)
from lib.util_common import (
run_command,
)
from lib.config import (
CoverageConfig,
CoverageReportConfig,

5
test/runner/lib/delegation.py
View file

@ -42,12 +42,15 @@ from lib.manage_ci import (
from lib.util import (
ApplicationError,
run_command,
common_environment,
pass_vars,
display,
)
from lib.util_common import (
run_command,
)
from lib.docker_util import (
docker_exec,
docker_get,

10
test/runner/lib/docker_util.py
View file

@ -6,16 +6,16 @@ import json
import os
import time
from lib.executor import (
SubprocessError,
)
from lib.util import (
ApplicationError,
run_command,
common_environment,
display,
find_executable,
SubprocessError,
)
from lib.util_common import (
run_command,
)
from lib.config import (

11
test/runner/lib/executor.py
View file

@ -44,26 +44,29 @@ from lib.util import (
ApplicationError,
SubprocessError,
display,
run_command,
intercept_command,
remove_tree,
make_dirs,
is_shippable,
is_binary_file,
find_executable,
raw_command,
get_python_path,
get_available_port,
generate_pip_command,
find_python,
get_docker_completion,
get_remote_completion,
named_temporary_file,
COVERAGE_OUTPUT_PATH,
cmd_quote,
INSTALL_ROOT,
)
from lib.util_common import (
get_python_path,
intercept_command,
named_temporary_file,
run_command,
)
from lib.docker_util import (
docker_pull,
docker_run,

7
test/runner/lib/http.py
View file

@ -22,13 +22,16 @@ except ImportError:
from urllib.parse import urlparse, urlunparse, parse_qs # pylint: disable=locally-disabled, ungrouped-imports
from lib.util import (
CommonConfig,
ApplicationError,
run_command,
SubprocessError,
display,
)
from lib.util_common import (
CommonConfig,
run_command,
)
class HttpClient(object):
"""Make HTTP requests via curl."""

5
test/runner/lib/integration/__init__.py
View file

@ -23,7 +23,6 @@ from lib.util import (
ApplicationError,
display,
make_dirs,
named_temporary_file,
COVERAGE_CONFIG_PATH,
COVERAGE_OUTPUT_PATH,
MODE_DIRECTORY,
@ -32,6 +31,10 @@ from lib.util import (
INSTALL_ROOT,
)
from lib.util_common import (
named_temporary_file,
)
from lib.cache import (
CommonCache,
)

7
test/runner/lib/manage_ci.py
View file

@ -11,11 +11,14 @@ import lib.pytar
from lib.util import (
SubprocessError,
ApplicationError,
run_command,
intercept_command,
cmd_quote,
)
from lib.util_common import (
intercept_command,
run_command,
)
from lib.core_ci import (
AnsibleCoreCI,
)

5
test/runner/lib/sanity/__init__.py
View file

@ -12,7 +12,6 @@ from lib.util import (
ApplicationError,
SubprocessError,
display,
run_command,
import_plugins,
load_plugins,
parse_to_list_of_dict,
@ -22,6 +21,10 @@ from lib.util import (
read_lines_without_comments,
)
from lib.util_common import (
run_command,
)
from lib.ansible_util import (
ansible_environment,
check_pyyaml,

5
test/runner/lib/sanity/ansible_doc.py
View file

@ -16,10 +16,13 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
display,
intercept_command,
read_lines_without_comments,
)
from lib.util_common import (
intercept_command,
)
from lib.ansible_util import (
ansible_environment,
)

5
test/runner/lib/sanity/compile.py
View file

@ -13,7 +13,6 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
display,
find_python,
read_lines_without_comments,
@ -21,6 +20,10 @@ from lib.util import (
INSTALL_ROOT,
)
from lib.util_common import (
run_command,
)
from lib.config import (
SanityConfig,
)

7
test/runner/lib/sanity/import.py
View file

@ -13,8 +13,6 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
intercept_command,
remove_tree,
display,
find_python,
@ -23,6 +21,11 @@ from lib.util import (
make_dirs,
)
from lib.util_common import (
intercept_command,
run_command,
)
from lib.ansible_util import (
ansible_environment,
)

5
test/runner/lib/sanity/pep8.py
View file

@ -14,12 +14,15 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
display,
run_command,
read_lines_without_comments,
parse_to_list_of_dict,
INSTALL_ROOT,
)
from lib.util_common import (
run_command,
)
from lib.config import (
SanityConfig,
)

5
test/runner/lib/sanity/pslint.py
View file

@ -16,11 +16,14 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
find_executable,
read_lines_without_comments,
)
from lib.util_common import (
run_command,
)
from lib.config import (
SanityConfig,
)

6
test/runner/lib/sanity/pylint.py
View file

@ -19,13 +19,17 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
display,
read_lines_without_comments,
ConfigParser,
INSTALL_ROOT,
)
from lib.util_common import (
intercept_command,
run_command,
)
from lib.executor import (
SUPPORTED_PYTHON_VERSIONS,
)

5
test/runner/lib/sanity/rstcheck.py
View file

@ -13,13 +13,16 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
parse_to_list_of_dict,
display,
read_lines_without_comments,
INSTALL_ROOT,
)
from lib.util_common import (
run_command,
)
from lib.config import (
SanityConfig,
)

5
test/runner/lib/sanity/shellcheck.py
View file

@ -18,10 +18,13 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
read_lines_without_comments,
)
from lib.util_common import (
run_command,
)
from lib.config import (
SanityConfig,
)

5
test/runner/lib/sanity/validate_modules.py
View file

@ -16,11 +16,14 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
display,
run_command,
read_lines_without_comments,
INSTALL_ROOT,
)
from lib.util_common import (
run_command,
)
from lib.ansible_util import (
ansible_environment,
)

5
test/runner/lib/sanity/yamllint.py
View file

@ -14,11 +14,14 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
display,
INSTALL_ROOT,
)
from lib.util_common import (
run_command,
)
from lib.config import (
SanityConfig,
)

231
test/runner/lib/util.py
View file

@ -2,7 +2,6 @@
from __future__ import absolute_import, print_function
import atexit
import contextlib
import errno
import fcntl
@ -17,8 +16,6 @@ import stat
import string
import subprocess
import sys
import tempfile
import textwrap
import time
from struct import unpack, pack
@ -158,127 +155,6 @@ def read_lines_without_comments(path, remove_blank_lines=False, optional=False):
return lines
def get_python_path(args, interpreter):
"""
:type args: TestConfig
:type interpreter: str
:rtype: str
"""
# When the python interpreter is already named "python" its directory can simply be added to the path.
# Using another level of indirection is only required when the interpreter has a different name.
if os.path.basename(interpreter) == 'python':
return os.path.dirname(interpreter)
python_path = PYTHON_PATHS.get(interpreter)
if python_path:
return python_path
prefix = 'python-'
suffix = '-ansible'
root_temp_dir = '/tmp'
if args.explain:
return os.path.join(root_temp_dir, ''.join((prefix, 'temp', suffix)))
python_path = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_temp_dir)
injected_interpreter = os.path.join(python_path, 'python')
# A symlink is faster than the execv wrapper, but isn't compatible with virtual environments.
# Attempt to detect when it is safe to use a symlink by checking the real path of the interpreter.
use_symlink = os.path.dirname(os.path.realpath(interpreter)) == os.path.dirname(interpreter)
if use_symlink:
display.info('Injecting "%s" as a symlink to the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1)
os.symlink(interpreter, injected_interpreter)
else:
display.info('Injecting "%s" as a execv wrapper for the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1)
code = textwrap.dedent('''
#!%s
from __future__ import absolute_import
from os import execv
from sys import argv
python = '%s'
execv(python, [python] + argv[1:])
''' % (interpreter, interpreter)).lstrip()
with open(injected_interpreter, 'w') as python_fd:
python_fd.write(code)
os.chmod(injected_interpreter, MODE_FILE_EXECUTE)
os.chmod(python_path, MODE_DIRECTORY)
if not PYTHON_PATHS:
atexit.register(cleanup_python_paths)
PYTHON_PATHS[interpreter] = python_path
return python_path
def cleanup_python_paths():
"""Clean up all temporary python directories."""
for path in sorted(PYTHON_PATHS.values()):
display.info('Cleaning up temporary python directory: %s' % path, verbosity=2)
shutil.rmtree(path)
def get_coverage_environment(args, target_name, version, temp_path, module_coverage):
"""
:type args: TestConfig
:type target_name: str
:type version: str
:type temp_path: str
:type module_coverage: bool
:rtype: dict[str, str]
"""
if temp_path:
# integration tests (both localhost and the optional testhost)
# config and results are in a temporary directory
coverage_config_base_path = temp_path
coverage_output_base_path = temp_path
else:
# unit tests, sanity tests and other special cases (localhost only)
# config and results are in the source tree
coverage_config_base_path = args.coverage_config_base_path or INSTALL_ROOT
coverage_output_base_path = os.path.abspath(os.path.join('test/results'))
config_file = os.path.join(coverage_config_base_path, COVERAGE_CONFIG_PATH)
coverage_file = os.path.join(coverage_output_base_path, COVERAGE_OUTPUT_PATH, '%s=%s=%s=%s=coverage' % (
args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version))
if args.coverage_check:
# cause the 'coverage' module to be found, but not imported or enabled
coverage_file = ''
# Enable code coverage collection on local Python programs (this does not include Ansible modules).
# Used by the injectors in test/runner/injector/ to support code coverage.
# Used by unit tests in test/units/conftest.py to support code coverage.
# The COVERAGE_FILE variable is also used directly by the 'coverage' module.
env = dict(
COVERAGE_CONF=config_file,
COVERAGE_FILE=coverage_file,
)
if module_coverage:
# Enable code coverage collection on Ansible modules (both local and remote).
# Used by the AnsiballZ wrapper generator in lib/ansible/executor/module_common.py to support code coverage.
env.update(dict(
_ANSIBLE_COVERAGE_CONFIG=config_file,
_ANSIBLE_COVERAGE_OUTPUT=coverage_file,
))
return env
def find_executable(executable, cwd=None, path=None, required=True):
"""
:type executable: str
@ -355,68 +231,6 @@ def generate_pip_command(python):
return [python, '-m', 'pip.__main__']
def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd=None, python_version=None, temp_path=None, module_coverage=True,
virtualenv=None):
"""
:type args: TestConfig
:type cmd: collections.Iterable[str]
:type target_name: str
:type env: dict[str, str]
:type capture: bool
:type data: str | None
:type cwd: str | None
:type python_version: str | None
:type temp_path: str | None
:type module_coverage: bool
:type virtualenv: str | None
:rtype: str | None, str | None
"""
if not env:
env = common_environment()
cmd = list(cmd)
version = python_version or args.python_version
interpreter = virtualenv or find_python(version)
inject_path = os.path.join(INSTALL_ROOT, 'test/runner/injector')
if not virtualenv:
# injection of python into the path is required when not activating a virtualenv
# otherwise scripts may find the wrong interpreter or possibly no interpreter
python_path = get_python_path(args, interpreter)
inject_path = python_path + os.path.pathsep + inject_path
env['PATH'] = inject_path + os.path.pathsep + env['PATH']
env['ANSIBLE_TEST_PYTHON_VERSION'] = version
env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter
if args.coverage:
# add the necessary environment variables to enable code coverage collection
env.update(get_coverage_environment(args, target_name, version, temp_path, module_coverage))
return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd)
def run_command(args, cmd, capture=False, env=None, data=None, cwd=None, always=False, stdin=None, stdout=None,
cmd_verbosity=1, str_errors='strict'):
"""
:type args: CommonConfig
:type cmd: collections.Iterable[str]
:type capture: bool
:type env: dict[str, str] | None
:type data: str | None
:type cwd: str | None
:type always: bool
:type stdin: file | None
:type stdout: file | None
:type cmd_verbosity: int
:type str_errors: str
:rtype: str | None, str | None
"""
explain = args.explain and not always
return raw_command(cmd, capture=capture, env=env, data=data, cwd=cwd, explain=explain, stdin=stdin, stdout=stdout,
cmd_verbosity=cmd_verbosity, str_errors=str_errors)
def raw_command(cmd, capture=False, env=None, data=None, cwd=None, explain=False, stdin=None, stdout=None,
cmd_verbosity=1, str_errors='strict'):
"""
@ -859,28 +673,6 @@ class MissingEnvironmentVariable(ApplicationError):
self.name = name
class CommonConfig(object):
"""Configuration common to all commands."""
def __init__(self, args, command):
"""
:type args: any
:type command: str
"""
self.command = command
self.color = args.color # type: bool
self.explain = args.explain # type: bool
self.verbosity = args.verbosity # type: int
self.debug = args.debug # type: bool
self.truncate = args.truncate # type: int
self.redact = args.redact # type: bool
if is_shippable():
self.redact = True
self.cache = {}
def docker_qualify_image(name):
"""
:type name: str
@ -891,29 +683,6 @@ def docker_qualify_image(name):
return config.get('name', name)
@contextlib.contextmanager
def named_temporary_file(args, prefix, suffix, directory, content):
"""
:param args: CommonConfig
:param prefix: str
:param suffix: str
:param directory: str
:param content: str | bytes | unicode
:rtype: str
"""
if not isinstance(content, bytes):
content = content.encode('utf-8')
if args.explain:
yield os.path.join(directory, '%stemp%s' % (prefix, suffix))
else:
with tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=directory) as tempfile_fd:
tempfile_fd.write(content)
tempfile_fd.flush()
yield tempfile_fd.name
def parse_to_list_of_dict(pattern, value):
"""
:type pattern: str

251
test/runner/lib/util_common.py Normal file
View file

@ -0,0 +1,251 @@
"""Common utility code that depends on CommonConfig."""
from __future__ import absolute_import, print_function
import atexit
import contextlib
import os
import shutil
import tempfile
import textwrap
from lib.util import (
common_environment,
COVERAGE_CONFIG_PATH,
COVERAGE_OUTPUT_PATH,
display,
find_python,
INSTALL_ROOT,
is_shippable,
MODE_DIRECTORY,
MODE_FILE_EXECUTE,
PYTHON_PATHS,
raw_command,
)
class CommonConfig(object):
"""Configuration common to all commands."""
def __init__(self, args, command):
"""
:type args: any
:type command: str
"""
self.command = command
self.color = args.color # type: bool
self.explain = args.explain # type: bool
self.verbosity = args.verbosity # type: int
self.debug = args.debug # type: bool
self.truncate = args.truncate # type: int
self.redact = args.redact # type: bool
if is_shippable():
self.redact = True
self.cache = {}
@contextlib.contextmanager
def named_temporary_file(args, prefix, suffix, directory, content):
"""
:param args: CommonConfig
:param prefix: str
:param suffix: str
:param directory: str
:param content: str | bytes | unicode
:rtype: str
"""
if not isinstance(content, bytes):
content = content.encode('utf-8')
if args.explain:
yield os.path.join(directory, '%stemp%s' % (prefix, suffix))
else:
with tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=directory) as tempfile_fd:
tempfile_fd.write(content)
tempfile_fd.flush()
yield tempfile_fd.name
def get_python_path(args, interpreter):
"""
:type args: TestConfig
:type interpreter: str
:rtype: str
"""
# When the python interpreter is already named "python" its directory can simply be added to the path.
# Using another level of indirection is only required when the interpreter has a different name.
if os.path.basename(interpreter) == 'python':
return os.path.dirname(interpreter)
python_path = PYTHON_PATHS.get(interpreter)
if python_path:
return python_path
prefix = 'python-'
suffix = '-ansible'
root_temp_dir = '/tmp'
if args.explain:
return os.path.join(root_temp_dir, ''.join((prefix, 'temp', suffix)))
python_path = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_temp_dir)
injected_interpreter = os.path.join(python_path, 'python')
# A symlink is faster than the execv wrapper, but isn't compatible with virtual environments.
# Attempt to detect when it is safe to use a symlink by checking the real path of the interpreter.
use_symlink = os.path.dirname(os.path.realpath(interpreter)) == os.path.dirname(interpreter)
if use_symlink:
display.info('Injecting "%s" as a symlink to the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1)
os.symlink(interpreter, injected_interpreter)
else:
display.info('Injecting "%s" as a execv wrapper for the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1)
code = textwrap.dedent('''
#!%s
from __future__ import absolute_import
from os import execv
from sys import argv
python = '%s'
execv(python, [python] + argv[1:])
''' % (interpreter, interpreter)).lstrip()
with open(injected_interpreter, 'w') as python_fd:
python_fd.write(code)
os.chmod(injected_interpreter, MODE_FILE_EXECUTE)
os.chmod(python_path, MODE_DIRECTORY)
if not PYTHON_PATHS:
atexit.register(cleanup_python_paths)
PYTHON_PATHS[interpreter] = python_path
return python_path
def cleanup_python_paths():
"""Clean up all temporary python directories."""
for path in sorted(PYTHON_PATHS.values()):
display.info('Cleaning up temporary python directory: %s' % path, verbosity=2)
shutil.rmtree(path)
def get_coverage_environment(args, target_name, version, temp_path, module_coverage):
"""
:type args: TestConfig
:type target_name: str
:type version: str
:type temp_path: str
:type module_coverage: bool
:rtype: dict[str, str]
"""
if temp_path:
# integration tests (both localhost and the optional testhost)
# config and results are in a temporary directory
coverage_config_base_path = temp_path
coverage_output_base_path = temp_path
else:
# unit tests, sanity tests and other special cases (localhost only)
# config and results are in the source tree
coverage_config_base_path = args.coverage_config_base_path or INSTALL_ROOT
coverage_output_base_path = os.path.abspath(os.path.join('test/results'))
config_file = os.path.join(coverage_config_base_path, COVERAGE_CONFIG_PATH)
coverage_file = os.path.join(coverage_output_base_path, COVERAGE_OUTPUT_PATH, '%s=%s=%s=%s=coverage' % (
args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version))
if args.coverage_check:
# cause the 'coverage' module to be found, but not imported or enabled
coverage_file = ''
# Enable code coverage collection on local Python programs (this does not include Ansible modules).
# Used by the injectors in test/runner/injector/ to support code coverage.
# Used by unit tests in test/units/conftest.py to support code coverage.
# The COVERAGE_FILE variable is also used directly by the 'coverage' module.
env = dict(
COVERAGE_CONF=config_file,
COVERAGE_FILE=coverage_file,
)
if module_coverage:
# Enable code coverage collection on Ansible modules (both local and remote).
# Used by the AnsiballZ wrapper generator in lib/ansible/executor/module_common.py to support code coverage.
env.update(dict(
_ANSIBLE_COVERAGE_CONFIG=config_file,
_ANSIBLE_COVERAGE_OUTPUT=coverage_file,
))
return env
def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd=None, python_version=None, temp_path=None, module_coverage=True,
virtualenv=None):
"""
:type args: TestConfig
:type cmd: collections.Iterable[str]
:type target_name: str
:type env: dict[str, str]
:type capture: bool
:type data: str | None
:type cwd: str | None
:type python_version: str | None
:type temp_path: str | None
:type module_coverage: bool
:type virtualenv: str | None
:rtype: str | None, str | None
"""
if not env:
env = common_environment()
cmd = list(cmd)
version = python_version or args.python_version
interpreter = virtualenv or find_python(version)
inject_path = os.path.join(INSTALL_ROOT, 'test/runner/injector')
if not virtualenv:
# injection of python into the path is required when not activating a virtualenv
# otherwise scripts may find the wrong interpreter or possibly no interpreter
python_path = get_python_path(args, interpreter)
inject_path = python_path + os.path.pathsep + inject_path
env['PATH'] = inject_path + os.path.pathsep + env['PATH']
env['ANSIBLE_TEST_PYTHON_VERSION'] = version
env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter
if args.coverage:
# add the necessary environment variables to enable code coverage collection
env.update(get_coverage_environment(args, target_name, version, temp_path, module_coverage))
return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd)
def run_command(args, cmd, capture=False, env=None, data=None, cwd=None, always=False, stdin=None, stdout=None,
cmd_verbosity=1, str_errors='strict'):
"""
:type args: CommonConfig
:type cmd: collections.Iterable[str]
:type capture: bool
:type env: dict[str, str] | None
:type data: str | None
:type cwd: str | None
:type always: bool
:type stdin: file | None
:type stdout: file | None
:type cmd_verbosity: int
:type str_errors: str
:rtype: str | None, str | None
"""
explain = args.explain and not always
return raw_command(cmd, capture=capture, env=env, data=data, cwd=cwd, explain=explain, stdin=stdin, stdout=stdout,
cmd_verbosity=cmd_verbosity, str_errors=str_errors)