cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix invalid type hints in ansible-test.

Browse source
This commit is contained in:
Matt Clay 2019-07-11 13:03:49 -07:00
parent b91a90a274
commit bc3a599678
18 changed files with 108 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
test/runner/injector/python.py
View file

@ -18,8 +18,10 @@ def main():
args += ['-m', 'coverage.__main__', 'run', '--rcfile', coverage_config]
else:
if sys.version_info >= (3, 4):
# noinspection PyUnresolvedReferences
import importlib.util
# noinspection PyUnresolvedReferences
found = bool(importlib.util.find_spec('coverage'))
else:
import imp

3
test/runner/lib/cli.py
View file

@ -170,6 +170,7 @@ def parse_args():
action='store_true',
help='run ansible commands in debug mode')
# noinspection PyTypeChecker
common.add_argument('--truncate',
dest='truncate',
metavar='COLUMNS',
@ -527,6 +528,7 @@ def parse_args():
action='store_true',
help='dump environment to disk')
# noinspection PyTypeChecker
env.add_argument('--timeout',
type=int,
metavar='MINUTES',
@ -749,6 +751,7 @@ def add_extra_docker_options(parser, integration=True):
action='store_true',
help='run docker container in privileged mode')
# noinspection PyTypeChecker
docker.add_argument('--docker-memory',
help='memory limit for docker in bytes', type=int)

20
test/runner/lib/config.py
View file

@ -79,7 +79,7 @@ class EnvironmentConfig(CommonConfig):
self.python_interpreter = args.python_interpreter
self.delegate = self.tox or self.docker or self.remote
self.delegate_args = [] # type: list[str]
self.delegate_args = [] # type: t.List[str]
if self.delegate:
self.requirements = True
@ -118,9 +118,9 @@ class TestConfig(EnvironmentConfig):
self.coverage_label = args.coverage_label # type: str
self.coverage_check = args.coverage_check # type: bool
self.coverage_config_base_path = None # type: t.Optional[str]
self.include = args.include or [] # type: list [str]
self.exclude = args.exclude or [] # type: list [str]
self.require = args.require or [] # type: list [str]
self.include = args.include or [] # type: t.List[str]
self.exclude = args.exclude or [] # type: t.List[str]
self.require = args.require or [] # type: t.List[str]
self.changed = args.changed # type: bool
self.tracked = args.tracked # type: bool
@ -129,7 +129,7 @@ class TestConfig(EnvironmentConfig):
self.staged = args.staged # type: bool
self.unstaged = args.unstaged # type: bool
self.changed_from = args.changed_from # type: str
self.changed_path = args.changed_path # type: list [str]
self.changed_path = args.changed_path # type: t.List[str]
self.lint = args.lint if 'lint' in args else False # type: bool
self.junit = args.junit if 'junit' in args else False # type: bool
@ -164,8 +164,8 @@ class SanityConfig(TestConfig):
"""
super(SanityConfig, self).__init__(args, 'sanity')
self.test = args.test # type: list [str]
self.skip_test = args.skip_test # type: list [str]
self.test = args.test # type: t.List[str]
self.skip_test = args.skip_test # type: t.List[str]
self.list_tests = args.list_tests # type: bool
self.allow_disabled = args.allow_disabled # type: bool
@ -232,7 +232,7 @@ class WindowsIntegrationConfig(IntegrationConfig):
"""
super(WindowsIntegrationConfig, self).__init__(args, 'windows-integration')
self.windows = args.windows # type: list [str]
self.windows = args.windows # type: t.List[str]
if self.windows:
self.allow_destructive = True
@ -247,7 +247,7 @@ class NetworkIntegrationConfig(IntegrationConfig):
"""
super(NetworkIntegrationConfig, self).__init__(args, 'network-integration')
self.platform = args.platform # type: list [str]
self.platform = args.platform # type: t.List[str]
self.inventory = args.inventory # type: str
self.testcase = args.testcase # type: str
@ -278,7 +278,7 @@ class CoverageConfig(EnvironmentConfig):
"""
super(CoverageConfig, self).__init__(args, 'coverage')
self.group_by = frozenset(args.group_by) if 'group_by' in args and args.group_by else set() # type: frozenset [str]
self.group_by = frozenset(args.group_by) if 'group_by' in args and args.group_by else set() # type: t.FrozenSet[str]
self.all = args.all if 'all' in args else False # type: bool
self.stub = args.stub if 'stub' in args else False # type: bool

1
test/runner/lib/csharp_import_analysis.py
View file

@ -51,6 +51,7 @@ def extract_csharp_module_utils_imports(path, module_utils, is_pure_csharp):
"""Return a list of module_utils imports found in the specified source file.
:type path: str
:type module_utils: set[str]
:type is_pure_csharp: bool
:rtype: set[str]
"""
imports = set()

18
test/runner/lib/diff.py
View file

@ -5,6 +5,8 @@ import re
import textwrap
import traceback
import lib.types as t
from lib.util import (
ApplicationError,
)
@ -27,7 +29,7 @@ class FileDiff(object):
"""
self.old = DiffSide(old_path, new=False)
self.new = DiffSide(new_path, new=True)
self.headers = [] # list [str]
self.headers = [] # type: t.List[str]
self.binary = False
def append_header(self, line):
@ -57,9 +59,9 @@ class DiffSide(object):
self.eof_newline = True
self.exists = True
self.lines = [] # type: list [tuple[int, str]]
self.lines_and_context = [] # type: list [tuple[int, str]]
self.ranges = [] # type: list [tuple[int, int]]
self.lines = [] # type: t.List[t.Tuple[int, str]]
self.lines_and_context = [] # type: t.List[t.Tuple[int, str]]
self.ranges = [] # type: t.List[t.Tuple[int, int]]
self._next_line_number = 0
self._lines_remaining = 0
@ -138,13 +140,13 @@ class DiffParser(object):
:type lines: list[str]
"""
self.lines = lines
self.files = [] # type: list [FileDiff]
self.files = [] # type: t.List[FileDiff]
self.action = self.process_start
self.line_number = 0
self.previous_line = None # type: str
self.line = None # type: str
self.file = None # type: FileDiff
self.previous_line = None # type: t.Optional[str]
self.line = None # type: t.Optional[str]
self.file = None # type: t.Optional[FileDiff]
for self.line in self.lines:
self.line_number += 1

4
test/runner/lib/docker_util.py
View file

@ -226,8 +226,8 @@ def docker_exec(args, container_id, cmd, options=None, capture=False, stdin=None
:type cmd: list[str]
:type options: list[str] | None
:type capture: bool
:type stdin: file | None
:type stdout: file | None
:type stdin: BinaryIO | None
:type stdout: BinaryIO | None
:rtype: str | None, str | None
"""
if not options:

1
test/runner/lib/env.py
View file

@ -304,6 +304,7 @@ def get_git_details(args):
return git_details
# noinspection PyUnusedLocal
def get_merged_commit(args, commit): # pylint: disable=unused-argument
"""
:type args: CommonConfig

20
test/runner/lib/executor.py
View file

@ -18,6 +18,8 @@ import random
import string
import shutil
import lib.types as t
import lib.pytar
import lib.thread
@ -358,7 +360,7 @@ def command_network_integration(args):
all_targets = tuple(walk_network_integration_targets(include_hidden=True))
internal_targets = command_integration_filter(args, all_targets, init_callback=network_init)
instances = [] # type: list [lib.thread.WrappedThread]
instances = [] # type: t.List[lib.thread.WrappedThread]
if args.platform:
get_python_path(args, args.python_executable) # initialize before starting threads
@ -411,9 +413,9 @@ def network_init(args, internal_targets):
if args.metadata.instance_config is not None:
return
platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
platform_targets = set(a for target in internal_targets for a in target.aliases if a.startswith('network/'))
instances = [] # type: list [lib.thread.WrappedThread]
instances = [] # type: t.List[lib.thread.WrappedThread]
# generate an ssh key (if needed) up front once, instead of for each instance
SshKey(args)
@ -523,7 +525,7 @@ def command_windows_integration(args):
all_targets = tuple(walk_windows_integration_targets(include_hidden=True))
internal_targets = command_integration_filter(args, all_targets, init_callback=windows_init)
instances = [] # type: list [lib.thread.WrappedThread]
instances = [] # type: t.List[lib.thread.WrappedThread]
pre_target = None
post_target = None
httptester_id = None
@ -553,7 +555,7 @@ def command_windows_integration(args):
with open(filename, 'w') as inventory_fd:
inventory_fd.write(inventory)
use_httptester = args.httptester and any('needs/httptester/' in t.aliases for t in internal_targets)
use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in internal_targets)
# if running under Docker delegation, the httptester may have already been started
docker_httptester = bool(os.environ.get("HTTPTESTER", False))
@ -638,7 +640,7 @@ def windows_init(args, internal_targets): # pylint: disable=locally-disabled, u
if args.metadata.instance_config is not None:
return
instances = [] # type: list [lib.thread.WrappedThread]
instances = [] # type: t.List[lib.thread.WrappedThread]
for version in args.windows:
instance = lib.thread.WrappedThread(functools.partial(windows_start, args, version))
@ -765,7 +767,7 @@ def command_integration_filter(args, targets, init_callback=None):
if not internal_targets:
raise AllTargetsSkipped()
if args.start_at and not any(t.name == args.start_at for t in internal_targets):
if args.start_at and not any(target.name == args.start_at for target in internal_targets):
raise ApplicationError('Start at target matches nothing: %s' % args.start_at)
if init_callback:
@ -835,7 +837,7 @@ def command_integration_filtered(args, targets, all_targets, inventory_path, pre
results = {}
current_environment = None # type: EnvironmentDescription | None
current_environment = None # type: t.Optional[EnvironmentDescription]
# common temporary directory path that will be valid on both the controller and the remote
# it must be common because it will be referenced in environment variables that are shared across multiple hosts
@ -1944,7 +1946,7 @@ class EnvironmentDescription(object):
def get_version(command, warnings):
"""
:type command: list[str]
:type warnings: list[str]
:type warnings: list[text]
:rtype: list[str]
"""
try:

2
test/runner/lib/http.py
View file

@ -113,6 +113,8 @@ class HttpClient(object):
6, # CURLE_COULDNT_RESOLVE_HOST
)
stdout = ''
while True:
attempts += 1

22
test/runner/lib/metadata.py
View file

@ -3,6 +3,8 @@ from __future__ import absolute_import, print_function
import json
import lib.types as t
from lib.util import (
display,
is_shippable,
@ -18,10 +20,10 @@ class Metadata(object):
"""Metadata object for passing data to delegated tests."""
def __init__(self):
"""Initialize metadata."""
self.changes = {} # type: dict [str, tuple[tuple[int, int]]
self.cloud_config = None # type: dict [str, str]
self.instance_config = None # type: list[dict[str, str]]
self.change_description = None # type: ChangeDescription
self.changes = {} # type: t.Dict[str, t.Tuple[t.Tuple[int, int]]]
self.cloud_config = None # type: t.Optional[t.Dict[str, str]]
self.instance_config = None # type: t.Optional[t.List[t.Dict[str, str]]]
self.change_description = None # type: t.Optional[ChangeDescription]
if is_shippable():
self.ci_provider = 'shippable'
@ -33,7 +35,7 @@ class Metadata(object):
:type diff: list[str] | None
"""
patches = parse_diff(diff)
patches = sorted(patches, key=lambda k: k.new.path) # type: list [FileDiff]
patches = sorted(patches, key=lambda k: k.new.path) # type: t.List[FileDiff]
self.changes = dict((patch.new.path, tuple(patch.new.ranges)) for patch in patches)
@ -103,11 +105,11 @@ class ChangeDescription(object):
"""Description of changes."""
def __init__(self):
self.command = '' # type: str
self.changed_paths = [] # type: list[str]
self.deleted_paths = [] # type: list[str]
self.regular_command_targets = {} # type: dict[str, list[str]]
self.focused_command_targets = {} # type: dict[str, list[str]]
self.no_integration_paths = [] # type: list[str]
self.changed_paths = [] # type: t.List[str]
self.deleted_paths = [] # type: t.List[str]
self.regular_command_targets = {} # type: t.Dict[str, t.List[str]]
self.focused_command_targets = {} # type: t.Dict[str, t.List[str]]
self.no_integration_paths = [] # type: t.List[str]
@property
def targets(self):

12
test/runner/lib/sanity/__init__.py
View file

@ -8,6 +8,8 @@ import os
import re
import sys
import lib.types as t
from lib.util import (
ApplicationError,
SubprocessError,
@ -76,16 +78,16 @@ def command_sanity(args):
tests = sanity_get_tests()
if args.test:
tests = [t for t in tests if t.name in args.test]
tests = [target for target in tests if target.name in args.test]
else:
disabled = [t.name for t in tests if not t.enabled and not args.allow_disabled]
tests = [t for t in tests if t.enabled or args.allow_disabled]
disabled = [target.name for target in tests if not target.enabled and not args.allow_disabled]
tests = [target for target in tests if target.enabled or args.allow_disabled]
if disabled:
display.warning('Skipping tests disabled by default without --allow-disabled: %s' % ', '.join(sorted(disabled)))
if args.skip_test:
tests = [t for t in tests if t.name not in args.skip_test]
tests = [target for target in tests if target.name not in args.skip_test]
total = 0
failed = []
@ -383,7 +385,7 @@ SANITY_TESTS = (
def sanity_init():
"""Initialize full sanity test list (includes code-smell scripts determined at runtime)."""
import_plugins('sanity')
sanity_plugins = {} # type: dict[str, type]
sanity_plugins = {} # type: t.Dict[str, type]
load_plugins(SanityFunc, sanity_plugins)
sanity_tests = tuple([plugin() for plugin in sanity_plugins.values()])
global SANITY_TESTS # pylint: disable=locally-disabled, global-statement

16
test/runner/lib/sanity/integration_aliases.py
View file

@ -6,6 +6,8 @@ import textwrap
import re
import os
import lib.types as t
from lib.sanity import (
SanitySingleVersion,
SanityMessage,
@ -80,8 +82,8 @@ class IntegrationAliasesTest(SanitySingleVersion):
def __init__(self):
super(IntegrationAliasesTest, self).__init__()
self._shippable_yml_lines = [] # type: list[str]
self._shippable_test_groups = {} # type: dict[str, set[int]]
self._shippable_yml_lines = [] # type: t.List[str]
self._shippable_test_groups = {} # type: t.Dict[str, t.Set[int]]
@property
def shippable_yml_lines(self):
@ -231,9 +233,9 @@ class IntegrationAliasesTest(SanitySingleVersion):
:type find: str
:rtype: list[SanityMessage]
"""
all_paths = set(t.path for t in targets)
supported_paths = set(t.path for t in filter_targets(targets, [find], include=True, directories=False, errors=False))
unsupported_paths = set(t.path for t in filter_targets(targets, [self.UNSUPPORTED], include=True, directories=False, errors=False))
all_paths = set(target.path for target in targets)
supported_paths = set(target.path for target in filter_targets(targets, [find], include=True, directories=False, errors=False))
unsupported_paths = set(target.path for target in filter_targets(targets, [self.UNSUPPORTED], include=True, directories=False, errors=False))
unassigned_paths = all_paths - supported_paths - unsupported_paths
conflicting_paths = supported_paths & unsupported_paths
@ -259,8 +261,8 @@ class IntegrationAliasesTest(SanitySingleVersion):
integration_targets = list(walk_integration_targets())
module_targets = list(walk_module_targets())
integration_targets_by_name = dict((t.name, t) for t in integration_targets)
module_names_by_path = dict((t.path, t.module) for t in module_targets)
integration_targets_by_name = dict((target.name, target) for target in integration_targets)
module_names_by_path = dict((target.path, target.module) for target in module_targets)
disabled_targets = []
unstable_targets = []

8
test/runner/lib/sanity/pslint.py
View file

@ -6,6 +6,8 @@ import json
import os
import re
import lib.types as t
from lib.sanity import (
SanitySingleVersion,
SanityMessage,
@ -50,7 +52,7 @@ class PslintTest(SanitySingleVersion):
invalid_ignores = []
ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH, optional=True)
ignore = collections.defaultdict(dict)
ignore = collections.defaultdict(dict) # type: t.Dict[str, t.Dict[str, int]]
line = 0
for ignore_entry in ignore_entries:
@ -85,6 +87,8 @@ class PslintTest(SanitySingleVersion):
['test/sanity/pslint/pslint.ps1'] + paths
]
stdout = ''
for cmd in cmds:
try:
stdout, stderr = run_command(args, cmd, capture=True)
@ -131,7 +135,7 @@ class PslintTest(SanitySingleVersion):
for error in errors:
if error.code in ignore[error.path]:
ignore[error.path][error.code] = None # error ignored, clear line number of ignore entry to track usage
ignore[error.path][error.code] = 0 # error ignored, clear line number of ignore entry to track usage
else:
filtered.append(error) # error not ignored

5
test/runner/lib/sanity/pylint.py
View file

@ -26,7 +26,6 @@ from lib.util import (
)
from lib.util_common import (
intercept_command,
run_command,
)
@ -80,7 +79,7 @@ class PylintTest(SanitySingleVersion):
supported_versions = set([v.split('.')[0] for v in supported_versions]) | supported_versions
ignore_entries = read_lines_without_comments(PYLINT_IGNORE_PATH, optional=True)
ignore = collections.defaultdict(dict)
ignore = collections.defaultdict(dict) # type: t.Dict[str, t.Dict[str, int]]
line = 0
for ignore_entry in ignore_entries:
@ -207,7 +206,7 @@ class PylintTest(SanitySingleVersion):
for error in errors:
if error.code in ignore[error.path]:
ignore[error.path][error.code] = None # error ignored, clear line number of ignore entry to track usage
ignore[error.path][error.code] = 0 # error ignored, clear line number of ignore entry to track usage
else:
filtered.append(error) # error not ignored

6
test/runner/lib/sanity/validate_modules.py
View file

@ -5,6 +5,8 @@ import collections
import json
import os
import lib.types as t
from lib.sanity import (
SanitySingleVersion,
SanityMessage,
@ -78,7 +80,7 @@ class ValidateModulesTest(SanitySingleVersion):
invalid_ignores = []
ignore_entries = read_lines_without_comments(VALIDATE_IGNORE_PATH, optional=True)
ignore = collections.defaultdict(dict)
ignore = collections.defaultdict(dict) # type: t.Dict[str, t.Dict[str, int]]
line = 0
for ignore_entry in ignore_entries:
@ -137,7 +139,7 @@ class ValidateModulesTest(SanitySingleVersion):
for error in errors:
if error.code in ignore[error.path]:
ignore[error.path][error.code] = None # error ignored, clear line number of ignore entry to track usage
ignore[error.path][error.code] = 0 # error ignored, clear line number of ignore entry to track usage
else:
filtered.append(error) # error not ignored

16
test/runner/lib/util.py
View file

@ -29,14 +29,16 @@ except ImportError:
try:
# noinspection PyCompatibility
from ConfigParser import SafeConfigParser as ConfigParser
from configparser import ConfigParser
except ImportError:
# noinspection PyCompatibility
from configparser import ConfigParser
from ConfigParser import SafeConfigParser as ConfigParser
try:
# noinspection PyProtectedMember
from shlex import quote as cmd_quote
except ImportError:
# noinspection PyProtectedMember
from pipes import quote as cmd_quote
import lib.types as t
@ -47,11 +49,12 @@ except AttributeError:
C = None
DOCKER_COMPLETION = {} # type: dict[str, dict[str, str]]
REMOTE_COMPLETION = {} # type: dict[str, dict[str, str]]
PYTHON_PATHS = {} # type: dict[str, str]
DOCKER_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]]
REMOTE_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]]
PYTHON_PATHS = {} # type: t.Dict[str, str]
try:
# noinspection PyUnresolvedReferences
MAXFD = subprocess.MAXFD
except AttributeError:
MAXFD = -1
@ -776,9 +779,12 @@ def load_plugins(base_type, database): # type: (t.Type[C], t.Dict[str, t.Type[C
def load_module(path, name): # type: (str, str) -> None
"""Load a Python module using the given name and path."""
if sys.version_info >= (3, 4):
# noinspection PyUnresolvedReferences
import importlib.util
# noinspection PyUnresolvedReferences
spec = importlib.util.spec_from_file_location(name, path)
# noinspection PyUnresolvedReferences
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)

2
test/runner/retry.py
View file

@ -75,12 +75,14 @@ def parse_args():
const='yes',
default='auto')
# noinspection PyTypeChecker
parser.add_argument('--tries',
metavar='TRIES',
type=int,
default=3,
help='number of tries to execute command (default: %(default)s)')
# noinspection PyTypeChecker
parser.add_argument('--sleep',
metavar='SECONDS',
type=int,

16
test/runner/shippable.py
View file

@ -22,6 +22,10 @@ from lib.util import (
MissingEnvironmentVariable,
)
from lib.util_common import (
CommonConfig,
)
try:
import argcomplete
@ -33,15 +37,21 @@ def main():
"""Main program function."""
try:
args = parse_args()
display.verbosity = args.verbosity
display.color = args.color
args.debug = False
args.truncate = False
args.redact = False
config = CommonConfig(args, '')
display.verbosity = config.verbosity
display.color = config.color
try:
run_id = os.environ['SHIPPABLE_BUILD_ID']
except KeyError as ex:
raise MissingEnvironmentVariable(ex.args[0])
client = HttpClient(args)
client = HttpClient(config)
response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id)
jobs = response.json()