Fix ansible-test multi-group smoke test handling. (#46363)

* Fix ansible-test smoke tests across groups.
* Fix ansible-test list arg defaults.
* Fix ansible-test require and exclude delegation.
* Fix detection of Windows specific changes.
* Add minimal Windows testing for Python 3.7.

test/runner/lib/cli.py

@@ -169,6 +169,11 @@ def parse_args():
                       nargs='*',
                       help='test the specified target').completer = complete_target
 
+    test.add_argument('--include',
+                      metavar='TARGET',
+                      action='append',
+                      help='include the specified target').completer = complete_target
+
     test.add_argument('--exclude',
                       metavar='TARGET',
                       action='append',
@@ -261,6 +266,11 @@ def parse_args():
                              default='all',
                              help='target to run when all tests are needed')
 
+    integration.add_argument('--changed-all-mode',
+                             metavar='MODE',
+                             choices=('default', 'include', 'exclude'),
+                             help='include/exclude behavior with --changed-all-target: %(choices)s')
+
     integration.add_argument('--list-targets',
                              action='store_true',
                              help='list matching targets instead of running tests')

test/runner/lib/config.py

@@ -71,6 +71,7 @@ class EnvironmentConfig(CommonConfig):
         self.python_version = self.python or '.'.join(str(i) for i in sys.version_info[:2])
 
         self.delegate = self.tox or self.docker or self.remote
+        self.delegate_args = []  # type: list[str]
 
         if self.delegate:
             self.requirements = True
@@ -104,9 +105,9 @@ class TestConfig(EnvironmentConfig):
 
         self.coverage = args.coverage  # type: bool
         self.coverage_label = args.coverage_label  # type: str
-        self.include = args.include  # type: list [str]
+        self.include = args.include or []  # type: list [str]
-        self.exclude = args.exclude  # type: list [str]
+        self.exclude = args.exclude or []  # type: list [str]
-        self.require = args.require  # type: list [str]
+        self.require = args.require or []  # type: list [str]
 
         self.changed = args.changed  # type: bool
         self.tracked = args.tracked  # type: bool
@@ -179,6 +180,7 @@ class IntegrationConfig(TestConfig):
         self.continue_on_error = args.continue_on_error  # type: bool
         self.debug_strategy = args.debug_strategy  # type: bool
         self.changed_all_target = args.changed_all_target  # type: str
+        self.changed_all_mode = args.changed_all_mode  # type: str
         self.list_targets = args.list_targets  # type: bool
         self.tags = args.tags
         self.skip_tags = args.skip_tags

test/runner/lib/delegation.py

@@ -458,6 +458,8 @@ def filter_options(args, argv, options, exclude, require):
             '--changed-from': 1,
             '--changed-path': 1,
             '--metadata': 1,
+            '--exclude': 1,
+            '--require': 1,
         })
     elif isinstance(args, SanityConfig):
         options.update({
@@ -482,6 +484,9 @@ def filter_options(args, argv, options, exclude, require):
 
         yield arg
 
+    for arg in args.delegate_args:
+        yield arg
+
     for target in exclude:
         yield '--exclude'
         yield target

test/runner/lib/executor.py

@@ -651,8 +651,19 @@ def command_integration_filter(args, targets, init_callback=None):
     """
     targets = tuple(target for target in targets if 'hidden/' not in target.aliases)
     changes = get_changes_filter(args)
-    require = (args.require or []) + changes
+
-    exclude = (args.exclude or [])
+    # special behavior when the --changed-all-target target is selected based on changes
+    if args.changed_all_target in changes:
+        # act as though the --changed-all-target target was in the include list
+        if args.changed_all_mode == 'include' and args.changed_all_target not in args.include:
+            args.include.append(args.changed_all_target)
+            args.delegate_args += ['--include', args.changed_all_target]
+        # act as though the --changed-all-target target was in the exclude list
+        elif args.changed_all_mode == 'exclude' and args.changed_all_target not in args.exclude:
+            args.exclude.append(args.changed_all_target)
+
+    require = args.require + changes
+    exclude = args.exclude
 
     internal_targets = walk_internal_targets(targets, args.include, exclude, require)
     environment_exclude = get_integration_filter(args, internal_targets)
@@ -675,7 +686,7 @@ def command_integration_filter(args, targets, init_callback=None):
     cloud_init(args, internal_targets)
 
     if args.delegate:
-        raise Delegate(require=changes, exclude=exclude, integration_targets=internal_targets)
+        raise Delegate(require=require, exclude=exclude, integration_targets=internal_targets)
 
     install_command_requirements(args)
 
@@ -1133,7 +1144,7 @@ def command_units(args):
     :type args: UnitsConfig
     """
     changes = get_changes_filter(args)
-    require = (args.require or []) + changes
+    require = args.require + changes
     include, exclude = walk_external_targets(walk_units_targets(), args.include, args.exclude, require)
 
     if not include:

test/runner/lib/sanity/__init__.py

@@ -58,7 +58,7 @@ def command_sanity(args):
     :type args: SanityConfig
     """
     changes = get_changes_filter(args)
-    require = (args.require or []) + changes
+    require = args.require + changes
     targets = SanityTargets(args.include, args.exclude, require)
 
     if not targets.include:

test/utils/shippable/cloud.sh

@@ -13,9 +13,11 @@ target="shippable/${cloud}/group${group}/"
 
 stage="${S:-prod}"
 
+changed_all_target="shippable/${cloud}/smoketest/"
+
 if [ "${group}" == "1" ]; then
     # only run smoketest tests for group1
-    changed_all_target="shippable/${cloud}/smoketest/"
+    changed_all_mode="include"
 
     if ! ansible-test integration "${changed_all_target}" --list-targets > /dev/null 2>&1; then
         # no smoketest tests are available for this cloud
@@ -23,10 +25,10 @@ if [ "${group}" == "1" ]; then
     fi
 else
     # smoketest tests already covered by group1
-    changed_all_target="none"
+    changed_all_mode="exclude"
 fi
 
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --remote-terminate always --remote-stage "${stage}" \
-    --docker --python "${python}" --changed-all-target "${changed_all_target}"
+    --docker --python "${python}" --changed-all-target "${changed_all_target}" --changed-all-mode "${changed_all_mode}"

test/utils/shippable/windows.sh

@@ -19,6 +19,7 @@ python_versions=(
     2.6
     3.5
     3.6
+    3.7
     2.7
 )
 
@@ -26,7 +27,7 @@ python_versions=(
 single_version=2012-R2
 
 # shellcheck disable=SC2086
-ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
+ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
     | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
 
 if [ -s /tmp/windows.txt ] || [ "${CHANGED:+$CHANGED}" == "" ]; then
@@ -54,6 +55,7 @@ fi
 
 for version in "${python_versions[@]}"; do
     changed_all_target="all"
+    changed_all_mode="default"
 
     if [ "${version}" == "2.7" ]; then
         # smoketest tests for python 2.7
@@ -61,12 +63,13 @@ for version in "${python_versions[@]}"; do
             # with change detection enabled run tests for anything changed
             # use the smoketest tests for any change that triggers all tests
             ci="${target}"
+            changed_all_target="shippable/windows/smoketest/"
             if [ "${target}" == "shippable/windows/group1/" ]; then
                 # only run smoketest tests for group1
-                changed_all_target="shippable/windows/smoketest/"
+                changed_all_mode="include"
             else
                 # smoketest tests already covered by group1
-                changed_all_target="none"
+                changed_all_mode="exclude"
             fi
         else
             # without change detection enabled run entire test group
@@ -88,7 +91,7 @@ for version in "${python_versions[@]}"; do
 
     # shellcheck disable=SC2086
     ansible-test windows-integration --color -v --retry-on-error "${ci}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
-        "${platforms[@]}" --changed-all-target "${changed_all_target}" \
+        "${platforms[@]}" --changed-all-target "${changed_all_target}" --changed-all-mode "${changed_all_mode}" \
         --docker default --python "${version}" \
         --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
 done