Improve handling of integration test aliases. (#38698)

* Include change classification data in metadata.
* Add support for disabled tests.
* Add support for unstable tests.
* Add support for unsupported tests.
* Overhaul integration aliases sanity test.
* Update Shippable scripts to handle unstable tests.
* Mark unstable Azure tests.
* Mark unstable Windows tests.
* Mark disabled tests.

test/integration/targets/azure_rm_containerinstance/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_keyvaultkey/aliases

@@ -1,3 +1,4 @@
 cloud/azure
-posix/ci/cloud/azure
+posix/ci/cloud/group2/azure
 destructive
+disabled

test/integration/targets/azure_rm_keyvaultsecret/aliases

@@ -1,3 +1,4 @@
 cloud/azure
-posix/ci/cloud/azure
+posix/ci/cloud/group2/azure
 destructive
+disabled

test/integration/targets/azure_rm_loadbalancer/aliases

@@ -1,2 +1,4 @@
 cloud/azure
+posix/ci/cloud/group3/azure
+unstable
 destructive

test/integration/targets/azure_rm_postgresqldatabase/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_postgresqlserver/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_securitygroup/aliases

@@ -1,3 +1,5 @@
 cloud/azure
+posix/ci/cloud/group2/azure
+unstable
 destructive
 azure_rm_securitygroup_facts

test/integration/targets/azure_rm_sqldatabase/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_sqlserver_facts/aliases

@@ -1,2 +1,4 @@
 cloud/azure
 destructive
+posix/ci/cloud/group2/azure
+unstable

test/integration/targets/azure_rm_virtualmachine/aliases

@@ -1,2 +1,4 @@
 cloud/azure
+posix/ci/cloud/group3/azure
+unstable
 destructive

test/integration/targets/ec2_ami/aliases

@@ -1,2 +1,4 @@
 cloud/aws
+posix/ci/cloud/group4/aws
+disabled
 ec2_ami_facts

test/integration/targets/ec2_group/aliases

@@ -1 +1,3 @@
 cloud/aws
+posix/ci/cloud/group4/aws
+disabled

test/integration/targets/ec2_vpc_route_table/aliases

@@ -1 +1,3 @@
 cloud/aws
+posix/ci/cloud/group4/aws
+disabled

test/integration/targets/github_issue/aliases

@@ -1 +1,3 @@
 destructive
+posix/ci/group1
+disabled

test/integration/targets/influxdb_user/aliases

@@ -1,4 +1,6 @@
 destructive
+posix/ci/group1
+disabled
 skip/osx
 skip/freebsd
 skip/rhel

test/integration/targets/nuage_vspk/aliases

@@ -1 +1,3 @@
+posix/ci/group1
 skip/python3
+disabled

test/integration/targets/vmware_datastore_maintenancemode/aliases

@@ -1,2 +1,4 @@
 cloud/vcenter
 destructive
+posix/ci/cloud/group4/vcenter
+disabled

test/integration/targets/win_msg/aliases

@@ -0,0 +1,2 @@
+windows/ci/group2
+unstable

test/integration/targets/win_psexec/aliases

@@ -0,0 +1,2 @@
+windows/ci/group3
+unstable

test/integration/targets/win_rabbitmq_plugin/aliases

@@ -0,0 +1,2 @@
+windows/ci/group3
+disabled

test/integration/targets/win_toast/aliases

@@ -1 +1,2 @@
-
+windows/ci/group1
+disabled

test/integration/targets/win_uri/aliases

@@ -0,0 +1,2 @@
+windows/ci/group3
+unstable

test/integration/targets/win_wait_for/aliases

@@ -0,0 +1,2 @@
+windows/ci/group1
+unstable

test/integration/targets/xml/aliases

@@ -1 +1,3 @@
 destructive
+posix/ci/group1
+disabled

test/runner/lib/classification.py

@@ -2,6 +2,7 @@
 
 from __future__ import absolute_import, print_function
 
+import collections
 import os
 import re
 import time
@@ -33,13 +34,19 @@ from lib.config import (
     IntegrationConfig,
 )
 
+from lib.metadata import (
+    ChangeDescription,
+)
+
+FOCUSED_TARGET = '__focused__'
+
 
 def categorize_changes(args, paths, verbose_command=None):
     """
     :type args: TestConfig
     :type paths: list[str]
     :type verbose_command: str
-    :rtype paths: dict[str, list[str]]
+    :rtype: ChangeDescription
     """
     mapper = PathMapper(args)
 
@@ -51,12 +58,20 @@ def categorize_changes(args, paths, verbose_command=None):
         'network-integration': set(),
     }
 
+    focused_commands = collections.defaultdict(set)
+
+    deleted_paths = set()
+    original_paths = set()
     additional_paths = set()
+    no_integration_paths = set()
 
     for path in paths:
         if not os.path.exists(path):
+            deleted_paths.add(path)
             continue
 
+        original_paths.add(path)
+
         dependent_paths = mapper.get_dependent_paths(path)
 
         if not dependent_paths:
@@ -80,18 +95,28 @@ def categorize_changes(args, paths, verbose_command=None):
         tests = mapper.classify(path)
 
         if tests is None:
+            focused_target = False
+
             display.info('%s -> all' % path, verbosity=1)
             tests = all_tests(args)  # not categorized, run all tests
             display.warning('Path not categorized: %s' % path)
         else:
+            focused_target = tests.pop(FOCUSED_TARGET, False) and path in original_paths
+
             tests = dict((key, value) for key, value in tests.items() if value)
 
+            if focused_target and not any('integration' in command for command in tests):
+                no_integration_paths.add(path)  # path triggers no integration tests
+
             if verbose_command:
                 result = '%s: %s' % (verbose_command, tests.get(verbose_command) or 'none')
 
                 # identify targeted integration tests (those which only target a single integration command)
                 if 'integration' in verbose_command and tests.get(verbose_command):
                     if not any('integration' in command for command in tests if command != verbose_command):
+                        if focused_target:
+                            result += ' (focused)'
+
                         result += ' (targeted)'
             else:
                 result = '%s' % tests
@@ -101,6 +126,9 @@ def categorize_changes(args, paths, verbose_command=None):
         for command, target in tests.items():
             commands[command].add(target)
 
+            if focused_target:
+                focused_commands[command].add(target)
+
     for command in commands:
         commands[command].discard('none')
 
@@ -108,8 +136,21 @@ def categorize_changes(args, paths, verbose_command=None):
             commands[command] = set(['all'])
 
     commands = dict((c, sorted(commands[c])) for c in commands if commands[c])
+    focused_commands = dict((c, sorted(focused_commands[c])) for c in focused_commands)
 
-    return commands
+    for command in commands:
+        if commands[command] == ['all']:
+            commands[command] = []  # changes require testing all targets, do not filter targets
+
+    changes = ChangeDescription()
+    changes.command = verbose_command
+    changes.changed_paths = sorted(original_paths)
+    changes.deleted_paths = sorted(deleted_paths)
+    changes.regular_command_targets = commands
+    changes.focused_command_targets = focused_commands
+    changes.no_integration_paths = sorted(no_integration_paths)
+
+    return changes
 
 
 class PathMapper(object):
@@ -278,6 +319,7 @@ class PathMapper(object):
                     'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None,
                     'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None,
                     'network-integration': self.network_integration_by_module.get(module_name),
+                    FOCUSED_TARGET: True,
                 }
 
             return minimal
@@ -459,6 +501,7 @@ class PathMapper(object):
                 'integration': target.name if 'posix/' in target.aliases else None,
                 'windows-integration': target.name if 'windows/' in target.aliases else None,
                 'network-integration': target.name if 'network/' in target.aliases else None,
+                FOCUSED_TARGET: True,
             }
 
         if path.startswith('test/integration/'):

test/runner/lib/config.py

@@ -162,6 +162,10 @@ class IntegrationConfig(TestConfig):
         self.start_at_task = args.start_at_task  # type: str
         self.allow_destructive = args.allow_destructive  # type: bool
         self.allow_root = args.allow_root  # type: bool
+        self.allow_disabled = args.allow_disabled  # type: bool
+        self.allow_unstable = args.allow_unstable  # type: bool
+        self.allow_unstable_changed = args.allow_unstable_changed  # type: bool
+        self.allow_unsupported = args.allow_unsupported  # type: bool
         self.retry_on_error = args.retry_on_error  # type: bool
         self.continue_on_error = args.continue_on_error  # type: bool
         self.debug_strategy = args.debug_strategy  # type: bool

test/runner/lib/executor.py

@@ -88,6 +88,10 @@ from lib.config import (
     WindowsIntegrationConfig,
 )
 
+from lib.metadata import (
+    ChangeDescription,
+)
+
 SUPPORTED_PYTHON_VERSIONS = (
     '2.6',
     '2.7',
@@ -1029,23 +1033,24 @@ def get_changes_filter(args):
     """
     paths = detect_changes(args)
 
+    if not args.metadata.change_description:
+        if paths:
+            changes = categorize_changes(args, paths, args.command)
+        else:
+            changes = ChangeDescription()
+
+        args.metadata.change_description = changes
+
     if paths is None:
         return []  # change detection not enabled, do not filter targets
 
     if not paths:
         raise NoChangesDetected()
 
-    commands = categorize_changes(args, paths, args.command)
-
-    targets = commands.get(args.command)
-
-    if targets is None:
+    if args.metadata.change_description.targets is None:
         raise NoTestsForChanges()
 
-    if targets == ['all']:
-        return []  # changes require testing all targets, do not filter targets
-
-    return targets
+    return args.metadata.change_description.targets
 
 
 def detect_changes(args):
@@ -1175,6 +1180,49 @@ def get_integration_filter(args, targets):
     return get_integration_local_filter(args, targets)
 
 
+def common_integration_filter(args, targets, exclude):
+    """
+    :type args: IntegrationConfig
+    :type targets: tuple[IntegrationTarget]
+    :type exclude: list[str]
+    """
+    override_disabled = set(target for target in args.include if target.startswith('disabled/'))
+
+    if not args.allow_disabled:
+        skip = 'disabled/'
+        override = [target.name for target in targets if override_disabled & set(target.aliases)]
+        skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
+        if skipped:
+            exclude.extend(skipped)
+            display.warning('Excluding tests marked "%s" which require --allow-disabled or prefixing with "disabled/": %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
+    override_unsupported = set(target for target in args.include if target.startswith('unsupported/'))
+
+    if not args.allow_unsupported:
+        skip = 'unsupported/'
+        override = [target.name for target in targets if override_unsupported & set(target.aliases)]
+        skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
+        if skipped:
+            exclude.extend(skipped)
+            display.warning('Excluding tests marked "%s" which require --allow-unsupported or prefixing with "unsupported/": %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
+    override_unstable = set(target for target in args.include if target.startswith('unstable/'))
+
+    if args.allow_unstable_changed:
+        override_unstable |= set(args.metadata.change_description.focused_targets or [])
+
+    if not args.allow_unstable:
+        skip = 'unstable/'
+        override = [target.name for target in targets if override_unstable & set(target.aliases)]
+        skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
+        if skipped:
+            exclude.extend(skipped)
+            display.warning('Excluding tests marked "%s" which require --allow-unstable or prefixing with "unstable/": %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
+
 def get_integration_local_filter(args, targets):
     """
     :type args: IntegrationConfig
@@ -1183,6 +1231,8 @@ def get_integration_local_filter(args, targets):
     """
     exclude = []
 
+    common_integration_filter(args, targets, exclude)
+
     if not args.allow_root and os.getuid() != 0:
         skip = 'needs/root/'
         skipped = [target.name for target in targets if skip in target.aliases]
@@ -1225,6 +1275,8 @@ def get_integration_docker_filter(args, targets):
     """
     exclude = []
 
+    common_integration_filter(args, targets, exclude)
+
     if not args.docker_privileged:
         skip = 'needs/privileged/'
         skipped = [target.name for target in targets if skip in target.aliases]
@@ -1271,6 +1323,8 @@ def get_integration_remote_filter(args, targets):
 
     exclude = []
 
+    common_integration_filter(args, targets, exclude)
+
     skip = 'skip/%s/' % platform
     skipped = [target.name for target in targets if skip in target.aliases]
     if skipped:

test/runner/lib/metadata.py

@@ -21,6 +21,7 @@ class Metadata(object):
         self.changes = {}  # type: dict [str, tuple[tuple[int, int]]
         self.cloud_config = None  # type: dict [str, str]
         self.instance_config = None  # type: list[dict[str, str]]
+        self.change_description = None  # type: ChangeDescription
 
         if is_shippable():
             self.ci_provider = 'shippable'
@@ -57,6 +58,7 @@ class Metadata(object):
             cloud_config=self.cloud_config,
             instance_config=self.instance_config,
             ci_provider=self.ci_provider,
+            change_description=self.change_description.to_dict(),
         )
 
     def to_file(self, path):
@@ -92,5 +94,60 @@ class Metadata(object):
         metadata.cloud_config = data['cloud_config']
         metadata.instance_config = data['instance_config']
         metadata.ci_provider = data['ci_provider']
+        metadata.change_description = ChangeDescription.from_dict(data['change_description'])
 
         return metadata
+
+
+class ChangeDescription(object):
+    """Description of changes."""
+    def __init__(self):
+        self.command = ''  # type: str
+        self.changed_paths = []  # type: list[str]
+        self.deleted_paths = []  # type: list[str]
+        self.regular_command_targets = {}  # type: dict[str, list[str]]
+        self.focused_command_targets = {}  # type: dict[str, list[str]]
+        self.no_integration_paths = []  # type: list[str]
+
+    @property
+    def targets(self):
+        """
+        :rtype: list[str] | None
+        """
+        return self.regular_command_targets.get(self.command)
+
+    @property
+    def focused_targets(self):
+        """
+        :rtype: list[str] | None
+        """
+        return self.focused_command_targets.get(self.command)
+
+    def to_dict(self):
+        """
+        :rtype: dict[str, any]
+        """
+        return dict(
+            command=self.command,
+            changed_paths=self.changed_paths,
+            deleted_paths=self.deleted_paths,
+            regular_command_targets=self.regular_command_targets,
+            focused_command_targets=self.focused_command_targets,
+            no_integration_paths=self.no_integration_paths,
+        )
+
+    @staticmethod
+    def from_dict(data):
+        """
+        :param data: dict[str, any]
+        :rtype: ChangeDescription
+        """
+        changes = ChangeDescription()
+        changes.command = data['command']
+        changes.changed_paths = data['changed_paths']
+        changes.deleted_paths = data['deleted_paths']
+        changes.regular_command_targets = data['regular_command_targets']
+        changes.focused_command_targets = data['focused_command_targets']
+        changes.no_integration_paths = data['no_integration_paths']
+
+        return changes

test/runner/lib/sanity/integration_aliases.py

@@ -0,0 +1,247 @@
+"""Sanity test to check integration test aliases."""
+from __future__ import absolute_import, print_function
+
+import json
+import textwrap
+
+from lib.sanity import (
+    SanitySingleVersion,
+    SanityMessage,
+    SanityFailure,
+    SanitySuccess,
+    SanityTargets,
+)
+
+from lib.config import (
+    SanityConfig,
+)
+
+from lib.target import (
+    filter_targets,
+    walk_posix_integration_targets,
+    walk_windows_integration_targets,
+    walk_integration_targets,
+    walk_module_targets,
+)
+
+from lib.cloud import (
+    get_cloud_platforms,
+)
+
+
+class IntegrationAliasesTest(SanitySingleVersion):
+    """Sanity test to evaluate integration test aliases."""
+    DISABLED = 'disabled/'
+    UNSTABLE = 'unstable/'
+    UNSUPPORTED = 'unsupported/'
+
+    EXPLAIN_URL = 'https://docs.ansible.com/ansible/devel/dev_guide/testing/sanity/integration-aliases.html'
+
+    TEMPLATE_DISABLED = """
+    The following integration tests are **disabled** [[explain]({explain_url}#disabled)]:
+
+    {tests}
+
+    Consider fixing the integration tests before or alongside changes.
+    """
+
+    TEMPLATE_UNSTABLE = """
+    The following integration tests are **unstable** [[explain]({explain_url}#unstable)]:
+
+    {tests}
+
+    Tests may need to be restarted due to failures unrelated to changes.
+    """
+
+    TEMPLATE_UNSUPPORTED = """
+    The following integration tests are **unsupported** [[explain]({explain_url}#unsupported)]:
+
+    {tests}
+
+    Consider running the tests manually or extending test infrastructure to add support.
+    """
+
+    TEMPLATE_UNTESTED = """
+    The following modules have **no integration tests** [[explain]({explain_url}#untested)]:
+
+    {tests}
+
+    Consider adding integration tests before or alongside changes.
+    """
+
+    def test(self, args, targets):
+        """
+        :type args: SanityConfig
+        :type targets: SanityTargets
+        :rtype: TestResult
+        """
+        if args.explain:
+            return SanitySuccess(self.name)
+
+        results = dict(
+            comments=[],
+            labels={},
+        )
+
+        self.check_changes(args, results)
+
+        with open('test/results/bot/data-sanity-ci.json', 'w') as results_fd:
+            json.dump(results, results_fd, sort_keys=True, indent=4)
+
+        messages = []
+
+        messages += self.check_posix_targets(args)
+        messages += self.check_windows_targets()
+
+        if messages:
+            return SanityFailure(self.name, messages=messages)
+
+        return SanitySuccess(self.name)
+
+    def check_posix_targets(self, args):
+        """
+        :type args: SanityConfig
+        :rtype: list[SanityMessage]
+        """
+        posix_targets = tuple(walk_posix_integration_targets())
+
+        clouds = get_cloud_platforms(args, posix_targets)
+        cloud_targets = ['cloud/%s/' % cloud for cloud in clouds]
+
+        all_cloud_targets = tuple(filter_targets(posix_targets, ['cloud/'], include=True, directories=False, errors=False))
+        invalid_cloud_targets = tuple(filter_targets(all_cloud_targets, cloud_targets, include=False, directories=False, errors=False))
+
+        messages = []
+
+        for target in invalid_cloud_targets:
+            for alias in target.aliases:
+                if alias.startswith('cloud/') and alias != 'cloud/':
+                    if any(alias.startswith(cloud_target) for cloud_target in cloud_targets):
+                        continue
+
+                    messages.append(SanityMessage('invalid alias `%s`' % alias, '%s/aliases' % target.path))
+
+        messages += self.check_ci_group(
+            targets=tuple(filter_targets(posix_targets, ['cloud/'], include=False, directories=False, errors=False)),
+            find='posix/ci/group[1-3]/',
+        )
+
+        for cloud in clouds:
+            messages += self.check_ci_group(
+                targets=tuple(filter_targets(posix_targets, ['cloud/%s/' % cloud], include=True, directories=False, errors=False)),
+                find='posix/ci/cloud/group[1-5]/%s/' % cloud,
+            )
+
+        return messages
+
+    def check_windows_targets(self):
+        """
+        :rtype: list[SanityMessage]
+        """
+        windows_targets = tuple(walk_windows_integration_targets())
+
+        messages = []
+
+        messages += self.check_ci_group(
+            targets=windows_targets,
+            find='windows/ci/group[1-3]/',
+        )
+
+        return messages
+
+    def check_ci_group(self, targets, find):
+        """
+        :type targets: tuple[CompletionTarget]
+        :type find: str
+        :rtype: list[SanityMessage]
+        """
+        all_paths = set(t.path for t in targets)
+        supported_paths = set(t.path for t in filter_targets(targets, [find], include=True, directories=False, errors=False))
+        unsupported_paths = set(t.path for t in filter_targets(targets, [self.UNSUPPORTED], include=True, directories=False, errors=False))
+
+        unassigned_paths = all_paths - supported_paths - unsupported_paths
+        conflicting_paths = supported_paths & unsupported_paths
+
+        unassigned_message = 'missing alias `%s` or `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
+        conflicting_message = 'conflicting alias `%s` and `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
+
+        messages = []
+
+        for path in unassigned_paths:
+            messages.append(SanityMessage(unassigned_message, '%s/aliases' % path))
+
+        for path in conflicting_paths:
+            messages.append(SanityMessage(conflicting_message, '%s/aliases' % path))
+
+        return messages
+
+    def check_changes(self, args, results):
+        """
+        :type args: SanityConfig
+        :type results: dict[str, any]
+        """
+        integration_targets = list(walk_integration_targets())
+        module_targets = list(walk_module_targets())
+
+        integration_targets_by_name = dict((t.name, t) for t in integration_targets)
+        module_names_by_path = dict((t.path, t.module) for t in module_targets)
+
+        disabled_targets = []
+        unstable_targets = []
+        unsupported_targets = []
+
+        for command in [command for command in args.metadata.change_description.focused_command_targets if 'integration' in command]:
+            for target in args.metadata.change_description.focused_command_targets[command]:
+                if self.DISABLED in integration_targets_by_name[target].aliases:
+                    disabled_targets.append(target)
+                elif self.UNSTABLE in integration_targets_by_name[target].aliases:
+                    unstable_targets.append(target)
+                elif self.UNSUPPORTED in integration_targets_by_name[target].aliases:
+                    unsupported_targets.append(target)
+
+        untested_modules = []
+
+        for path in args.metadata.change_description.no_integration_paths:
+            module = module_names_by_path.get(path)
+
+            if module:
+                untested_modules.append(module)
+
+        comments = [
+            self.format_comment(self.TEMPLATE_DISABLED, disabled_targets),
+            self.format_comment(self.TEMPLATE_UNSTABLE, unstable_targets),
+            self.format_comment(self.TEMPLATE_UNSUPPORTED, unsupported_targets),
+            self.format_comment(self.TEMPLATE_UNTESTED, untested_modules),
+        ]
+
+        comments = [comment for comment in comments if comment]
+
+        labels = dict(
+            needs_tests=bool(untested_modules),
+            disabled_tests=bool(disabled_targets),
+            unstable_tests=bool(unstable_targets),
+            unsupported_tests=bool(unsupported_targets),
+        )
+
+        results['comments'] += comments
+        results['labels'].update(labels)
+
+    def format_comment(self, template, targets):
+        """
+        :type template: str
+        :type targets: list[str]
+        :rtype: str | None
+        """
+        if not targets:
+            return None
+
+        tests = '\n'.join('- %s' % target for target in targets)
+
+        data = dict(
+            explain_url=self.EXPLAIN_URL,
+            tests=tests,
+        )
+
+        message = textwrap.dedent(template).strip().format(**data)
+
+        return message

test/runner/test.py

@@ -229,6 +229,22 @@ def parse_args():
                              action='store_true',
                              help='allow tests requiring root when not root')
 
+    integration.add_argument('--allow-disabled',
+                             action='store_true',
+                             help='allow tests which have been marked as disabled')
+
+    integration.add_argument('--allow-unstable',
+                             action='store_true',
+                             help='allow tests which have been marked as unstable')
+
+    integration.add_argument('--allow-unstable-changed',
+                             action='store_true',
+                             help='allow tests which have been marked as unstable when focused changes are detected')
+
+    integration.add_argument('--allow-unsupported',
+                             action='store_true',
+                             help='allow tests which have been marked as unsupported')
+
     integration.add_argument('--retry-on-error',
                              action='store_true',
                              help='retry failed test with increased verbosity')

test/sanity/code-smell/integration-aliases.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import textwrap
-
-
-def main():
-    targets_dir = 'test/integration/targets'
-
-    with open('test/integration/target-prefixes.network', 'r') as prefixes_fd:
-        network_prefixes = prefixes_fd.read().splitlines()
-
-    for target in sorted(os.listdir(targets_dir)):
-        target_dir = os.path.join(targets_dir, target)
-        aliases_path = os.path.join(target_dir, 'aliases')
-        files = sorted(os.listdir(target_dir))
-
-        # aliases already defined
-        if os.path.exists(aliases_path):
-            continue
-
-        # don't require aliases for support directories
-        if any(os.path.splitext(f)[0] == 'test' and os.access(os.path.join(target_dir, f), os.X_OK) for f in files):
-            continue
-
-        # don't require aliases for setup_ directories
-        if target.startswith('setup_'):
-            continue
-
-        # don't require aliases for prepare_ directories
-        if target.startswith('prepare_'):
-            continue
-
-        # TODO: remove this exclusion once the `ansible-test network-integration` command is working properly
-        # don't require aliases for network modules
-        if any(target.startswith('%s_' % prefix) for prefix in network_prefixes):
-            continue
-
-        print('%s: missing integration test `aliases` file' % aliases_path)
-
-
-if __name__ == '__main__':
-    main()

test/utils/shippable/cloud.sh

@@ -12,6 +12,6 @@ target="posix/ci/cloud/group${args[3]}/"
 stage="${S:-prod}"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --remote-terminate always --remote-stage "${stage}" \
     --docker "${image}" --python "${python}" --changed-all-target "${target}smoketest/"

test/utils/shippable/freebsd.sh

@@ -18,6 +18,6 @@ stage="${S:-prod}"
 provider="${P:-default}"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --exclude "posix/ci/cloud/" \
     --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

test/utils/shippable/linux.sh

@@ -14,6 +14,6 @@ else
 fi
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --exclude "posix/ci/cloud/" \
     --docker "${image}"

test/utils/shippable/network.sh

@@ -3,7 +3,8 @@
 set -o pipefail
 
 # shellcheck disable=SC2086
-ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
+ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
+    | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
 
 if [ "${COVERAGE}" ]; then
     # when on-demand coverage is enabled, force tests to run for all network platforms
@@ -49,7 +50,8 @@ for version in "${python_versions[@]}"; do
     fi
 
     # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --docker default --python "${version}" \
-        ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} "${platforms[@]}" \
+    ansible-test network-integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
+         "${platforms[@]}" \
+        --docker default --python "${version}" \
         --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
 done

test/utils/shippable/osx.sh

@@ -18,6 +18,6 @@ stage="${S:-prod}"
 provider="${P:-default}"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --exclude "posix/ci/cloud/" \
     --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

test/utils/shippable/rhel.sh

@@ -18,6 +18,6 @@ stage="${S:-prod}"
 provider="${P:-default}"
 
 # shellcheck disable=SC2086
-ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
     --exclude "posix/ci/cloud/" \
     --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

test/utils/shippable/shippable.sh

@@ -53,6 +53,14 @@ else
     export CHANGED="--changed"
 fi
 
+if [ "${IS_PULL_REQUEST:-}" == "true" ]; then
+    # run unstable tests which are targeted by focused changes on PRs
+    export UNSTABLE="--allow-unstable-changed"
+else
+    # do not run unstable tests outside PRs
+    export UNSTABLE=""
+fi
+
 # remove empty core/extras module directories from PRs created prior to the repo-merge
 find lib/ansible/modules -type d -empty -print -delete
 

test/utils/shippable/windows.sh

@@ -24,7 +24,8 @@ python_versions=(
 single_version=2012-R2
 
 # shellcheck disable=SC2086
-ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
+ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
+    | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
 
 if [ -s /tmp/windows.txt ] || [ "${CHANGED:+$CHANGED}" == "" ]; then
     echo "Detected changes requiring integration tests specific to Windows:"
@@ -84,7 +85,8 @@ for version in "${python_versions[@]}"; do
     fi
 
     # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${ci}" --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    ansible-test windows-integration --color -v --retry-on-error "${ci}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
         "${platforms[@]}" --changed-all-target "${changed_all_target}" \
+        --docker default --python "${version}" \
         --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
 done