diff --git a/.gitignore b/.gitignore index 3ccf9e1ae27..e0c057b05a3 100644 --- a/.gitignore +++ b/.gitignore @@ -84,7 +84,6 @@ Vagrantfile /lib/ansible_base.egg-info/ # First used in the `devel` branch during Ansible 2.11 development. /lib/ansible_core.egg-info/ -/shippable/ /test/integration/cloud-config-*.* !/test/integration/cloud-config-*.*.template .python-version diff --git a/Makefile b/Makefile index 7fb44ba1e6c..0ab82f78742 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,6 @@ clean: rm -f ./docs/man/man1/* @echo "Cleaning up output from test runs" rm -rf test/test_data - rm -rf shippable/ rm -rf logs/ rm -rf .cache/ rm -f test/units/.coverage* diff --git a/changelogs/fragments/ansible-test-remove-shippable.yml b/changelogs/fragments/ansible-test-remove-shippable.yml new file mode 100644 index 00000000000..45bab6353c1 --- /dev/null +++ b/changelogs/fragments/ansible-test-remove-shippable.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Remove CI provider support for Shippable, now that the service has been discontinued. diff --git a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py index 6ec22755fb4..23a279deba1 100644 --- a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py +++ b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py @@ -260,7 +260,6 @@ class Validator(with_metaclass(abc.ABCMeta, object)): class ModuleValidator(Validator): REJECTLIST_PATTERNS = ('.git*', '*.pyc', '*.pyo', '.*', '*.md', '*.rst', '*.txt') REJECTLIST_FILES = frozenset(('.git', '.gitignore', '.travis.yml', - 'shippable.yml', '.gitattributes', '.gitmodules', 'COPYING', '__init__.py', 'VERSION', 'test-docs.sh')) REJECTLIST = REJECTLIST_FILES.union(REJECTLIST['MODULE']) diff --git a/test/lib/ansible_test/_internal/ci/shippable.py b/test/lib/ansible_test/_internal/ci/shippable.py deleted file mode 100644 index f9f0a1928f9..00000000000 --- a/test/lib/ansible_test/_internal/ci/shippable.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Support code for working with Shippable.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import os -import re -import time - -from .. import types as t - -from ..config import ( - CommonConfig, - TestConfig, -) - -from ..git import ( - Git, -) - -from ..http import ( - HttpClient, - urlencode, -) - -from ..util import ( - ApplicationError, - display, - MissingEnvironmentVariable, - SubprocessError, -) - -from . import ( - AuthContext, - ChangeDetectionNotSupported, - CIProvider, - OpenSSLAuthHelper, -) - - -CODE = 'shippable' - - -class Shippable(CIProvider): - """CI provider implementation for Shippable.""" - def __init__(self): - self.auth = ShippableAuthHelper() - - @staticmethod - def is_supported(): # type: () -> bool - """Return True if this provider is supported in the current running environment.""" - return os.environ.get('SHIPPABLE') == 'true' - - @property - def code(self): # type: () -> str - """Return a unique code representing this provider.""" - return CODE - - @property - def name(self): # type: () -> str - """Return descriptive name for this provider.""" - return 'Shippable' - - def generate_resource_prefix(self): # type: () -> str - """Return a resource prefix specific to this CI provider.""" - try: - prefix = 'shippable-%s-%s' % ( - os.environ['SHIPPABLE_BUILD_NUMBER'], - os.environ['SHIPPABLE_JOB_NUMBER'], - ) - except KeyError as ex: - raise MissingEnvironmentVariable(name=ex.args[0]) - - return prefix - - def get_base_branch(self): # type: () -> str - """Return the base branch or an empty string.""" - base_branch = os.environ.get('BASE_BRANCH') - - if base_branch: - base_branch = 'origin/%s' % base_branch - - return base_branch or '' - - def detect_changes(self, args): # type: (TestConfig) -> t.Optional[t.List[str]] - """Initialize change detection.""" - result = ShippableChanges(args) - - if result.is_pr: - job_type = 'pull request' - elif result.is_tag: - job_type = 'tag' - else: - job_type = 'merge commit' - - display.info('Processing %s for branch %s commit %s' % (job_type, result.branch, result.commit)) - - if not args.metadata.changes: - args.metadata.populate_changes(result.diff) - - if result.paths is None: - # There are several likely causes of this: - # - First run on a new branch. - # - Too many pull requests passed since the last merge run passed. - display.warning('No successful commit found. All tests will be executed.') - - return result.paths - - def supports_core_ci_auth(self, context): # type: (AuthContext) -> bool - """Return True if Ansible Core CI is supported.""" - return True - - def prepare_core_ci_auth(self, context): # type: (AuthContext) -> t.Dict[str, t.Any] - """Return authentication details for Ansible Core CI.""" - try: - request = dict( - run_id=os.environ['SHIPPABLE_BUILD_ID'], - job_number=int(os.environ['SHIPPABLE_JOB_NUMBER']), - ) - except KeyError as ex: - raise MissingEnvironmentVariable(name=ex.args[0]) - - self.auth.sign_request(request) - - auth = dict( - shippable=request, - ) - - return auth - - def get_git_details(self, args): # type: (CommonConfig) -> t.Optional[t.Dict[str, t.Any]] - """Return details about git in the current environment.""" - commit = os.environ.get('COMMIT') - base_commit = os.environ.get('BASE_COMMIT') - - details = dict( - base_commit=base_commit, - commit=commit, - merged_commit=self._get_merged_commit(args, commit), - ) - - return details - - # noinspection PyUnusedLocal - def _get_merged_commit(self, args, commit): # type: (CommonConfig, str) -> t.Optional[str] # pylint: disable=unused-argument - """Find the merged commit that should be present.""" - if not commit: - return None - - git = Git() - - try: - show_commit = git.run_git(['show', '--no-patch', '--no-abbrev', commit]) - except SubprocessError as ex: - # This should only fail for pull requests where the commit does not exist. - # Merge runs would fail much earlier when attempting to checkout the commit. - raise ApplicationError('Commit %s was not found:\n\n%s\n\n' - 'GitHub may not have fully replicated the commit across their infrastructure.\n' - 'It is also possible the commit was removed by a force push between job creation and execution.\n' - 'Find the latest run for the pull request and restart failed jobs as needed.' - % (commit, ex.stderr.strip())) - - head_commit = git.run_git(['show', '--no-patch', '--no-abbrev', 'HEAD']) - - if show_commit == head_commit: - # Commit is HEAD, so this is not a pull request or the base branch for the pull request is up-to-date. - return None - - match_merge = re.search(r'^Merge: (?P[0-9a-f]{40} [0-9a-f]{40})$', head_commit, flags=re.MULTILINE) - - if not match_merge: - # The most likely scenarios resulting in a failure here are: - # A new run should or does supersede this job, but it wasn't cancelled in time. - # A job was superseded and then later restarted. - raise ApplicationError('HEAD is not commit %s or a merge commit:\n\n%s\n\n' - 'This job has likely been superseded by another run due to additional commits being pushed.\n' - 'Find the latest run for the pull request and restart failed jobs as needed.' - % (commit, head_commit.strip())) - - parents = set(match_merge.group('parents').split(' ')) - - if len(parents) != 2: - raise ApplicationError('HEAD is a %d-way octopus merge.' % len(parents)) - - if commit not in parents: - raise ApplicationError('Commit %s is not a parent of HEAD.' % commit) - - parents.remove(commit) - - last_commit = parents.pop() - - return last_commit - - -class ShippableAuthHelper(OpenSSLAuthHelper): - """ - Authentication helper for Shippable. - Based on OpenSSL since cryptography is not provided by the default Shippable environment. - """ - def publish_public_key(self, public_key_pem): # type: (str) -> None - """Publish the given public key.""" - # display the public key as a single line to avoid mangling such as when prefixing each line with a timestamp - display.info(public_key_pem.replace('\n', ' ')) - # allow time for logs to become available to reduce repeated API calls - time.sleep(3) - - -class ShippableChanges: - """Change information for Shippable build.""" - def __init__(self, args): # type: (TestConfig) -> None - self.args = args - self.git = Git() - - try: - self.branch = os.environ['BRANCH'] - self.is_pr = os.environ['IS_PULL_REQUEST'] == 'true' - self.is_tag = os.environ['IS_GIT_TAG'] == 'true' - self.commit = os.environ['COMMIT'] - self.project_id = os.environ['PROJECT_ID'] - self.commit_range = os.environ['SHIPPABLE_COMMIT_RANGE'] - except KeyError as ex: - raise MissingEnvironmentVariable(name=ex.args[0]) - - if self.is_tag: - raise ChangeDetectionNotSupported('Change detection is not supported for tags.') - - if self.is_pr: - self.paths = sorted(self.git.get_diff_names([self.commit_range])) - self.diff = self.git.get_diff([self.commit_range]) - else: - commits = self.get_successful_merge_run_commits(self.project_id, self.branch) - last_successful_commit = self.get_last_successful_commit(commits) - - if last_successful_commit: - self.paths = sorted(self.git.get_diff_names([last_successful_commit, self.commit])) - self.diff = self.git.get_diff([last_successful_commit, self.commit]) - else: - # first run for branch - self.paths = None # act as though change detection not enabled, do not filter targets - self.diff = [] - - def get_successful_merge_run_commits(self, project_id, branch): # type: (str, str) -> t.Set[str] - """Return a set of recent successsful merge commits from Shippable for the given project and branch.""" - parameters = dict( - isPullRequest='false', - projectIds=project_id, - branch=branch, - ) - - url = 'https://api.shippable.com/runs?%s' % urlencode(parameters) - - http = HttpClient(self.args, always=True) - response = http.get(url) - result = response.json() - - if 'id' in result and result['id'] == 4004: - # most likely due to a private project, which returns an HTTP 200 response with JSON - display.warning('Unable to find project. Cannot determine changes. All tests will be executed.') - return set() - - commits = set(run['commitSha'] for run in result if run['statusCode'] == 30) - - return commits - - def get_last_successful_commit(self, successful_commits): # type: (t.Set[str]) -> t.Optional[str] - """Return the last successful commit from git history that is found in the given commit list, or None.""" - commit_history = self.git.get_rev_list(max_count=100) - ordered_successful_commits = [commit for commit in commit_history if commit in successful_commits] - last_successful_commit = ordered_successful_commits[0] if ordered_successful_commits else None - return last_successful_commit diff --git a/test/lib/ansible_test/_internal/classification.py b/test/lib/ansible_test/_internal/classification.py index b94dbf45828..612760b6c22 100644 --- a/test/lib/ansible_test/_internal/classification.py +++ b/test/lib/ansible_test/_internal/classification.py @@ -380,7 +380,6 @@ class PathMapper: if os.path.sep not in path: if filename in ( 'azure-pipelines.yml', - 'shippable.yml', ): return all_tests(self.args) # test infrastructure, run all tests diff --git a/test/units/ansible_test/ci/test_shippable.py b/test/units/ansible_test/ci/test_shippable.py deleted file mode 100644 index 08b276c7b20..00000000000 --- a/test/units/ansible_test/ci/test_shippable.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -from .util import common_auth_test - - -def test_auth(): - # noinspection PyProtectedMember - from ansible_test._internal.ci.shippable import ( - ShippableAuthHelper, - ) - - class TestShippableAuthHelper(ShippableAuthHelper): - def __init__(self): - self.public_key_pem = None - self.private_key_pem = None - - def publish_public_key(self, public_key_pem): - # avoid publishing key - self.public_key_pem = public_key_pem - - def initialize_private_key(self): - # cache in memory instead of on disk - if not self.private_key_pem: - self.private_key_pem = self.generate_private_key() - - return self.private_key_pem - - auth = TestShippableAuthHelper() - - common_auth_test(auth)