a81dd4f06a
* Move ansible-test completion code. * Fix a few type hints. * Change docker completion based on context. Collections now use version 2.0 of the default-test-container. This is an updated version of the earlier 1.x default-test-container with ansible-base and cloud specific requirements removed. Testing of ansible-base now uses version 1.0 of a new ansible-base-test-container. This container is similar to the earlier 1.x default-test-container, but with unnecessary duplication of requirements across Python versions removed. Collections which have tests that depend on requirements no longer present in the default test container should specify them in their test requirements files: * tests/integration/requirements.txt * tests/unit/requirements.txt * Bump test container versions Co-authored-by: Jordan Borean <jborean93@gmail.com>
270 lines
9.8 KiB
Python
270 lines
9.8 KiB
Python
"""Support code for working with Shippable."""
|
|
from __future__ import (absolute_import, division, print_function)
|
|
__metaclass__ = type
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from .. import types as t
|
|
|
|
from ..config import (
|
|
CommonConfig,
|
|
TestConfig,
|
|
)
|
|
|
|
from ..git import (
|
|
Git,
|
|
)
|
|
|
|
from ..http import (
|
|
HttpClient,
|
|
urlencode,
|
|
)
|
|
|
|
from ..util import (
|
|
ApplicationError,
|
|
display,
|
|
MissingEnvironmentVariable,
|
|
SubprocessError,
|
|
)
|
|
|
|
from . import (
|
|
AuthContext,
|
|
ChangeDetectionNotSupported,
|
|
CIProvider,
|
|
OpenSSLAuthHelper,
|
|
)
|
|
|
|
|
|
CODE = 'shippable'
|
|
|
|
|
|
class Shippable(CIProvider):
|
|
"""CI provider implementation for Shippable."""
|
|
def __init__(self):
|
|
self.auth = ShippableAuthHelper()
|
|
|
|
@staticmethod
|
|
def is_supported(): # type: () -> bool
|
|
"""Return True if this provider is supported in the current running environment."""
|
|
return os.environ.get('SHIPPABLE') == 'true'
|
|
|
|
@property
|
|
def code(self): # type: () -> str
|
|
"""Return a unique code representing this provider."""
|
|
return CODE
|
|
|
|
@property
|
|
def name(self): # type: () -> str
|
|
"""Return descriptive name for this provider."""
|
|
return 'Shippable'
|
|
|
|
def generate_resource_prefix(self): # type: () -> str
|
|
"""Return a resource prefix specific to this CI provider."""
|
|
try:
|
|
prefix = 'shippable-%s-%s' % (
|
|
os.environ['SHIPPABLE_BUILD_NUMBER'],
|
|
os.environ['SHIPPABLE_JOB_NUMBER'],
|
|
)
|
|
except KeyError as ex:
|
|
raise MissingEnvironmentVariable(name=ex.args[0])
|
|
|
|
return prefix
|
|
|
|
def get_base_branch(self): # type: () -> str
|
|
"""Return the base branch or an empty string."""
|
|
base_branch = os.environ.get('BASE_BRANCH')
|
|
|
|
if base_branch:
|
|
base_branch = 'origin/%s' % base_branch
|
|
|
|
return base_branch or ''
|
|
|
|
def detect_changes(self, args): # type: (TestConfig) -> t.Optional[t.List[str]]
|
|
"""Initialize change detection."""
|
|
result = ShippableChanges(args)
|
|
|
|
if result.is_pr:
|
|
job_type = 'pull request'
|
|
elif result.is_tag:
|
|
job_type = 'tag'
|
|
else:
|
|
job_type = 'merge commit'
|
|
|
|
display.info('Processing %s for branch %s commit %s' % (job_type, result.branch, result.commit))
|
|
|
|
if not args.metadata.changes:
|
|
args.metadata.populate_changes(result.diff)
|
|
|
|
if result.paths is None:
|
|
# There are several likely causes of this:
|
|
# - First run on a new branch.
|
|
# - Too many pull requests passed since the last merge run passed.
|
|
display.warning('No successful commit found. All tests will be executed.')
|
|
|
|
return result.paths
|
|
|
|
def supports_core_ci_auth(self, context): # type: (AuthContext) -> bool
|
|
"""Return True if Ansible Core CI is supported."""
|
|
return True
|
|
|
|
def prepare_core_ci_auth(self, context): # type: (AuthContext) -> t.Dict[str, t.Any]
|
|
"""Return authentication details for Ansible Core CI."""
|
|
try:
|
|
request = dict(
|
|
run_id=os.environ['SHIPPABLE_BUILD_ID'],
|
|
job_number=int(os.environ['SHIPPABLE_JOB_NUMBER']),
|
|
)
|
|
except KeyError as ex:
|
|
raise MissingEnvironmentVariable(name=ex.args[0])
|
|
|
|
self.auth.sign_request(request)
|
|
|
|
auth = dict(
|
|
shippable=request,
|
|
)
|
|
|
|
return auth
|
|
|
|
def get_git_details(self, args): # type: (CommonConfig) -> t.Optional[t.Dict[str, t.Any]]
|
|
"""Return details about git in the current environment."""
|
|
commit = os.environ.get('COMMIT')
|
|
base_commit = os.environ.get('BASE_COMMIT')
|
|
|
|
details = dict(
|
|
base_commit=base_commit,
|
|
commit=commit,
|
|
merged_commit=self._get_merged_commit(args, commit),
|
|
)
|
|
|
|
return details
|
|
|
|
# noinspection PyUnusedLocal
|
|
def _get_merged_commit(self, args, commit): # type: (CommonConfig, str) -> t.Optional[str] # pylint: disable=unused-argument
|
|
"""Find the merged commit that should be present."""
|
|
if not commit:
|
|
return None
|
|
|
|
git = Git()
|
|
|
|
try:
|
|
show_commit = git.run_git(['show', '--no-patch', '--no-abbrev', commit])
|
|
except SubprocessError as ex:
|
|
# This should only fail for pull requests where the commit does not exist.
|
|
# Merge runs would fail much earlier when attempting to checkout the commit.
|
|
raise ApplicationError('Commit %s was not found:\n\n%s\n\n'
|
|
'GitHub may not have fully replicated the commit across their infrastructure.\n'
|
|
'It is also possible the commit was removed by a force push between job creation and execution.\n'
|
|
'Find the latest run for the pull request and restart failed jobs as needed.'
|
|
% (commit, ex.stderr.strip()))
|
|
|
|
head_commit = git.run_git(['show', '--no-patch', '--no-abbrev', 'HEAD'])
|
|
|
|
if show_commit == head_commit:
|
|
# Commit is HEAD, so this is not a pull request or the base branch for the pull request is up-to-date.
|
|
return None
|
|
|
|
match_merge = re.search(r'^Merge: (?P<parents>[0-9a-f]{40} [0-9a-f]{40})$', head_commit, flags=re.MULTILINE)
|
|
|
|
if not match_merge:
|
|
# The most likely scenarios resulting in a failure here are:
|
|
# A new run should or does supersede this job, but it wasn't cancelled in time.
|
|
# A job was superseded and then later restarted.
|
|
raise ApplicationError('HEAD is not commit %s or a merge commit:\n\n%s\n\n'
|
|
'This job has likely been superseded by another run due to additional commits being pushed.\n'
|
|
'Find the latest run for the pull request and restart failed jobs as needed.'
|
|
% (commit, head_commit.strip()))
|
|
|
|
parents = set(match_merge.group('parents').split(' '))
|
|
|
|
if len(parents) != 2:
|
|
raise ApplicationError('HEAD is a %d-way octopus merge.' % len(parents))
|
|
|
|
if commit not in parents:
|
|
raise ApplicationError('Commit %s is not a parent of HEAD.' % commit)
|
|
|
|
parents.remove(commit)
|
|
|
|
last_commit = parents.pop()
|
|
|
|
return last_commit
|
|
|
|
|
|
class ShippableAuthHelper(OpenSSLAuthHelper):
|
|
"""
|
|
Authentication helper for Shippable.
|
|
Based on OpenSSL since cryptography is not provided by the default Shippable environment.
|
|
"""
|
|
def publish_public_key(self, public_key_pem): # type: (str) -> None
|
|
"""Publish the given public key."""
|
|
# display the public key as a single line to avoid mangling such as when prefixing each line with a timestamp
|
|
display.info(public_key_pem.replace('\n', ' '))
|
|
# allow time for logs to become available to reduce repeated API calls
|
|
time.sleep(3)
|
|
|
|
|
|
class ShippableChanges:
|
|
"""Change information for Shippable build."""
|
|
def __init__(self, args): # type: (TestConfig) -> None
|
|
self.args = args
|
|
self.git = Git()
|
|
|
|
try:
|
|
self.branch = os.environ['BRANCH']
|
|
self.is_pr = os.environ['IS_PULL_REQUEST'] == 'true'
|
|
self.is_tag = os.environ['IS_GIT_TAG'] == 'true'
|
|
self.commit = os.environ['COMMIT']
|
|
self.project_id = os.environ['PROJECT_ID']
|
|
self.commit_range = os.environ['SHIPPABLE_COMMIT_RANGE']
|
|
except KeyError as ex:
|
|
raise MissingEnvironmentVariable(name=ex.args[0])
|
|
|
|
if self.is_tag:
|
|
raise ChangeDetectionNotSupported('Change detection is not supported for tags.')
|
|
|
|
if self.is_pr:
|
|
self.paths = sorted(self.git.get_diff_names([self.commit_range]))
|
|
self.diff = self.git.get_diff([self.commit_range])
|
|
else:
|
|
commits = self.get_successful_merge_run_commits(self.project_id, self.branch)
|
|
last_successful_commit = self.get_last_successful_commit(commits)
|
|
|
|
if last_successful_commit:
|
|
self.paths = sorted(self.git.get_diff_names([last_successful_commit, self.commit]))
|
|
self.diff = self.git.get_diff([last_successful_commit, self.commit])
|
|
else:
|
|
# first run for branch
|
|
self.paths = None # act as though change detection not enabled, do not filter targets
|
|
self.diff = []
|
|
|
|
def get_successful_merge_run_commits(self, project_id, branch): # type: (str, str) -> t.Set[str]
|
|
"""Return a set of recent successsful merge commits from Shippable for the given project and branch."""
|
|
parameters = dict(
|
|
isPullRequest='false',
|
|
projectIds=project_id,
|
|
branch=branch,
|
|
)
|
|
|
|
url = 'https://api.shippable.com/runs?%s' % urlencode(parameters)
|
|
|
|
http = HttpClient(self.args, always=True)
|
|
response = http.get(url)
|
|
result = response.json()
|
|
|
|
if 'id' in result and result['id'] == 4004:
|
|
# most likely due to a private project, which returns an HTTP 200 response with JSON
|
|
display.warning('Unable to find project. Cannot determine changes. All tests will be executed.')
|
|
return set()
|
|
|
|
commits = set(run['commitSha'] for run in result if run['statusCode'] == 30)
|
|
|
|
return commits
|
|
|
|
def get_last_successful_commit(self, successful_commits): # type: (t.Set[str]) -> t.Optional[str]
|
|
"""Return the last successful commit from git history that is found in the given commit list, or None."""
|
|
commit_history = self.git.get_rev_list(max_count=100)
|
|
ordered_successful_commits = [commit for commit in commit_history if commit in successful_commits]
|
|
last_successful_commit = ordered_successful_commits[0] if ordered_successful_commits else None
|
|
return last_successful_commit
|