269 lines
9.8 KiB
Python
269 lines
9.8 KiB
Python
"""Support code for working with Shippable."""
|
|
from __future__ import (absolute_import, division, print_function)
|
|
__metaclass__ = type
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from .. import types as t
|
|
|
|
from ..config import (
|
|
CommonConfig,
|
|
TestConfig,
|
|
)
|
|
|
|
from ..git import (
|
|
Git,
|
|
)
|
|
|
|
from ..http import (
|
|
HttpClient,
|
|
urlencode,
|
|
)
|
|
|
|
from ..util import (
|
|
ApplicationError,
|
|
display,
|
|
MissingEnvironmentVariable,
|
|
SubprocessError,
|
|
)
|
|
|
|
from . import (
|
|
AuthContext,
|
|
ChangeDetectionNotSupported,
|
|
CIProvider,
|
|
OpenSSLAuthHelper,
|
|
)
|
|
|
|
|
|
CODE = 'shippable'
|
|
|
|
|
|
class Shippable(CIProvider):
|
|
"""CI provider implementation for Shippable."""
|
|
def __init__(self):
|
|
self.auth = ShippableAuthHelper()
|
|
|
|
@staticmethod
|
|
def is_supported(): # type: () -> bool
|
|
"""Return True if this provider is supported in the current running environment."""
|
|
return os.environ.get('SHIPPABLE') == 'true'
|
|
|
|
@property
|
|
def code(self): # type: () -> str
|
|
"""Return a unique code representing this provider."""
|
|
return CODE
|
|
|
|
@property
|
|
def name(self): # type: () -> str
|
|
"""Return descriptive name for this provider."""
|
|
return 'Shippable'
|
|
|
|
def generate_resource_prefix(self): # type: () -> str
|
|
"""Return a resource prefix specific to this CI provider."""
|
|
try:
|
|
prefix = 'shippable-%s-%s' % (
|
|
os.environ['SHIPPABLE_BUILD_NUMBER'],
|
|
os.environ['SHIPPABLE_JOB_NUMBER'],
|
|
)
|
|
except KeyError as ex:
|
|
raise MissingEnvironmentVariable(name=ex.args[0])
|
|
|
|
return prefix
|
|
|
|
def get_base_branch(self): # type: () -> str
|
|
"""Return the base branch or an empty string."""
|
|
base_branch = os.environ.get('BASE_BRANCH')
|
|
|
|
if base_branch:
|
|
base_branch = 'origin/%s' % base_branch
|
|
|
|
return base_branch or ''
|
|
|
|
def detect_changes(self, args): # type: (TestConfig) -> t.Optional[t.List[str]]
|
|
"""Initialize change detection."""
|
|
result = ShippableChanges(args)
|
|
|
|
if result.is_pr:
|
|
job_type = 'pull request'
|
|
elif result.is_tag:
|
|
job_type = 'tag'
|
|
else:
|
|
job_type = 'merge commit'
|
|
|
|
display.info('Processing %s for branch %s commit %s' % (job_type, result.branch, result.commit))
|
|
|
|
if not args.metadata.changes:
|
|
args.metadata.populate_changes(result.diff)
|
|
|
|
if result.paths is None:
|
|
# There are several likely causes of this:
|
|
# - First run on a new branch.
|
|
# - Too many pull requests passed since the last merge run passed.
|
|
display.warning('No successful commit found. All tests will be executed.')
|
|
|
|
return result.paths
|
|
|
|
def supports_core_ci_auth(self, context): # type: (AuthContext) -> bool
|
|
"""Return True if Ansible Core CI is supported."""
|
|
return True
|
|
|
|
def prepare_core_ci_auth(self, context): # type: (AuthContext) -> t.Dict[str, t.Any]
|
|
"""Return authentication details for Ansible Core CI."""
|
|
try:
|
|
request = dict(
|
|
run_id=os.environ['SHIPPABLE_BUILD_ID'],
|
|
job_number=int(os.environ['SHIPPABLE_JOB_NUMBER']),
|
|
)
|
|
except KeyError as ex:
|
|
raise MissingEnvironmentVariable(name=ex.args[0])
|
|
|
|
self.auth.sign_request(request)
|
|
|
|
auth = dict(
|
|
shippable=request,
|
|
)
|
|
|
|
return auth
|
|
|
|
def get_git_details(self, args): # type: (CommonConfig) -> t.Optional[t.Dict[str, t.Any]]
|
|
"""Return details about git in the current environment."""
|
|
commit = os.environ.get('COMMIT')
|
|
base_commit = os.environ.get('BASE_COMMIT')
|
|
|
|
details = dict(
|
|
base_commit=base_commit,
|
|
commit=commit,
|
|
merged_commit=self._get_merged_commit(args, commit),
|
|
)
|
|
|
|
return details
|
|
|
|
# noinspection PyUnusedLocal
|
|
def _get_merged_commit(self, args, commit): # type: (CommonConfig, str) -> t.Optional[str] # pylint: disable=unused-argument
|
|
"""Find the merged commit that should be present."""
|
|
if not commit:
|
|
return None
|
|
|
|
git = Git()
|
|
|
|
try:
|
|
show_commit = git.run_git(['show', '--no-patch', '--no-abbrev', commit])
|
|
except SubprocessError as ex:
|
|
# This should only fail for pull requests where the commit does not exist.
|
|
# Merge runs would fail much earlier when attempting to checkout the commit.
|
|
raise ApplicationError('Commit %s was not found:\n\n%s\n\n'
|
|
'GitHub may not have fully replicated the commit across their infrastructure.\n'
|
|
'It is also possible the commit was removed by a force push between job creation and execution.\n'
|
|
'Find the latest run for the pull request and restart failed jobs as needed.'
|
|
% (commit, ex.stderr.strip()))
|
|
|
|
head_commit = git.run_git(['show', '--no-patch', '--no-abbrev', 'HEAD'])
|
|
|
|
if show_commit == head_commit:
|
|
# Commit is HEAD, so this is not a pull request or the base branch for the pull request is up-to-date.
|
|
return None
|
|
|
|
match_merge = re.search(r'^Merge: (?P<parents>[0-9a-f]{40} [0-9a-f]{40})$', head_commit, flags=re.MULTILINE)
|
|
|
|
if not match_merge:
|
|
# The most likely scenarios resulting in a failure here are:
|
|
# A new run should or does supersede this job, but it wasn't cancelled in time.
|
|
# A job was superseded and then later restarted.
|
|
raise ApplicationError('HEAD is not commit %s or a merge commit:\n\n%s\n\n'
|
|
'This job has likely been superseded by another run due to additional commits being pushed.\n'
|
|
'Find the latest run for the pull request and restart failed jobs as needed.'
|
|
% (commit, head_commit.strip()))
|
|
|
|
parents = set(match_merge.group('parents').split(' '))
|
|
|
|
if len(parents) != 2:
|
|
raise ApplicationError('HEAD is a %d-way octopus merge.' % len(parents))
|
|
|
|
if commit not in parents:
|
|
raise ApplicationError('Commit %s is not a parent of HEAD.' % commit)
|
|
|
|
parents.remove(commit)
|
|
|
|
last_commit = parents.pop()
|
|
|
|
return last_commit
|
|
|
|
|
|
class ShippableAuthHelper(OpenSSLAuthHelper):
|
|
"""
|
|
Authentication helper for Shippable.
|
|
Based on OpenSSL since cryptography is not provided by the default Shippable environment.
|
|
"""
|
|
def publish_public_key(self, public_key_pem): # type: (str) -> None
|
|
"""Publish the given public key."""
|
|
# display the public key as a single line to avoid mangling such as when prefixing each line with a timestamp
|
|
display.info(public_key_pem.replace('\n', ' '))
|
|
# allow time for logs to become available to reduce repeated API calls
|
|
time.sleep(3)
|
|
|
|
|
|
class ShippableChanges:
|
|
"""Change information for Shippable build."""
|
|
def __init__(self, args): # type: (CommonConfig) -> None
|
|
self.args = args
|
|
self.git = Git()
|
|
|
|
try:
|
|
self.branch = os.environ['BRANCH']
|
|
self.is_pr = os.environ['IS_PULL_REQUEST'] == 'true'
|
|
self.is_tag = os.environ['IS_GIT_TAG'] == 'true'
|
|
self.commit = os.environ['COMMIT']
|
|
self.project_id = os.environ['PROJECT_ID']
|
|
self.commit_range = os.environ['SHIPPABLE_COMMIT_RANGE']
|
|
except KeyError as ex:
|
|
raise MissingEnvironmentVariable(name=ex.args[0])
|
|
|
|
if self.is_tag:
|
|
raise ChangeDetectionNotSupported('Change detection is not supported for tags.')
|
|
|
|
if self.is_pr:
|
|
self.paths = sorted(self.git.get_diff_names([self.commit_range]))
|
|
self.diff = self.git.get_diff([self.commit_range])
|
|
else:
|
|
commits = self.get_successful_merge_run_commits(self.project_id, self.branch)
|
|
last_successful_commit = self.get_last_successful_commit(commits)
|
|
|
|
if last_successful_commit:
|
|
self.paths = sorted(self.git.get_diff_names([last_successful_commit, self.commit]))
|
|
self.diff = self.git.get_diff([last_successful_commit, self.commit])
|
|
else:
|
|
# first run for branch
|
|
self.paths = None # act as though change detection not enabled, do not filter targets
|
|
self.diff = []
|
|
|
|
def get_successful_merge_run_commits(self, project_id, branch): # type: (str, str) -> t.Set[str]
|
|
"""Return a set of recent successsful merge commits from Shippable for the given project and branch."""
|
|
parameters = dict(
|
|
isPullRequest='false',
|
|
projectIds=project_id,
|
|
branch=branch,
|
|
)
|
|
|
|
url = 'https://api.shippable.com/runs?%s' % urlencode(parameters)
|
|
|
|
http = HttpClient(self.args, always=True)
|
|
response = http.get(url)
|
|
result = response.json()
|
|
|
|
if 'id' in result and result['id'] == 4004:
|
|
# most likely due to a private project, which returns an HTTP 200 response with JSON
|
|
display.warning('Unable to find project. Cannot determine changes. All tests will be executed.')
|
|
return set()
|
|
|
|
commits = set(run['commitSha'] for run in result if run['statusCode'] == 30)
|
|
|
|
return commits
|
|
|
|
def get_last_successful_commit(self, successful_commits): # type: (t.Set[str]) -> t.Optional[str]
|
|
"""Return the last successful commit from git history that is found in the given commit list, or None."""
|
|
commit_history = self.git.get_rev_list(max_count=100)
|
|
ordered_successful_commits = [commit for commit in commit_history if commit in successful_commits]
|
|
last_successful_commit = ordered_successful_commits[0] if ordered_successful_commits else None
|
|
return last_successful_commit
|