cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Retrofit shippable scripts to work with AZP

Browse source 
Co-authored-by: Matt Clay <matt@mystile.com>
This commit is contained in:
Alexander Sowitzki 2021-04-01 14:55:09 +02:00 committed by Matt Clay
parent 57d661e96f
commit 26214788ee
5 changed files with 197 additions and 407 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
hacking/shippable/README.md
View file

@ -4,10 +4,11 @@
This directory contains the following scripts: This directory contains the following scripts:
- download.py - Download results from Shippable. - download.py - Download results from CI.
- get_recent_coverage_runs.py - Retrieve Shippable URLs of recent coverage test runs. - get_recent_coverage_runs.py - Retrieve CI URLs of recent coverage test runs.
- incidental.py - Report on incidental code coverage using data from Shippable. - incidental.py - Report on incidental code coverage using data from CI.
- run.py - Start new runs on Shippable. - run.py - Start new runs on CI.
- rebalance.py - Re-balance CI group(s) from a downloaded results directory.
## Incidental Code Coverage ## Incidental Code Coverage
@ -31,14 +32,14 @@ As additional intentional tests are added, the exclusive coverage provided by in
Reducing incidental test coverage, and eventually removing incidental tests involves the following process: Reducing incidental test coverage, and eventually removing incidental tests involves the following process:
1. Run the entire test suite with code coverage enabled. 1. Run the entire test suite with code coverage enabled.
This is done automatically each day on Shippable. This is done automatically each day on Shippable.
The URLs and statuses of the most recent such test runs can be found with: The URLs and statuses of the most recent such test runs can be found with:
```shell ```shell
hacking/shippable/get_recent_coverage_runs.py <optional branch name> hacking/shippable/get_recent_coverage_runs.py <optional branch name>
``` ```
The branch name defaults to `devel`. The branch name defaults to `devel`.
2. Download code coverage data from Shippable for local analysis. 2. Download code coverage data from Shippable for local analysis.
Example: Example:
```shell ```shell
# download results to ansible/ansible directory under cwd # download results to ansible/ansible directory under cwd

377
hacking/shippable/download.py
View file

@ -18,6 +18,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""CLI tool for downloading results from Shippable CI runs.""" """CLI tool for downloading results from Shippable CI runs."""
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
@ -27,6 +28,8 @@ import json
import os import os
import re import re
import sys import sys
import io
import zipfile
import requests import requests
@ -35,22 +38,33 @@ try:
except ImportError: except ImportError:
argcomplete = None argcomplete = None
# Following changes should be made to improve the overall style:
# TODO use new style formatting method.
# TODO use requests session.
# TODO type hints.
# TODO pathlib.
def main(): def main():
"""Main program body.""" """Main program body."""
args = parse_args() args = parse_args()
download_run(args) download_run(args)
def run_id_arg(arg):
m = re.fullmatch(r"(?:https:\/\/dev\.azure\.com\/ansible\/ansible\/_build\/results\?buildId=)?(\d+)", arg)
if not m:
raise ValueError("run does not seems to be a URI or an ID")
return m.group(1)
def parse_args(): def parse_args():
"""Parse and return args.""" """Parse and return args."""
api_key = get_api_key()
parser = argparse.ArgumentParser(description='Download results from a Shippable run.') parser = argparse.ArgumentParser(description='Download results from a CI run.')
parser.add_argument('run_id', parser.add_argument('run', metavar='RUN', type=run_id_arg, help='AZP run id or URI')
metavar='RUN',
help='shippable run id, run url or run name formatted as: account/project/run_number')
parser.add_argument('-v', '--verbose', parser.add_argument('-v', '--verbose',
dest='verbose', dest='verbose',
@ -62,28 +76,16 @@ def parse_args():
action='store_true', action='store_true',
help='show what would be downloaded without downloading') help='show what would be downloaded without downloading')
parser.add_argument('--key', parser.add_argument('-p', '--pipeline-id', type=int, default=20, help='pipeline to download the job from')
dest='api_key',
default=api_key, parser.add_argument('--artifacts',
required=api_key is None, action='store_true',
help='api key for accessing Shippable') help='download artifacts')
parser.add_argument('--console-logs', parser.add_argument('--console-logs',
action='store_true', action='store_true',
help='download console logs') help='download console logs')
parser.add_argument('--test-results',
action='store_true',
help='download test results')
parser.add_argument('--coverage-results',
action='store_true',
help='download code coverage results')
parser.add_argument('--job-metadata',
action='store_true',
help='download job metadata')
parser.add_argument('--run-metadata', parser.add_argument('--run-metadata',
action='store_true', action='store_true',
help='download run metadata') help='download run metadata')
@ -92,35 +94,30 @@ def parse_args():
action='store_true', action='store_true',
help='download everything') help='download everything')
parser.add_argument('--job-number', parser.add_argument('--match-artifact-name',
metavar='N', default=re.compile('.*'),
action='append', type=re.compile,
type=int, help='only download artifacts which names match this regex')
help='limit downloads to the given job number')
parser.add_argument('--match-job-name',
default=re.compile('.*'),
type=re.compile,
help='only download artifacts from jobs which names match this regex')
if argcomplete: if argcomplete:
argcomplete.autocomplete(parser) argcomplete.autocomplete(parser)
args = parser.parse_args() args = parser.parse_args()
old_runs_prefix = 'https://app.shippable.com/runs/'
if args.run_id.startswith(old_runs_prefix):
args.run_id = args.run_id[len(old_runs_prefix):]
if args.all: if args.all:
args.console_logs = True args.artifacts = True
args.test_results = True
args.coverage_results = True
args.job_metadata = True
args.run_metadata = True args.run_metadata = True
args.console_logs = True
selections = ( selections = (
args.console_logs, args.artifacts,
args.test_results,
args.coverage_results,
args.job_metadata,
args.run_metadata, args.run_metadata,
args.console_logs
) )
if not any(selections): if not any(selections):
@ -130,256 +127,100 @@ def parse_args():
def download_run(args): def download_run(args):
"""Download a Shippable run.""" """Download a run."""
headers = dict(
Authorization='apiToken %s' % args.api_key,
)
match = re.search( output_dir = '%s' % args.run
r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)(?:/summary|(/(?P<job_number>[0-9]+)))?$',
args.run_id)
if not match: if not args.test and not os.path.exists(output_dir):
match = re.search(r'^(?P<account>[^/]+)/(?P<project>[^/]+)/(?P<run_number>[0-9]+)$', args.run_id) os.makedirs(output_dir)
if match:
account = match.group('account')
project = match.group('project')
run_number = int(match.group('run_number'))
job_number = int(match.group('job_number')) if match.group('job_number') else None
if job_number:
if args.job_number:
sys.exit('ERROR: job number found in url and specified with --job-number')
args.job_number = [job_number]
url = 'https://api.shippable.com/projects'
response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers)
if response.status_code != 200:
raise Exception(response.content)
project_id = response.json()[0]['id']
url = 'https://api.shippable.com/runs?projectIds=%s&runNumbers=%s' % (project_id, run_number)
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
run = [run for run in response.json() if run['runNumber'] == run_number][0]
args.run_id = run['id']
elif re.search('^[a-f0-9]+$', args.run_id):
url = 'https://api.shippable.com/runs/%s' % args.run_id
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
run = response.json()
account = run['subscriptionOrgName']
project = run['projectName']
run_number = run['runNumber']
else:
sys.exit('ERROR: invalid run: %s' % args.run_id)
output_dir = '%s/%s/%s' % (account, project, run_number)
if not args.test:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if args.run_metadata: if args.run_metadata:
run_url = 'https://dev.azure.com/ansible/ansible/_apis/pipelines/%s/runs/%s?api-version=6.0-preview.1' % (args.pipeline_id, args.run)
run_info_response = requests.get(run_url)
run_info_response.raise_for_status()
run = run_info_response.json()
path = os.path.join(output_dir, 'run.json') path = os.path.join(output_dir, 'run.json')
contents = json.dumps(run, sort_keys=True, indent=4) contents = json.dumps(run, sort_keys=True, indent=4)
if args.verbose or args.test: if args.verbose:
print(path) print(path)
if not args.test: if not args.test:
with open(path, 'w') as metadata_fd: with open(path, 'w') as metadata_fd:
metadata_fd.write(contents) metadata_fd.write(contents)
download_run_recursive(args, headers, output_dir, run, True) timeline_response = requests.get('https://dev.azure.com/ansible/ansible/_apis/build/builds/%s/timeline?api-version=6.0' % args.run)
timeline_response.raise_for_status()
timeline = timeline_response.json()
roots = set()
by_id = {}
children_of = {}
parent_of = {}
for r in timeline['records']:
thisId = r['id']
parentId = r['parentId']
by_id[thisId] = r
def download_run_recursive(args, headers, output_dir, run, is_given=False): if parentId is None:
# Notes: roots.add(thisId)
# - The /runs response tells us if we need to eventually go up another layer
# or not (if we are a re-run attempt or not).
# - Given a run id, /jobs will tell us all the jobs in that run, and whether
# or not we can pull results from them.
#
# When we initially run (i.e., in download_run), we'll have a /runs output
# which we can use to get a /jobs output. Using the /jobs output, we filter
# on the jobs we need to fetch (usually only successful ones unless we are
# processing the initial/given run and not one of its parent runs) and
# download them accordingly.
#
# Lastly, we check if the run we are currently processing has another
# parent (reRunBatchId). If it does, we pull that /runs result and
# recurse using it to start the process over again.
response = requests.get('https://api.shippable.com/jobs?runIds=%s' % run['id'], headers=headers)
if response.status_code != 200:
raise Exception(response.content)
jobs = sorted(response.json(), key=lambda job: int(job['jobNumber']))
if is_given:
needed_jobs = [j for j in jobs if j['isConsoleArchived']]
else:
needed_jobs = [j for j in jobs if j['isConsoleArchived'] and j['statusCode'] == 30]
if not args.test:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
download_jobs(args, needed_jobs, headers, output_dir)
rerun_batch_id = run.get('reRunBatchId')
if rerun_batch_id:
print('Downloading previous run: %s' % rerun_batch_id)
response = requests.get('https://api.shippable.com/runs/%s' % rerun_batch_id, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
run = response.json()
download_run_recursive(args, headers, output_dir, run)
def download_jobs(args, jobs, headers, output_dir):
"""Download Shippable jobs."""
for j in jobs:
job_id = j['id']
job_number = j['jobNumber']
if args.job_number and job_number not in args.job_number:
continue
if args.job_metadata:
path = os.path.join(output_dir, '%s/job.json' % job_number)
contents = json.dumps(j, sort_keys=True, indent=4).encode('utf-8')
if args.verbose or args.test:
print(path)
if not args.test:
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'wb') as metadata_fd:
metadata_fd.write(contents)
if args.console_logs:
path = os.path.join(output_dir, '%s/console.log' % job_number)
url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id
download(args, headers, path, url, is_json=False)
if args.test_results:
path = os.path.join(output_dir, '%s/test.json' % job_number)
url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id
download(args, headers, path, url)
extract_contents(args, path, os.path.join(output_dir, '%s/test' % job_number))
if args.coverage_results:
path = os.path.join(output_dir, '%s/coverage.json' % job_number)
url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id
download(args, headers, path, url)
extract_contents(args, path, os.path.join(output_dir, '%s/coverage' % job_number))
def extract_contents(args, path, output_dir):
"""
:type args: any
:type path: str
:type output_dir: str
"""
if not args.test:
if not os.path.exists(path):
return
with open(path, 'r') as json_fd:
items = json.load(json_fd)
for item in items:
contents = item['contents'].encode('utf-8')
path = output_dir + '/' + re.sub('^/*', '', item['path'])
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
if args.verbose:
print(path)
if path.endswith('.json'):
contents = json.dumps(json.loads(contents), sort_keys=True, indent=4).encode('utf-8')
if not os.path.exists(path):
with open(path, 'wb') as output_fd:
output_fd.write(contents)
def download(args, headers, path, url, is_json=True):
"""
:type args: any
:type headers: dict[str, str]
:type path: str
:type url: str
:type is_json: bool
"""
if args.verbose or args.test:
print(path)
if os.path.exists(path):
return
if not args.test:
response = requests.get(url, headers=headers)
if response.status_code != 200:
path += '.error'
if is_json:
content = json.dumps(response.json(), sort_keys=True, indent=4).encode(response.encoding)
else: else:
content = response.content parent_of[thisId] = parentId
children_of[parentId] = children_of.get(parentId, []) + [thisId]
directory = os.path.dirname(path) allowed = set()
if not os.path.exists(directory): def allow_recursive(ei):
os.makedirs(directory) allowed.add(ei)
for ci in children_of.get(ei, []):
allow_recursive(ci)
with open(path, 'wb') as content_fd: for ri in roots:
content_fd.write(content) r = by_id[ri]
allowed.add(ri)
for ci in children_of.get(r['id'], []):
c = by_id[ci]
if not args.match_job_name.match("%s %s" % (r['name'], c['name'])):
continue
allow_recursive(c['id'])
if args.artifacts:
artifact_list_url = 'https://dev.azure.com/ansible/ansible/_apis/build/builds/%s/artifacts?api-version=6.0' % args.run
artifact_list_response = requests.get(artifact_list_url)
artifact_list_response.raise_for_status()
for artifact in artifact_list_response.json()['value']:
if artifact['source'] not in allowed or not args.match_artifact_name.match(artifact['name']):
continue
if args.verbose:
print('%s/%s' % (output_dir, artifact['name']))
if not args.test:
response = requests.get(artifact['resource']['downloadUrl'])
response.raise_for_status()
archive = zipfile.ZipFile(io.BytesIO(response.content))
archive.extractall(path=output_dir)
def get_api_key(): if args.console_logs:
""" for r in timeline['records']:
rtype: str if not r['log'] or r['id'] not in allowed or not args.match_artifact_name.match(r['name']):
""" continue
key = os.environ.get('SHIPPABLE_KEY', None) names = []
parent_id = r['id']
while parent_id is not None:
p = by_id[parent_id]
name = p['name']
if name not in names:
names = [name] + names
parent_id = parent_of.get(p['id'], None)
if key: path = " ".join(names)
return key log_path = os.path.join(output_dir, '%s.log' % path)
if args.verbose:
path = os.path.join(os.environ['HOME'], '.shippable.key') print(log_path)
if not args.test:
try: log = requests.get(r['log']['url'])
with open(path, 'r') as key_fd: log.raise_for_status()
return key_fd.read().strip() open(log_path, 'wb').write(log.content)
except IOError:
return None
if __name__ == '__main__': if __name__ == '__main__':

63
hacking/shippable/get_recent_coverage_runs.py
View file

@ -23,31 +23,48 @@ __metaclass__ = type
from ansible.utils.color import stringc from ansible.utils.color import stringc
import requests import requests
import sys import sys
import datetime
# Following changes should be made to improve the overall style:
# TODO use argparse for arguments.
# TODO use new style formatting method.
# TODO use requests session.
# TODO type hints.
BRANCH = 'devel' BRANCH = 'devel'
PIPELINE_ID = 20
MAX_AGE = datetime.timedelta(hours=24)
if len(sys.argv) > 1: if len(sys.argv) > 1:
BRANCH = sys.argv[1] BRANCH = sys.argv[1]
def get_coverage_runs(): def get_coverage_runs():
response = requests.get( list_response = requests.get("https://dev.azure.com/ansible/ansible/_apis/pipelines/%s/runs?api-version=6.0-preview.1" % PIPELINE_ID)
'https://api.shippable.com/runs?projectIds=573f79d02a8192902e20e34b' list_response.raise_for_status()
'&branch=%s&limit=1000' % BRANCH)
if response.status_code != 200: runs = list_response.json()
raise Exception(response.content)
runs = response.json()
coverage_runs = [] coverage_runs = []
criteria = ['COMPLETE="yes"', 'COVERAGE="yes"'] for run_summary in runs["value"][0:1000]:
run_response = requests.get(run_summary['url'])
run_response.raise_for_status()
run = run_response.json()
for run in runs: if run['resources']['repositories']['self']['refName'] != 'refs/heads/%s' % BRANCH:
injected_vars = run.get('cleanRunYml', {}).get('env', {}).get('injected')
if not injected_vars:
continue continue
if all(criterion in injected_vars for criterion in criteria):
if 'finishedDate' in run_summary:
age = datetime.datetime.now() - datetime.datetime.strptime(run['finishedDate'].split(".")[0], "%Y-%m-%dT%H:%M:%S")
if age > MAX_AGE:
break
artifact_response = requests.get("https://dev.azure.com/ansible/ansible/_apis/build/builds/%s/artifacts?api-version=6.0" % run['id'])
artifact_response.raise_for_status()
artifacts = artifact_response.json()['value']
if any([a["name"].startswith("Coverage") for a in artifacts]):
# TODO wrongfully skipped if all jobs failed.
coverage_runs.append(run) coverage_runs.append(run)
return coverage_runs return coverage_runs
@ -57,29 +74,29 @@ def pretty_coverage_runs(runs):
ended = [] ended = []
in_progress = [] in_progress = []
for run in runs: for run in runs:
if run.get('endedAt'): if run.get('finishedDate'):
ended.append(run) ended.append(run)
else: else:
in_progress.append(run) in_progress.append(run)
for run in sorted(ended, key=lambda x: x['endedAt']): for run in sorted(ended, key=lambda x: x['finishedDate']):
if run['statusCode'] == 30: if run['result'] == "succeeded":
print('🙂 [%s] https://app.shippable.com/github/ansible/ansible/runs/%s (%s)' % ( print('🙂 [%s] https://dev.azure.com/ansible/ansible/_build/results?buildId=%s (%s)' % (
stringc('PASS', 'green'), stringc('PASS', 'green'),
run['runNumber'], run['id'],
run['endedAt'])) run['finishedDate']))
else: else:
print('😢 [%s] https://app.shippable.com/github/ansible/ansible/runs/%s (%s)' % ( print('😢 [%s] https://dev.azure.com/ansible/ansible/_build/results?buildId=%s (%s)' % (
stringc('FAIL', 'red'), stringc('FAIL', 'red'),
run['runNumber'], run['id'],
run['endedAt'])) run['finishedDate']))
if in_progress: if in_progress:
print('The following runs are ongoing:') print('The following runs are ongoing:')
for run in in_progress: for run in in_progress:
print('🤔 [%s] https://app.shippable.com/github/ansible/ansible/runs/%s' % ( print('🤔 [%s] https://dev.azure.com/ansible/ansible/_build/results?buildId=%s' % (
stringc('FATE', 'yellow'), stringc('FATE', 'yellow'),
run['runNumber'])) run['id']))
def main(): def main():

37
hacking/shippable/incidental.py
View file

@ -37,6 +37,11 @@ try:
except ImportError: except ImportError:
argcomplete = None argcomplete = None
# Following changes should be made to improve the overall style:
# TODO use new style formatting method.
# TODO type hints.
# TODO pathlib.
def main(): def main():
"""Main program body.""" """Main program body."""
@ -132,14 +137,9 @@ def incidental_report(args):
raise ApplicationError('%s: commit not found: %s\n' raise ApplicationError('%s: commit not found: %s\n'
'make sure your source repository is up-to-date' % (git.path, coverage_data.result_sha)) 'make sure your source repository is up-to-date' % (git.path, coverage_data.result_sha))
if coverage_data.status_code != 30: if coverage_data.result != "succeeded":
check_failed(args, 'results from Shippable indicate tests did not pass (status code: %d)\n' check_failed(args, 'results indicate tests did not pass (result: %s)\n'
're-run until passing, then download the latest results and re-run the report using those results' % coverage_data.status_code) 're-run until passing, then download the latest results and re-run the report using those results' % coverage_data.result)
if coverage_data.missing_jobs or coverage_data.extra_jobs:
check_failed(args, 'unexpected results from Shippable -- missing jobs: %s, extra jobs: %s\n'
'make sure the tests were successful and the all results were downloaded\n' % (
sorted(coverage_data.missing_jobs), sorted(coverage_data.extra_jobs)))
if not coverage_data.paths: if not coverage_data.paths:
raise ApplicationError('no coverage data found\n' raise ApplicationError('no coverage data found\n'
@ -280,26 +280,13 @@ class CoverageData:
with open(os.path.join(result_path, 'run.json')) as run_file: with open(os.path.join(result_path, 'run.json')) as run_file:
run = json.load(run_file) run = json.load(run_file)
self.org_name = run['subscriptionOrgName'] self.result_sha = run["resources"]["repositories"]["self"]["version"]
self.project_name = run['projectName'] self.result = run['result']
self.result_sha = run['commitSha']
self.status_code = run['statusCode']
self.github_base_url = 'https://github.com/%s/%s/blob/%s/' % (self.org_name, self.project_name, self.result_sha) self.github_base_url = 'https://github.com/ansible/ansible/blob/%s/' % self.result_sha
# locate available results # locate available results
self.paths = sorted(glob.glob(os.path.join(result_path, '*', 'test', 'testresults', 'coverage-analyze-targets.json'))) self.paths = sorted(glob.glob(os.path.join(result_path, '*', 'coverage-analyze-targets.json')))
# make sure the test matrix is complete
matrix_include = run['cleanRunYml']['matrix']['include']
matrix_jobs = list((idx, dict(tuple(item.split('=', 1)) for item in value['env'])) for idx, value in enumerate(matrix_include, start=1))
sanity_job_numbers = set(idx for idx, env in matrix_jobs if env['T'].startswith('sanity/'))
units_job_numbers = set(idx for idx, env in matrix_jobs if env['T'].startswith('units/'))
expected_job_numbers = set(idx for idx, env in matrix_jobs)
actual_job_numbers = set(int(os.path.relpath(path, result_path).split(os.path.sep)[0]) for path in self.paths)
self.missing_jobs = expected_job_numbers - actual_job_numbers - sanity_job_numbers - units_job_numbers
self.extra_jobs = actual_job_numbers - expected_job_numbers - sanity_job_numbers - units_job_numbers
class Git: class Git:

114
hacking/shippable/run.py
View file

@ -17,7 +17,9 @@
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""CLI tool for starting new Shippable CI runs."""
"""CLI tool for starting new CI runs."""
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
@ -25,45 +27,42 @@ __metaclass__ = type
import argparse import argparse
import json import json
import os import os
import sys
import requests import requests
import requests.auth
try: try:
import argcomplete import argcomplete
except ImportError: except ImportError:
argcomplete = None argcomplete = None
# TODO: Dev does not have a token for AZP, somebody please test this.
# Following changes should be made to improve the overall style:
# TODO use new style formatting method.
# TODO type hints.
def main(): def main():
"""Main program body.""" """Main program body."""
args = parse_args() args = parse_args()
start_run(args)
key = os.environ.get('AZP_TOKEN', None)
if not key:
sys.stderr.write("please set you AZP token in AZP_TOKEN")
sys.exit(1)
start_run(args, key)
def parse_args(): def parse_args():
"""Parse and return args.""" """Parse and return args."""
api_key = get_api_key()
parser = argparse.ArgumentParser(description='Start a new Shippable run.') parser = argparse.ArgumentParser(description='Start a new CI run.')
parser.add_argument('project', parser.add_argument('-p', '--pipeline-id', type=int, default=20, help='pipeline to download the job from')
metavar='account/project', parser.add_argument('--ref', help='git ref name to run on')
help='Shippable account/project')
target = parser.add_mutually_exclusive_group()
target.add_argument('--branch',
help='branch name')
target.add_argument('--run',
metavar='ID',
help='Shippable run ID')
parser.add_argument('--key',
metavar='KEY',
default=api_key,
required=not api_key,
help='Shippable API key')
parser.add_argument('--env', parser.add_argument('--env',
nargs=2, nargs=2,
@ -79,71 +78,16 @@ def parse_args():
return args return args
def start_run(args): def start_run(args, key):
"""Start a new Shippable run.""" """Start a new CI run."""
headers = dict(
Authorization='apiToken %s' % args.key,
)
# get project ID url = "https://dev.azure.com/ansible/ansible/_apis/pipelines/%s/runs?api-version=6.0-preview.1" % args.pipeline_id
payload = {"resources": {"repositories": {"self": {"refName": args.ref}}}}
data = dict( resp = requests.post(url, auth=requests.auth.HTTPBasicAuth('user', key), data=payload)
projectFullNames=args.project, resp.raise_for_status()
)
url = 'https://api.shippable.com/projects' print(json.dumps(resp.json(), indent=4, sort_keys=True))
response = requests.get(url, data, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
result = response.json()
if len(result) != 1:
raise Exception(
'Received %d items instead of 1 looking for %s in:\n%s' % (
len(result),
args.project,
json.dumps(result, indent=4, sort_keys=True)))
project_id = response.json()[0]['id']
# new build
data = dict(
globalEnv=dict((kp[0], kp[1]) for kp in args.env or [])
)
if args.branch:
data['branchName'] = args.branch
elif args.run:
data['runId'] = args.run
url = 'https://api.shippable.com/projects/%s/newBuild' % project_id
response = requests.post(url, json=data, headers=headers)
if response.status_code != 200:
raise Exception("HTTP %s: %s\n%s" % (response.status_code, response.reason, response.content))
print(json.dumps(response.json(), indent=4, sort_keys=True))
def get_api_key():
"""
rtype: str
"""
key = os.environ.get('SHIPPABLE_KEY', None)
if key:
return key
path = os.path.join(os.environ['HOME'], '.shippable.key')
try:
with open(path, 'r') as key_fd:
return key_fd.read().strip()
except IOError:
return None
if __name__ == '__main__': if __name__ == '__main__':