From dfe8eca3d2588aad2acced3b4c7c8966569af629 Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Fri, 24 Mar 2017 12:36:13 -0700
Subject: [PATCH] Update Shippable downloader to support new API.
- Download run/jobs by URL (old or new).
- Download run metadata.
- Download job metadata.
- Format downloaded JSON.
- Handle and save coverage/testresults download errors.
---
test/utils/shippable/download.py | 141 +++++++++++++++++++++++++++----
1 file changed, 125 insertions(+), 16 deletions(-)
diff --git a/test/utils/shippable/download.py b/test/utils/shippable/download.py
index cbda144a846..787fde95d8a 100755
--- a/test/utils/shippable/download.py
+++ b/test/utils/shippable/download.py
@@ -39,7 +39,8 @@ def main():
parser = ArgumentParser(description='Download results from a Shippable run.')
parser.add_argument('run_id',
- help='shippable run id.')
+ metavar='RUN',
+ help='shippable run id, run url or run name formatted as: account/project/run_number')
parser.add_argument('-v', '--verbose',
dest='verbose',
@@ -69,11 +70,20 @@ def main():
action='store_true',
help='download code coverage results')
+ parser.add_argument('--job-metadata',
+ action='store_true',
+ help='download job metadata')
+
+ parser.add_argument('--run-metadata',
+ action='store_true',
+ help='download run metadata')
+
parser.add_argument('--all',
action='store_true',
help='download everything')
parser.add_argument('--job-number',
+ metavar='N',
action='append',
type=int,
help='limit downloads to the given job number')
@@ -83,57 +93,145 @@ def main():
args = parser.parse_args()
+ old_runs_prefix = 'https://app.shippable.com/runs/'
+
+ if args.run_id.startswith(old_runs_prefix):
+ args.run_id = args.run_id[len(old_runs_prefix):]
+
if args.all:
args.console_logs = True
args.test_results = True
args.coverage_results = True
+ args.job_metadata = True
+ args.run_metadata = True
- if not args.console_logs and not args.test_results and not args.coverage_results:
- parser.error('At least one download option is required: --console-logs, --test-results, --coverage-results')
+ selections = (
+ args.console_logs,
+ args.test_results,
+ args.coverage_results,
+ args.job_metadata,
+ args.run_metadata,
+ )
+
+ if not any(selections):
+ parser.error('At least one download option is required.')
headers = dict(
Authorization='apiToken %s' % args.api_key,
)
+ match = re.search(r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)$', args.run_id)
+
+ if not match:
+ match = re.search(r'^(?P<account>[^/]+)/(?P<project>[^/]+)/(?P<run_number>[0-9]+)$', args.run_id)
+
+ if match:
+ account = match.group('account')
+ project = match.group('project')
+ run_number = int(match.group('run_number'))
+
+ url = 'https://api.shippable.com/projects'
+ response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers)
+
+ if response.status_code != 200:
+ raise Exception(response.content)
+
+ project_id = response.json()[0]['id']
+
+ url = 'https://api.shippable.com/runs?projectIds=%s' % project_id
+
+ response = requests.get(url, headers=headers)
+
+ if response.status_code != 200:
+ raise Exception(response.content)
+
+ run = [run for run in response.json() if run['runNumber'] == run_number][0]
+
+ args.run_id = run['id']
+ else:
+ url = 'https://api.shippable.com/runs/%s' % args.run_id
+
+ response = requests.get(url, headers=headers)
+
+ if response.status_code != 200:
+ raise Exception(response.content)
+
+ run = response.json()
+
+ account = run['subscriptionOrgName']
+ project = run['projectName']
+ run_number = run['runNumber']
+
+ output_dir = '%s/%s/%s' % (account, project, run_number)
+
response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
- body = response.json()
- output_dir = args.run_id
+ jobs = sorted(response.json(), key=lambda job: int(job['jobNumber']))
if not args.test:
if not os.path.exists(output_dir):
- os.mkdir(output_dir)
+ os.makedirs(output_dir)
- for j in body:
+ if args.run_metadata:
+ path = os.path.join(output_dir, 'run.json')
+ contents = json.dumps(run, sort_keys=True, indent=4)
+
+ if args.verbose or args.test:
+ print(path)
+
+ if not args.test:
+ with open(path, 'w') as metadata_fd:
+ metadata_fd.write(contents)
+
+ for j in jobs:
job_id = j['id']
job_number = j['jobNumber']
if args.job_number and job_number not in args.job_number:
continue
+ if args.job_metadata:
+ path = os.path.join(output_dir, '%s/job.json' % job_number)
+ contents = json.dumps(j, sort_keys=True, indent=4)
+
+ if args.verbose or args.test:
+ print(path)
+
+ if not args.test:
+ directory = os.path.dirname(path)
+
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+
+ with open(path, 'w') as metadata_fd:
+ metadata_fd.write(contents)
+
if args.console_logs:
- path = os.path.join(output_dir, '%s-console.log' % job_number)
+ path = os.path.join(output_dir, '%s/console.log' % job_number)
url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id
- download(args, headers, path, url)
+ download(args, headers, path, url, is_json=False)
if args.test_results:
- path = os.path.join(output_dir, '%s-test.json' % job_number)
+ path = os.path.join(output_dir, '%s/test.json' % job_number)
url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id
download(args, headers, path, url)
- extract_contents(args, path, os.path.join(output_dir, '%s-test' % job_number))
+ extract_contents(args, path, os.path.join(output_dir, '%s/test' % job_number))
if args.coverage_results:
- path = os.path.join(output_dir, '%s-coverage.json' % job_number)
+ path = os.path.join(output_dir, '%s/coverage.json' % job_number)
url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id
download(args, headers, path, url)
- extract_contents(args, path, os.path.join(output_dir, '%s-coverage' % job_number))
+ extract_contents(args, path, os.path.join(output_dir, '%s/coverage' % job_number))
def extract_contents(args, path, output_dir):
if not args.test:
+ if not os.path.exists(path):
+ return
+
with open(path, 'r') as json_fd:
items = json.load(json_fd)
@@ -149,12 +247,15 @@ def extract_contents(args, path, output_dir):
if args.verbose:
print(path)
+ if path.endswith('.json'):
+ contents = json.dumps(json.loads(contents), sort_keys=True, indent=4)
+
if not os.path.exists(path):
with open(path, 'w') as output_fd:
output_fd.write(contents)
-def download(args, headers, path, url):
+def download(args, headers, path, url, is_json=True):
if args.verbose or args.test:
print(path)
@@ -165,9 +266,17 @@ def download(args, headers, path, url):
response = requests.get(url, headers=headers)
if response.status_code != 200:
- raise Exception(response.content)
+ path += '.error'
- content = response.content
+ if is_json:
+ content = json.dumps(response.json(), sort_keys=True, indent=4)
+ else:
+ content = response.content
+
+ directory = os.path.dirname(path)
+
+ if not os.path.exists(directory):
+ os.makedirs(directory)
with open(path, 'w') as f:
f.write(content)