Update Shippable downloader to support new API.
- Download run/jobs by URL (old or new). - Download run metadata. - Download job metadata. - Format downloaded JSON. - Handle and save coverage/testresults download errors.
This commit is contained in:
Matt Clay
parent
318445fe19
commit
dfe8eca3d2
1 changed files with 125 additions and 16 deletions
|
|
@ -39,7 +39,8 @@ def main():
|
|||
parser = ArgumentParser(description='Download results from a Shippable run.')
|
||||
|
||||
parser.add_argument('run_id',
|
||||
help='shippable run id.')
|
||||
metavar='RUN',
|
||||
help='shippable run id, run url or run name formatted as: account/project/run_number')
|
||||
|
||||
parser.add_argument('-v', '--verbose',
|
||||
dest='verbose',
|
||||
|
|
@ -69,11 +70,20 @@ def main():
|
|||
action='store_true',
|
||||
help='download code coverage results')
|
||||
|
||||
parser.add_argument('--job-metadata',
|
||||
action='store_true',
|
||||
help='download job metadata')
|
||||
|
||||
parser.add_argument('--run-metadata',
|
||||
action='store_true',
|
||||
help='download run metadata')
|
||||
|
||||
parser.add_argument('--all',
|
||||
action='store_true',
|
||||
help='download everything')
|
||||
|
||||
parser.add_argument('--job-number',
|
||||
metavar='N',
|
||||
action='append',
|
||||
type=int,
|
||||
help='limit downloads to the given job number')
|
||||
|
|
@ -83,57 +93,145 @@ def main():
|
|||
|
||||
args = parser.parse_args()
|
||||
|
||||
old_runs_prefix = 'https://app.shippable.com/runs/'
|
||||
|
||||
if args.run_id.startswith(old_runs_prefix):
|
||||
args.run_id = args.run_id[len(old_runs_prefix):]
|
||||
|
||||
if args.all:
|
||||
args.console_logs = True
|
||||
args.test_results = True
|
||||
args.coverage_results = True
|
||||
args.job_metadata = True
|
||||
args.run_metadata = True
|
||||
|
||||
if not args.console_logs and not args.test_results and not args.coverage_results:
|
||||
parser.error('At least one download option is required: --console-logs, --test-results, --coverage-results')
|
||||
selections = (
|
||||
args.console_logs,
|
||||
args.test_results,
|
||||
args.coverage_results,
|
||||
args.job_metadata,
|
||||
args.run_metadata,
|
||||
)
|
||||
|
||||
if not any(selections):
|
||||
parser.error('At least one download option is required.')
|
||||
|
||||
headers = dict(
|
||||
Authorization='apiToken %s' % args.api_key,
|
||||
)
|
||||
|
||||
match = re.search(r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)$', args.run_id)
|
||||
|
||||
if not match:
|
||||
match = re.search(r'^(?P<account>[^/]+)/(?P<project>[^/]+)/(?P<run_number>[0-9]+)$', args.run_id)
|
||||
|
||||
if match:
|
||||
account = match.group('account')
|
||||
project = match.group('project')
|
||||
run_number = int(match.group('run_number'))
|
||||
|
||||
url = 'https://api.shippable.com/projects'
|
||||
response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
|
||||
project_id = response.json()[0]['id']
|
||||
|
||||
url = 'https://api.shippable.com/runs?projectIds=%s' % project_id
|
||||
|
||||
response = requests.get(url, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
|
||||
run = [run for run in response.json() if run['runNumber'] == run_number][0]
|
||||
|
||||
args.run_id = run['id']
|
||||
else:
|
||||
url = 'https://api.shippable.com/runs/%s' % args.run_id
|
||||
|
||||
response = requests.get(url, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
|
||||
run = response.json()
|
||||
|
||||
account = run['subscriptionOrgName']
|
||||
project = run['projectName']
|
||||
run_number = run['runNumber']
|
||||
|
||||
output_dir = '%s/%s/%s' % (account, project, run_number)
|
||||
|
||||
response = requests.get('https://api.shippable.com/jobs?runIds=%s' % args.run_id, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
|
||||
body = response.json()
|
||||
output_dir = args.run_id
|
||||
jobs = sorted(response.json(), key=lambda job: int(job['jobNumber']))
|
||||
|
||||
if not args.test:
|
||||
if not os.path.exists(output_dir):
|
||||
os.mkdir(output_dir)
|
||||
os.makedirs(output_dir)
|
||||
|
||||
for j in body:
|
||||
if args.run_metadata:
|
||||
path = os.path.join(output_dir, 'run.json')
|
||||
contents = json.dumps(run, sort_keys=True, indent=4)
|
||||
|
||||
if args.verbose or args.test:
|
||||
print(path)
|
||||
|
||||
if not args.test:
|
||||
with open(path, 'w') as metadata_fd:
|
||||
metadata_fd.write(contents)
|
||||
|
||||
for j in jobs:
|
||||
job_id = j['id']
|
||||
job_number = j['jobNumber']
|
||||
|
||||
if args.job_number and job_number not in args.job_number:
|
||||
continue
|
||||
|
||||
if args.job_metadata:
|
||||
path = os.path.join(output_dir, '%s/job.json' % job_number)
|
||||
contents = json.dumps(j, sort_keys=True, indent=4)
|
||||
|
||||
if args.verbose or args.test:
|
||||
print(path)
|
||||
|
||||
if not args.test:
|
||||
directory = os.path.dirname(path)
|
||||
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
|
||||
with open(path, 'w') as metadata_fd:
|
||||
metadata_fd.write(contents)
|
||||
|
||||
if args.console_logs:
|
||||
path = os.path.join(output_dir, '%s-console.log' % job_number)
|
||||
path = os.path.join(output_dir, '%s/console.log' % job_number)
|
||||
url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id
|
||||
download(args, headers, path, url)
|
||||
download(args, headers, path, url, is_json=False)
|
||||
|
||||
if args.test_results:
|
||||
path = os.path.join(output_dir, '%s-test.json' % job_number)
|
||||
path = os.path.join(output_dir, '%s/test.json' % job_number)
|
||||
url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id
|
||||
download(args, headers, path, url)
|
||||
extract_contents(args, path, os.path.join(output_dir, '%s-test' % job_number))
|
||||
extract_contents(args, path, os.path.join(output_dir, '%s/test' % job_number))
|
||||
|
||||
if args.coverage_results:
|
||||
path = os.path.join(output_dir, '%s-coverage.json' % job_number)
|
||||
path = os.path.join(output_dir, '%s/coverage.json' % job_number)
|
||||
url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id
|
||||
download(args, headers, path, url)
|
||||
extract_contents(args, path, os.path.join(output_dir, '%s-coverage' % job_number))
|
||||
extract_contents(args, path, os.path.join(output_dir, '%s/coverage' % job_number))
|
||||
|
||||
|
||||
def extract_contents(args, path, output_dir):
|
||||
if not args.test:
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
||||
with open(path, 'r') as json_fd:
|
||||
items = json.load(json_fd)
|
||||
|
||||
|
|
@ -149,12 +247,15 @@ def extract_contents(args, path, output_dir):
|
|||
if args.verbose:
|
||||
print(path)
|
||||
|
||||
if path.endswith('.json'):
|
||||
contents = json.dumps(json.loads(contents), sort_keys=True, indent=4)
|
||||
|
||||
if not os.path.exists(path):
|
||||
with open(path, 'w') as output_fd:
|
||||
output_fd.write(contents)
|
||||
|
||||
|
||||
def download(args, headers, path, url):
|
||||
def download(args, headers, path, url, is_json=True):
|
||||
if args.verbose or args.test:
|
||||
print(path)
|
||||
|
||||
|
|
@ -165,9 +266,17 @@ def download(args, headers, path, url):
|
|||
response = requests.get(url, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(response.content)
|
||||
path += '.error'
|
||||
|
||||
content = response.content
|
||||
if is_json:
|
||||
content = json.dumps(response.json(), sort_keys=True, indent=4)
|
||||
else:
|
||||
content = response.content
|
||||
|
||||
directory = os.path.dirname(path)
|
||||
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
|
||||
with open(path, 'w') as f:
|
||||
f.write(content)
|
||||
|
|
|
|||
Loading…
Reference in a new issue