diff --git a/test/runner/lib/http.py b/test/runner/lib/http.py index 53df1a6bc60..e0a3da6a57a 100644 --- a/test/runner/lib/http.py +++ b/test/runner/lib/http.py @@ -6,6 +6,7 @@ Avoids use of urllib2 due to lack of SNI support. from __future__ import absolute_import, print_function import json +import time try: from urllib import urlencode @@ -24,6 +25,8 @@ from lib.util import ( CommonConfig, ApplicationError, run_command, + SubprocessError, + display, ) @@ -83,7 +86,28 @@ class HttpClient(object): cmd += [url] - stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2) + attempts = 0 + max_attempts = 3 + sleep_seconds = 3 + + # curl error codes which are safe to retry (request never sent to server) + retry_on_status = ( + 6, # CURLE_COULDNT_RESOLVE_HOST + ) + + while True: + attempts += 1 + + try: + stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2) + break + except SubprocessError as ex: + if ex.status in retry_on_status and attempts < max_attempts: + display.warning(u'%s' % ex) + time.sleep(sleep_seconds) + continue + + raise if self.args.explain and not self.always: return HttpResponse(method, url, 200, '') diff --git a/test/runner/shippable.py b/test/runner/shippable.py index c9526818d2f..29a485c9ffc 100755 --- a/test/runner/shippable.py +++ b/test/runner/shippable.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function # noinspection PyCompatibility import argparse import errno +import json import os import sys @@ -44,6 +45,9 @@ def main(): response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id) jobs = response.json() + if not isinstance(jobs, list): + raise ApplicationError(json.dumps(jobs, indent=4, sort_keys=True)) + if len(jobs) == 1: raise ApplicationError('Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?' % run_id) except ApplicationWarning as ex: