Add retries to Shippable CI scripts. (#21108)

* Add retry.py for auto-retry in CI scripts.
* Add retries to other and units CI scripts.

test/runner/retry.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+"""Automatically retry failed commands."""
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import errno
+import os
+import sys
+import time
+
+from lib.util import (
+    display,
+    raw_command,
+    ApplicationError,
+    ApplicationWarning,
+    SubprocessError,
+)
+
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
+
+def main():
+    """Main program function."""
+    try:
+        args = parse_args()
+        display.verbosity = args.verbosity
+        display.color = args.color
+
+        command = [args.command] + args.args
+
+        for attempt in range(0, args.tries):
+            if attempt > 0:
+                time.sleep(args.sleep)
+
+            try:
+                raw_command(command, env=os.environ)
+                return
+            except SubprocessError as ex:
+                display.error(ex)
+    except ApplicationWarning as ex:
+        display.warning(str(ex))
+        exit(0)
+    except ApplicationError as ex:
+        display.error(str(ex))
+        exit(1)
+    except KeyboardInterrupt:
+        exit(2)
+    except IOError as ex:
+        if ex.errno == errno.EPIPE:
+            exit(3)
+        raise
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-v', '--verbose',
+                        dest='verbosity',
+                        action='count',
+                        default=0,
+                        help='display more output')
+
+    parser.add_argument('--color',
+                        metavar='COLOR',
+                        nargs='?',
+                        help='generate color output: %(choices)s',
+                        choices=('yes', 'no', 'auto'),
+                        const='yes',
+                        default='auto')
+
+    parser.add_argument('--tries',
+                        metavar='TRIES',
+                        type=int,
+                        default=3,
+                        help='number of tries to execute command (default: %(default)s)')
+
+    parser.add_argument('--sleep',
+                        metavar='SECONDS',
+                        type=int,
+                        default=3,
+                        help='seconds to sleep between tries (default: %(default)s)')
+
+    parser.add_argument('command',
+                        help='command to execute')
+
+    parser.add_argument('args',
+                        metavar='...',
+                        nargs=argparse.REMAINDER,
+                        help='optional arguments for command')
+
+    if argcomplete:
+        argcomplete.autocomplete(parser)
+
+    args = parser.parse_args()
+
+    if args.color == 'yes':
+        args.color = True
+    elif args.color == 'no':
+        args.color = False
+    elif 'SHIPPABLE' in os.environ:
+        args.color = True
+    else:
+        args.color = sys.stdout.isatty()
+
+    return args
+
+
+if __name__ == '__main__':
+    main()

test/utils/shippable/other.sh

@@ -2,12 +2,12 @@
 
 set -o pipefail
 
-add-apt-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe'
-add-apt-repository 'ppa:ubuntu-toolchain-r/test'
-add-apt-repository 'ppa:fkrull/deadsnakes'
+retry.py add-apt-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe'
+retry.py add-apt-repository 'ppa:ubuntu-toolchain-r/test'
+retry.py add-apt-repository 'ppa:fkrull/deadsnakes'
 
-apt-get update -qq
-apt-get install -qq \
+retry.py apt-get update -qq
+retry.py apt-get install -qq \
     shellcheck \
     python2.4 \
     g++-4.9 \
@@ -15,7 +15,7 @@ apt-get install -qq \
 
 ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc
 
-pip install tox --disable-pip-version-check
+retry.py pip install tox --disable-pip-version-check
 
 ansible-test compile --color -v
 ansible-test sanity --color -v --tox --skip-test ansible-doc --python 2.7

test/utils/shippable/units.sh

@@ -2,16 +2,16 @@
 
 set -o pipefail
 
-add-apt-repository 'ppa:ubuntu-toolchain-r/test'
-add-apt-repository 'ppa:fkrull/deadsnakes'
+retry.py add-apt-repository 'ppa:ubuntu-toolchain-r/test'
+retry.py add-apt-repository 'ppa:fkrull/deadsnakes'
 
-apt-get update -qq
-apt-get install -qq \
+retry.py apt-get update -qq
+retry.py apt-get install -qq \
     g++-4.9 \
     python3.6-dev \
 
 ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc
 
-pip install tox --disable-pip-version-check
+retry.py pip install tox --disable-pip-version-check
 
 ansible-test units --color -v --tox --coverage