[kbn/es] auto retry snapshot download on unexpected errors (#38944)

* [kbn/es] auto retry snapshot download on unexpected errors

* missed an await

* pass log to retry()

packages/kbn-es/src/artifact.js

@@ -31,7 +31,7 @@ const asyncPipeline = promisify(pipeline);
 const V1_VERSIONS_API = 'https://artifacts-api.elastic.co/v1/versions';
 
 const { cache } = require('./utils');
-const { createCliError } = require('./errors');
+const { createCliError, isCliError } = require('./errors');
 
 const TEST_ES_SNAPSHOT_VERSION = process.env.TEST_ES_SNAPSHOT_VERSION
   ? process.env.TEST_ES_SNAPSHOT_VERSION
@@ -66,6 +66,25 @@ function headersToString(headers, indent = '') {
   );
 }
 
+async function retry(log, fn) {
+  async function doAttempt(attempt) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (isCliError(error) || attempt >= 5) {
+        throw error;
+      }
+
+      log.warning('...failure, retrying in 5 seconds:', error.message);
+      await new Promise(resolve => setTimeout(resolve, 5000));
+      log.info('...retrying');
+      return await doAttempt(attempt + 1);
+    }
+  }
+
+  return await doAttempt(1);
+}
+
 exports.Artifact = class Artifact {
   /**
    * Fetch an Artifact from the Artifact API for a license level and version
@@ -78,7 +97,9 @@ exports.Artifact = class Artifact {
     const urlBuild = encodeURIComponent(TEST_ES_SNAPSHOT_VERSION);
     const url = `${V1_VERSIONS_API}/${urlVersion}/builds/${urlBuild}/projects/elasticsearch`;
 
+    const json = await retry(log, async () => {
       log.info('downloading artifact info from %s', chalk.bold(url));
+
       const abc = new AbortController();
       const resp = await fetch(url, { signal: abc.signal });
       const json = await resp.text();
@@ -95,6 +116,9 @@ exports.Artifact = class Artifact {
         throw new Error(`Unable to read artifact info from ${url}: ${resp.statusText}\n  ${json}`);
       }
 
+      return json;
+    });
+
     // parse the api response into an array of Artifact objects
     const {
       project: { packages: artifactInfoMap },
@@ -184,6 +208,7 @@ exports.Artifact = class Artifact {
    * @return {Promise<void>}
    */
   async download(dest) {
+    await retry(this._log, async () => {
       const cacheMeta = cache.readMeta(dest);
       const tmpPath = `${dest}.tmp`;
 
@@ -199,6 +224,7 @@ exports.Artifact = class Artifact {
 
       // rename temp download to the destination location
       fs.renameSync(tmpPath, dest);
+    });
   }
 
   /**