[kbn/es] auto retry snapshot download on unexpected errors (#38944)
* [kbn/es] auto retry snapshot download on unexpected errors * missed an await * pass log to retry()
This commit is contained in:
parent
0003a32f56
commit
b5ac046314
|
@ -31,7 +31,7 @@ const asyncPipeline = promisify(pipeline);
|
||||||
const V1_VERSIONS_API = 'https://artifacts-api.elastic.co/v1/versions';
|
const V1_VERSIONS_API = 'https://artifacts-api.elastic.co/v1/versions';
|
||||||
|
|
||||||
const { cache } = require('./utils');
|
const { cache } = require('./utils');
|
||||||
const { createCliError } = require('./errors');
|
const { createCliError, isCliError } = require('./errors');
|
||||||
|
|
||||||
const TEST_ES_SNAPSHOT_VERSION = process.env.TEST_ES_SNAPSHOT_VERSION
|
const TEST_ES_SNAPSHOT_VERSION = process.env.TEST_ES_SNAPSHOT_VERSION
|
||||||
? process.env.TEST_ES_SNAPSHOT_VERSION
|
? process.env.TEST_ES_SNAPSHOT_VERSION
|
||||||
|
@ -66,6 +66,25 @@ function headersToString(headers, indent = '') {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function retry(log, fn) {
|
||||||
|
async function doAttempt(attempt) {
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} catch (error) {
|
||||||
|
if (isCliError(error) || attempt >= 5) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.warning('...failure, retrying in 5 seconds:', error.message);
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||||
|
log.info('...retrying');
|
||||||
|
return await doAttempt(attempt + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return await doAttempt(1);
|
||||||
|
}
|
||||||
|
|
||||||
exports.Artifact = class Artifact {
|
exports.Artifact = class Artifact {
|
||||||
/**
|
/**
|
||||||
* Fetch an Artifact from the Artifact API for a license level and version
|
* Fetch an Artifact from the Artifact API for a license level and version
|
||||||
|
@ -78,22 +97,27 @@ exports.Artifact = class Artifact {
|
||||||
const urlBuild = encodeURIComponent(TEST_ES_SNAPSHOT_VERSION);
|
const urlBuild = encodeURIComponent(TEST_ES_SNAPSHOT_VERSION);
|
||||||
const url = `${V1_VERSIONS_API}/${urlVersion}/builds/${urlBuild}/projects/elasticsearch`;
|
const url = `${V1_VERSIONS_API}/${urlVersion}/builds/${urlBuild}/projects/elasticsearch`;
|
||||||
|
|
||||||
log.info('downloading artifact info from %s', chalk.bold(url));
|
const json = await retry(log, async () => {
|
||||||
const abc = new AbortController();
|
log.info('downloading artifact info from %s', chalk.bold(url));
|
||||||
const resp = await fetch(url, { signal: abc.signal });
|
|
||||||
const json = await resp.text();
|
|
||||||
|
|
||||||
if (resp.status === 404) {
|
const abc = new AbortController();
|
||||||
abc.abort();
|
const resp = await fetch(url, { signal: abc.signal });
|
||||||
throw createCliError(
|
const json = await resp.text();
|
||||||
`Snapshots for ${version}/${TEST_ES_SNAPSHOT_VERSION} are not available`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!resp.ok) {
|
if (resp.status === 404) {
|
||||||
abc.abort();
|
abc.abort();
|
||||||
throw new Error(`Unable to read artifact info from ${url}: ${resp.statusText}\n ${json}`);
|
throw createCliError(
|
||||||
}
|
`Snapshots for ${version}/${TEST_ES_SNAPSHOT_VERSION} are not available`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!resp.ok) {
|
||||||
|
abc.abort();
|
||||||
|
throw new Error(`Unable to read artifact info from ${url}: ${resp.statusText}\n ${json}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return json;
|
||||||
|
});
|
||||||
|
|
||||||
// parse the api response into an array of Artifact objects
|
// parse the api response into an array of Artifact objects
|
||||||
const {
|
const {
|
||||||
|
@ -184,21 +208,23 @@ exports.Artifact = class Artifact {
|
||||||
* @return {Promise<void>}
|
* @return {Promise<void>}
|
||||||
*/
|
*/
|
||||||
async download(dest) {
|
async download(dest) {
|
||||||
const cacheMeta = cache.readMeta(dest);
|
await retry(this._log, async () => {
|
||||||
const tmpPath = `${dest}.tmp`;
|
const cacheMeta = cache.readMeta(dest);
|
||||||
|
const tmpPath = `${dest}.tmp`;
|
||||||
|
|
||||||
const artifactResp = await this._download(tmpPath, cacheMeta.etag, cacheMeta.ts);
|
const artifactResp = await this._download(tmpPath, cacheMeta.etag, cacheMeta.ts);
|
||||||
if (artifactResp.cached) {
|
if (artifactResp.cached) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this._verifyChecksum(artifactResp);
|
await this._verifyChecksum(artifactResp);
|
||||||
|
|
||||||
// cache the etag for future downloads
|
// cache the etag for future downloads
|
||||||
cache.writeMeta(dest, { etag: artifactResp.etag });
|
cache.writeMeta(dest, { etag: artifactResp.etag });
|
||||||
|
|
||||||
// rename temp download to the destination location
|
// rename temp download to the destination location
|
||||||
fs.renameSync(tmpPath, dest);
|
fs.renameSync(tmpPath, dest);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
Loading…
Reference in a new issue