[kbn-es] Add support to kbn-es and kbn-test for data archives (#28723)

* Add support to kbn-es and kbn-test for data archives

* Fix log indent

* Decompress directly to data directory

Co-Authored-By: joshdover <me@joshdover.com>

* Improve logs

packages/kbn-es/src/cli_commands/snapshot.js

@@ -33,6 +33,7 @@ exports.help = (defaults = {}) => {
       --version       Version of ES to download [default: ${defaults.version}]
       --base-path     Path containing cache/installations [default: ${basePath}]
       --install-path  Installation path, defaults to 'source' within base-path
+      --data-archive  Path to zip or tarball containing an ES data directory to seed the cluster with.
       --password      Sets password for elastic user [default: ${password}]
       -E              Additional key=value settings to pass to Elasticsearch
       --download-only Download the snapshot but don't actually start it
@@ -49,6 +50,7 @@ exports.run = async (defaults = {}) => {
     alias: {
       basePath: 'base-path',
       installPath: 'install-path',
+      dataArchive: 'data-archive',
       esArgs: 'E',
     },
 
@@ -62,6 +64,11 @@ exports.run = async (defaults = {}) => {
     await cluster.downloadSnapshot(options);
   } else {
     const { installPath } = await cluster.installSnapshot(options);
+
+    if (options.dataArchive) {
+      await cluster.extractDataDirectory(installPath, options.dataArchive);
+    }
+
     await cluster.run(installPath, { esArgs: options.esArgs });
   }
 };

packages/kbn-es/src/cli_commands/source.js

@@ -33,6 +33,7 @@ exports.help = (defaults = {}) => {
       --source-path   Path to ES source [default: ${defaults['source-path']}]
       --base-path     Path containing cache/installations [default: ${basePath}]
       --install-path  Installation path, defaults to 'source' within base-path
+      --data-archive  Path to zip or tarball containing an ES data directory to seed the cluster with.
       --password      Sets password for elastic user [default: ${password}]
       -E              Additional key=value settings to pass to Elasticsearch
 
@@ -49,6 +50,7 @@ exports.run = async (defaults = {}) => {
       basePath: 'base-path',
       installPath: 'install-path',
       sourcePath: 'source-path',
+      dataArchive: 'data-archive',
       esArgs: 'E',
     },
 
@@ -57,5 +59,10 @@ exports.run = async (defaults = {}) => {
 
   const cluster = new Cluster();
   const { installPath } = await cluster.installSource(options);
+
+  if (options.dataArchive) {
+    await cluster.extractDataDirectory(installPath, options.dataArchive);
+  }
+
   await cluster.run(installPath, { esArgs: options.esArgs });
 };

packages/kbn-es/src/cluster.js

@@ -19,9 +19,10 @@
 
 const execa = require('execa');
 const chalk = require('chalk');
+const path = require('path');
 const { downloadSnapshot, installSnapshot, installSource, installArchive } = require('./install');
 const { ES_BIN } = require('./paths');
-const { log: defaultLog, parseEsLog, extractConfigFiles } = require('./utils');
+const { log: defaultLog, parseEsLog, extractConfigFiles, decompress } = require('./utils');
 const { createCliError } = require('./errors');
 const { promisify } = require('util');
 const treeKillAsync = promisify(require('tree-kill'));
@@ -116,6 +117,28 @@ exports.Cluster = class Cluster {
     return { installPath };
   }
 
+  /**
+   * Unpakcs a tar or zip file containing the data directory for an
+   * ES cluster.
+   *
+   * @param {String} installPath
+   * @param {String} archivePath
+   */
+  async extractDataDirectory(installPath, archivePath) {
+    this._log.info(chalk.bold(`Extracting data directory`));
+    this._log.indent(4);
+
+    // decompress excludes the root directory as that is how our archives are
+    // structured. This works in our favor as we can explicitly extract into the data dir
+    const extractPath = path.resolve(installPath, 'data');
+    this._log.info(`Data archive: ${archivePath}`);
+    this._log.info(`Extract path: ${extractPath}`);
+
+    await decompress(archivePath, extractPath);
+
+    this._log.indent(-4);
+  }
+
   /**
    * Starts ES and returns resolved promise once started
    *

packages/kbn-test/src/es/es_test_cluster.js

@@ -36,6 +36,7 @@ export function createEsTestCluster(options = {}) {
     log,
     basePath = resolve(KIBANA_ROOT, '.es'),
     esFrom = esTestConfig.getBuildFrom(),
+    dataArchive,
   } = options;
 
   const randomHash = Math.random()
@@ -74,6 +75,10 @@ export function createEsTestCluster(options = {}) {
         throw new Error(`unknown option esFrom "${esFrom}"`);
       }
 
+      if (dataArchive) {
+        await cluster.extractDataDirectory(installPath, dataArchive);
+      }
+
       await cluster.start(installPath, {
         esArgs: [
           `cluster.name=${clusterName}`,

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js

@@ -37,6 +37,7 @@ export async function runElasticsearch({ config, options }) {
     log,
     basePath: resolve(KIBANA_ROOT, '.es'),
     esFrom: esFrom || config.get('esTestCluster.from'),
+    dataArchive: config.get('esTestCluster.dataArchive'),
   });
 
   const esArgs = config.get('esTestCluster.serverArgs');

src/functional_test_runner/lib/config/schema.js

@@ -139,6 +139,7 @@ export const schema = Joi.object().keys({
     license: Joi.string().default('oss'),
     from: Joi.string().default('snapshot'),
     serverArgs: Joi.array(),
+    dataArchive: Joi.string(),
   }).default(),
 
   kbnTestServer: Joi.object().keys({