[kbn/test] add support for using test groups (#25776) (#25839)

packages/kbn-dev-utils/index.d.ts

@@ -18,3 +18,4 @@
  */
 
 export * from './src/tooling_log';
+export * from './src/serializers';

packages/kbn-dev-utils/src/index.js

@@ -19,3 +19,4 @@
 
 export { withProcRunner } from './proc_runner';
 export { ToolingLog, ToolingLogTextWriter, pickLevelFromFlags } from './tooling_log';
+export { createAbsolutePathSerializer } from './serializers';

packages/kbn-dev-utils/src/serializers/absolute_path_serializer.d.ts

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export function createAbsolutePathSerializer(
+  rootPath: string
+): { print(...args: any[]): string; test(value: any): boolean };

packages/kbn-dev-utils/src/serializers/absolute_path_serializer.js

@@ -17,30 +17,9 @@
  * under the License.
  */
 
-import { createFunctionalTestRunner } from '../../../../../src/functional_test_runner';
-import { CliError } from './run_cli';
-
-export async function runFtr({
-  configPath,
-  options: { log, bail, grep, updateBaselines, suiteTags },
-}) {
-  const ftr = createFunctionalTestRunner({
-    log,
-    configFile: configPath,
-    configOverrides: {
-      mochaOpts: {
-        bail: !!bail,
-        grep,
-      },
-      updateBaselines,
-      suiteTags,
-    },
-  });
-
-  const failureCount = await ftr.run();
-  if (failureCount > 0) {
-    throw new CliError(
-      `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}`
-    );
-  }
+export function createAbsolutePathSerializer(rootPath) {
+  return {
+    print: value => value.replace(rootPath, '<absolute path>').replace(/\\/g, '/'),
+    test: value => typeof value === 'string' && value.startsWith(rootPath),
+  };
 }

packages/kbn-dev-utils/src/serializers/index.d.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { createAbsolutePathSerializer } from './absolute_path_serializer';

packages/kbn-dev-utils/src/serializers/index.js

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { createAbsolutePathSerializer } from './absolute_path_serializer';

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -18,6 +18,7 @@ Options:
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
   --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
   --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
+  --assert-none-excluded         Exit with 1/0 based on if any test is excluded with the current set of tags. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 
@@ -26,8 +27,9 @@ Options:
 
 exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
@@ -41,8 +43,9 @@ Object {
 
 exports[`process options for run tests CLI accepts debug option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "debug": true,
@@ -56,9 +59,10 @@ Object {
 
 exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "config": "",
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
@@ -74,8 +78,9 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": Object {
@@ -90,8 +95,9 @@ Object {
 
 exports[`process options for run tests CLI accepts quiet option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
@@ -105,8 +111,9 @@ Object {
 
 exports[`process options for run tests CLI accepts silent option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
@@ -120,8 +127,9 @@ Object {
 
 exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "esFrom": "source",
@@ -135,8 +143,9 @@ Object {
 
 exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
@@ -150,8 +159,9 @@ Object {
 
 exports[`process options for run tests CLI accepts value for grep 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
@@ -165,8 +175,9 @@ Object {
 
 exports[`process options for run tests CLI accepts verbose option 1`] = `
 Object {
+  "assertNoneExcluded": false,
   "configs": Array [
-    "foo",
+    <absolute path>/foo,
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -18,6 +18,7 @@ Options:
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
   --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
   --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
+  --assert-none-excluded         Exit with 1/0 based on if any test is excluded with the current set of tags. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import dedent from 'dedent';
 import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -52,6 +54,9 @@ const options = {
     arg: '<tag>',
     desc: 'Tags that suites must NOT include to be run, can be included multiple times.',
   },
+  'assert-none-excluded': {
+    desc: 'Exit with 1/0 based on if any test is excluded with the current set of tags.',
+  },
   verbose: { desc: 'Log everything.' },
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
@@ -113,6 +118,9 @@ export function processOptions(userOptions, defaultConfigPaths) {
   delete userOptions['include-tag'];
   delete userOptions['exclude-tag'];
 
+  userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded'];
+  delete userOptions['assert-none-excluded'];
+
   function createLogger() {
     return new ToolingLog({
       level: pickLevelFromFlags(userOptions),
@@ -122,7 +130,7 @@ export function processOptions(userOptions, defaultConfigPaths) {
 
   return {
     ...userOptions,
-    configs,
+    configs: configs.map(c => resolve(c)),
     createLogger,
     extraKbnOpts: userOptions._,
   };

packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js

@@ -18,6 +18,9 @@
  */
 
 import { displayHelp, processOptions } from './args';
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd()));
 
 describe('display help for run tests CLI', () => {
   it('displays as expected', () => {

packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { Writable } from 'stream';
+
 import { runTestsCli } from './cli';
 import { checkMockConsoleLogSnapshot } from '../../test_helpers';
 
@@ -36,7 +38,7 @@ describe('run tests CLI', () => {
     const processMock = {
       exit: exitMock,
       argv: argvMock,
-      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      stdout: new Writable(),
       cwd: jest.fn(),
     };
 

packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap

@@ -21,9 +21,7 @@ Options:
 
 exports[`process options for start servers CLI accepts debug option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "debug": true,
   "esFrom": "snapshot",
@@ -33,9 +31,7 @@ Object {
 
 exports[`process options for start servers CLI accepts empty config value if default passed 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
@@ -47,9 +43,7 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "snapshot",
   "extraKbnOpts": Object {
@@ -60,9 +54,7 @@ Object {
 
 exports[`process options for start servers CLI accepts quiet option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
@@ -72,9 +64,7 @@ Object {
 
 exports[`process options for start servers CLI accepts silent option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
@@ -84,9 +74,7 @@ Object {
 
 exports[`process options for start servers CLI accepts source value for esFrom 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "source",
   "extraKbnOpts": undefined,
@@ -95,9 +83,7 @@ Object {
 
 exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,
@@ -107,9 +93,7 @@ Object {
 
 exports[`process options for start servers CLI accepts verbose option 1`] = `
 Object {
-  "config": Array [
-    "foo",
-  ],
+  "config": <absolute path>/foo,
   "createLogger": [Function],
   "esFrom": "snapshot",
   "extraKbnOpts": undefined,

packages/kbn-test/src/functional_tests/cli/start_servers/args.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import dedent from 'dedent';
 import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
 
@@ -97,7 +99,7 @@ export function processOptions(userOptions, defaultConfigPath) {
 
   return {
     ...userOptions,
-    config,
+    config: resolve(config),
     createLogger,
     extraKbnOpts: userOptions._,
   };

packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js

@@ -18,6 +18,9 @@
  */
 
 import { displayHelp, processOptions } from './args';
+import { createAbsolutePathSerializer } from '@kbn/dev-utils';
+
+expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd()));
 
 describe('display help for start servers CLI', () => {
   it('displays as expected', () => {
@@ -39,60 +42,60 @@ describe('process options for start servers CLI', () => {
   });
 
   it('accepts empty config value if default passed', () => {
-    const options = processOptions({ config: '' }, ['foo']);
+    const options = processOptions({ config: '' }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid option', () => {
     expect(() => {
-      processOptions({ bail: true }, ['foo']);
+      processOptions({ bail: true }, 'foo');
     }).toThrow('functional_tests_server: invalid option [bail]');
   });
 
   it('accepts string value for kibana-install-dir', () => {
-    const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    const options = processOptions({ 'kibana-install-dir': 'foo' }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('rejects boolean value for kibana-install-dir', () => {
     expect(() => {
-      processOptions({ 'kibana-install-dir': true }, ['foo']);
+      processOptions({ 'kibana-install-dir': true }, 'foo');
     }).toThrow('functional_tests_server: invalid argument [true] to option [kibana-install-dir]');
   });
 
   it('accepts source value for esFrom', () => {
-    const options = processOptions({ esFrom: 'source' }, ['foo']);
+    const options = processOptions({ esFrom: 'source' }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts debug option', () => {
-    const options = processOptions({ debug: true }, ['foo']);
+    const options = processOptions({ debug: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts silent option', () => {
-    const options = processOptions({ silent: true }, ['foo']);
+    const options = processOptions({ silent: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts quiet option', () => {
-    const options = processOptions({ quiet: true }, ['foo']);
+    const options = processOptions({ quiet: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts verbose option', () => {
-    const options = processOptions({ verbose: true }, ['foo']);
+    const options = processOptions({ verbose: true }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('accepts extra server options', () => {
-    const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    const options = processOptions({ _: { 'server.foo': 'bar' } }, 'foo');
     expect(options).toMatchSnapshot();
   });
 
   it('rejects invalid options even if valid options exist', () => {
     expect(() => {
-      processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+      processOptions({ debug: true, aintnothang: true, bail: true }, 'foo');
     }).toThrow('functional_tests_server: invalid option [aintnothang]');
   });
 });

packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { Writable } from 'stream';
+
 import { startServersCli } from './cli';
 import { checkMockConsoleLogSnapshot } from '../../test_helpers';
 
@@ -36,7 +38,7 @@ describe('start servers CLI', () => {
     const processMock = {
       exit: exitMock,
       argv: argvMock,
-      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      stdout: new Writable(),
       cwd: jest.fn(),
     };
 

packages/kbn-test/src/functional_tests/lib/fun_ftr.js

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import * as FunctionalTestRunner from '../../../../../src/functional_test_runner';
+import { CliError } from './run_cli';
+
+function createFtr({ configPath, options: { log, bail, grep, updateBaselines, suiteTags } }) {
+  return FunctionalTestRunner.createFunctionalTestRunner({
+    log,
+    configFile: configPath,
+    configOverrides: {
+      mochaOpts: {
+        bail: !!bail,
+        grep,
+      },
+      updateBaselines,
+      suiteTags,
+    },
+  });
+}
+
+export async function assertNoneExcluded({ configPath, options }) {
+  const ftr = createFtr({ configPath, options });
+
+  const stats = await ftr.getTestStats();
+  if (stats.excludedTests > 0) {
+    throw new CliError(`
+      ${stats.excludedTests} tests in the ${configPath} config
+      are excluded when filtering by the tags run on CI. Make sure that all suites are
+      tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh
+
+      ${JSON.stringify(options.suiteTags)}
+
+    `);
+  }
+}
+
+export async function runFtr({ configPath, options }) {
+  const ftr = createFtr({ configPath, options });
+
+  const failureCount = await ftr.run();
+  if (failureCount > 0) {
+    throw new CliError(
+      `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}`
+    );
+  }
+}
+
+export async function hasTests({ configPath, options }) {
+  const ftr = createFtr({ configPath, options });
+  const stats = await ftr.getTestStats();
+  return stats.tests > 0;
+}

packages/kbn-test/src/functional_tests/lib/index.js

@@ -19,6 +19,6 @@
 
 export { runKibanaServer } from './run_kibana_server';
 export { runElasticsearch } from './run_elasticsearch';
-export { runFtr } from './run_ftr';
+export { runFtr, hasTests, assertNoneExcluded } from './fun_ftr';
 export { KIBANA_ROOT, KIBANA_FTR_SCRIPT, FUNCTIONAL_CONFIG_PATH, API_CONFIG_PATH } from './paths';
 export { runCli } from './run_cli';

packages/kbn-test/src/functional_tests/tasks.js

@@ -17,12 +17,19 @@
  * under the License.
  */
 
-import { relative, resolve } from 'path';
+import { relative } from 'path';
 import * as Rx from 'rxjs';
 import { startWith, switchMap, take } from 'rxjs/operators';
 import { withProcRunner } from '@kbn/dev-utils';
 
-import { runElasticsearch, runKibanaServer, runFtr, KIBANA_FTR_SCRIPT } from './lib';
+import {
+  runElasticsearch,
+  runKibanaServer,
+  runFtr,
+  assertNoneExcluded,
+  hasTests,
+  KIBANA_FTR_SCRIPT,
+} from './lib';
 
 import { readConfigFile } from '../../../../src/functional_test_runner/lib';
 
@@ -38,37 +45,63 @@ in another terminal session by running this command from this directory:
 /**
  * Run servers and tests for each config
  * @param {object} options                   Optional
- * @property {string[]} configPaths          Array of paths to configs
- * @property {function} options.createLogger Optional logger creation function
+ * @property {string[]} options.configs      Array of paths to configs
+ * @property {function} options.log          An instance of the ToolingLog
  * @property {string} options.installDir     Optional installation dir from which to run Kibana
  * @property {boolean} options.bail          Whether to exit test run at the first failure
  * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function runTests(options) {
   for (const configPath of options.configs) {
-    await runSingleConfig(resolve(process.cwd(), configPath), options);
+    const log = options.createLogger();
+    const opts = {
+      ...options,
+      log,
+    };
+
+    log.info('Running', configPath);
+    log.indent(2);
+
+    if (options.assertNoneExcluded) {
+      await assertNoneExcluded({ configPath, options: opts });
+      continue;
+    }
+
+    if (!(await hasTests({ configPath, options: opts }))) {
+      log.info('Skipping', configPath, 'since all tests are excluded');
+      continue;
+    }
+
+    await withProcRunner(log, async procs => {
+      const config = await readConfigFile(log, configPath);
+
+      const es = await runElasticsearch({ config, options: opts });
+      await runKibanaServer({ procs, config, options: opts });
+      await runFtr({ configPath, options: opts });
+
+      await procs.stop('kibana');
+      await es.cleanup();
+    });
   }
 }
 
 /**
  * Start only servers using single config
  * @param {object} options                   Optional
- * @property {string} options.configPath     Path to a config file
- * @property {function} options.createLogger Optional logger creation function
+ * @property {string} options.config         Path to a config file
+ * @property {function} options.log          An instance of the ToolingLog
  * @property {string} options.installDir     Optional installation dir from which to run Kibana
  * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function startServers(options) {
-  const { config: configOption, createLogger } = options;
-  const configPath = resolve(process.cwd(), configOption);
-  const log = createLogger();
+  const log = options.createLogger();
   const opts = {
     ...options,
     log,
   };
 
   await withProcRunner(log, async procs => {
-    const config = await readConfigFile(log, configPath);
+    const config = await readConfigFile(log, options.config);
 
     const es = await runElasticsearch({ config, options: opts });
     await runKibanaServer({
@@ -100,25 +133,3 @@ async function silence(milliseconds, { log }) {
     )
     .toPromise();
 }
-
-/*
- * Start servers and run tests for single config
- */
-async function runSingleConfig(configPath, options) {
-  const log = options.createLogger();
-  const opts = {
-    ...options,
-    log,
-  };
-
-  await withProcRunner(log, async procs => {
-    const config = await readConfigFile(log, configPath);
-
-    const es = await runElasticsearch({ config, options: opts });
-    await runKibanaServer({ procs, config, options: opts });
-    await runFtr({ configPath, options: opts });
-
-    await procs.stop('kibana');
-    await es.cleanup();
-  });
-}