[Functional Tests] Use @kbn/test on Kibana CI (#18967)

* Replace test:api with @kbn/test runTests

* Improve CLI help menu 🆘

* Use --es-from

* Replace jenkins:selenium with kbn-test

* Validate cli args, fixing test in the process

* Clean up some stuff

* Code review fixes

* Explanation for collectCliArgs

* Remove exit codes, they're useless anyway.

* Make markdown vis test pass with dev_mode setting

* Tests

* Remove unneeded export

* Code review: move console logging up to cli.js

* Code review: refactor startServers and runTests to take single options collection

* Code review: Remove all things I am sure we do not use

* Improve tests

* Code review fixes

* Pass created log to runFtr, runElasticsearch, runKibanaServer

* Update --es-from option to --esFrom

packages/kbn-test/src/es/es_test_cluster.js

@@ -34,8 +34,7 @@ export function createEsTestCluster(options = {}) {
     license = 'oss',
     log,
     basePath = resolve(KIBANA_ROOT, '.es'),
-    // Use source when running on CI
-    from = esTestConfig.getBuildFrom(),
+    esFrom = esTestConfig.getBuildFrom(),
   } = options;
 
   const randomHash = Math.random()
@@ -58,12 +57,12 @@ export function createEsTestCluster(options = {}) {
       const second = 1000;
       const minute = second * 60;
 
-      return from === 'snapshot' ? minute : minute * 6;
+      return esFrom === 'snapshot' ? minute : minute * 6;
     }
 
     async start(esArgs = []) {
       const { installPath } =
-        from === 'source'
+        esFrom === 'source'
           ? await cluster.installSource(config)
           : await cluster.installSnapshot(config);
 

packages/kbn-test/src/functional_tests/cli/index.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-export { runTestsCli } from './run_tests_cli';
-export { startServersCli } from './start_servers_cli';
+export { runTestsCli } from './run_tests/cli';
+export { startServersCli } from './start_servers/cli';

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -0,0 +1,137 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`display help for run tests CLI displays as expected 1`] = `
+"Run Functional Tests
+
+Usage:
+  node scripts/functional_tests --help
+  node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+  node scripts/functional_tests [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config. Can pass in multiple configs. 
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --bail                         Stop the test run at the first failure. 
+  --grep <pattern>               Pattern to select which tests to run. 
+  --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing."
+`;
+
+exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "updateBaselines": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts debug option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "debug": true,
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
+Object {
+  "config": "",
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts extra server options 1`] = `
+Object {
+  "_": Object {
+    "server.foo": "bar",
+  },
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": Object {
+    "server.foo": "bar",
+  },
+}
+`;
+
+exports[`process options for run tests CLI accepts quiet option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "quiet": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts silent option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "silent": true,
+}
+`;
+
+exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "esFrom": "source",
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "kibana-install-dir": "foo",
+}
+`;
+
+exports[`process options for run tests CLI accepts value for grep 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "grep": "management",
+}
+`;
+
+exports[`process options for run tests CLI accepts verbose option 1`] = `
+Object {
+  "configs": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "verbose": true,
+}
+`;

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -0,0 +1,75 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`run tests CLI options accepts help option even if invalid options passed 1`] = `
+Array [
+  Array [
+    "Run Functional Tests
+
+Usage:
+  node scripts/functional_tests --help
+  node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+  node scripts/functional_tests [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config. Can pass in multiple configs. 
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --bail                         Stop the test run at the first failure. 
+  --grep <pattern>               Pattern to select which tests to run. 
+  --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing.",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects boolean config value 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [true] to option [config]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects boolean value for kibana-install-dir 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [true] to option [kibana-install-dir]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects empty config value if no default passed 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: config is required",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects invalid options even if valid options exist 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid option [aintnothang]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects non-boolean value for bail 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [peanut] to option [bail]",
+  ],
+]
+`;
+
+exports[`run tests CLI options rejects non-enum value for esFrom 1`] = `
+Array [
+  Array [
+    "Error: functional_tests: invalid argument [butter] to option [esFrom]",
+  ],
+]
+`;

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import dedent from 'dedent';
+import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+
+const options = {
+  help: { desc: 'Display this menu and exit.' },
+  config: {
+    arg: '<file>',
+    desc: 'Pass in a config. Can pass in multiple configs.',
+  },
+  esFrom: {
+    arg: '<snapshot|source>',
+    choices: ['snapshot', 'source'],
+    desc: 'Build Elasticsearch from source or run from snapshot.',
+    default: 'snapshot',
+  },
+  'kibana-install-dir': {
+    arg: '<dir>',
+    desc: 'Run Kibana from existing install directory instead of from source.',
+  },
+  bail: { desc: 'Stop the test run at the first failure.' },
+  grep: {
+    arg: '<pattern>',
+    desc: 'Pattern to select which tests to run.',
+  },
+  updateBaselines: {
+    desc:
+      'Replace baseline screenshots with whatever is generated from the test.',
+  },
+  verbose: { desc: 'Log everything.' },
+  debug: { desc: 'Run in debug mode.' },
+  quiet: { desc: 'Only log errors.' },
+  silent: { desc: 'Log nothing.' },
+};
+
+export function displayHelp() {
+  const helpOptions = Object.keys(options)
+    .filter(name => name !== '_')
+    .map(name => {
+      const option = options[name];
+      return {
+        ...option,
+        usage: `${name} ${option.arg || ''}`,
+        default: option.default ? `Default: ${option.default}` : '',
+      };
+    })
+    .map(option => {
+      return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
+    })
+    .join(`\n      `);
+
+  return dedent(`
+    Run Functional Tests
+
+    Usage:
+      node scripts/functional_tests --help
+      node scripts/functional_tests [--config <file1> [--config <file2> ...]]
+      node scripts/functional_tests [options] [-- --<other args>]
+
+    Options:
+      ${helpOptions}
+    `);
+}
+
+export function processOptions(userOptions, defaultConfigPaths) {
+  validateOptions(userOptions);
+
+  let configs;
+  if (userOptions.config) {
+    configs = [].concat(userOptions.config);
+  } else {
+    if (!defaultConfigPaths || defaultConfigPaths.length === 0) {
+      throw new Error(`functional_tests: config is required`);
+    } else {
+      configs = defaultConfigPaths;
+    }
+  }
+
+  function createLogger() {
+    const log = createToolingLog(pickLevelFromFlags(userOptions));
+    log.pipe(process.stdout);
+    return log;
+  }
+
+  return {
+    ...userOptions,
+    configs,
+    createLogger,
+    extraKbnOpts: userOptions._,
+  };
+}
+
+function validateOptions(userOptions) {
+  Object.entries(userOptions).forEach(([key, val]) => {
+    if (key === '_') return;
+
+    // Validate flags passed
+    if (options[key] === undefined) {
+      throw new Error(`functional_tests: invalid option [${key}]`);
+    }
+
+    if (
+      // Validate boolean flags
+      (!options[key].arg && typeof val !== 'boolean') ||
+      // Validate string/array flags
+      (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
+      // Validate enum flags
+      (options[key].choices && !options[key].choices.includes(val))
+    ) {
+      throw new Error(
+        `functional_tests: invalid argument [${val}] to option [${key}]`
+      );
+    }
+  });
+}

packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js

@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { displayHelp, processOptions } from './args';
+
+describe('display help for run tests CLI', () => {
+  it('displays as expected', () => {
+    expect(displayHelp()).toMatchSnapshot();
+  });
+});
+
+describe('process options for run tests CLI', () => {
+  it('rejects boolean config value', () => {
+    expect(() => {
+      processOptions({ config: true });
+    }).toThrow('functional_tests: invalid argument [true] to option [config]');
+  });
+
+  it('rejects empty config value if no default passed', () => {
+    expect(() => {
+      processOptions({});
+    }).toThrow('functional_tests: config is required');
+  });
+
+  it('accepts empty config value if default passed', () => {
+    const options = processOptions({ config: '' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects non-boolean value for bail', () => {
+    expect(() => {
+      processOptions({ bail: 'peanut' }, ['foo']);
+    }).toThrow('functional_tests: invalid argument [peanut] to option [bail]');
+  });
+
+  it('accepts string value for kibana-install-dir', () => {
+    const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects boolean value for kibana-install-dir', () => {
+    expect(() => {
+      processOptions({ 'kibana-install-dir': true }, ['foo']);
+    }).toThrow(
+      'functional_tests: invalid argument [true] to option [kibana-install-dir]'
+    );
+  });
+
+  it('accepts boolean value for updateBaselines', () => {
+    const options = processOptions({ updateBaselines: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts source value for esFrom', () => {
+    const options = processOptions({ esFrom: 'source' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects non-enum value for esFrom', () => {
+    expect(() => {
+      processOptions({ esFrom: 'butter' }, ['foo']);
+    }).toThrow(
+      'functional_tests: invalid argument [butter] to option [esFrom]'
+    );
+  });
+
+  it('accepts value for grep', () => {
+    const options = processOptions({ grep: 'management' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts debug option', () => {
+    const options = processOptions({ debug: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts silent option', () => {
+    const options = processOptions({ silent: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts quiet option', () => {
+    const options = processOptions({ quiet: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts verbose option', () => {
+    const options = processOptions({ verbose: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts extra server options', () => {
+    const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects invalid options even if valid options exist', () => {
+    expect(() => {
+      processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+    }).toThrow('functional_tests: invalid option [aintnothang]');
+  });
+});

packages/kbn-test/src/functional_tests/cli/run_tests/cli.js

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import getopts from 'getopts';
+import { runTests } from '../../tasks';
+import { processOptions, displayHelp } from './args';
+
+/**
+ * Run servers and tests for each config
+ * Only cares about --config option. Other options
+ * are passed directly to functional_test_runner, such as
+ * --bail, --verbose, etc.
+ * @param {string[]} defaultConfigPaths Optional paths to configs
+ *                                      if no config option is passed
+ */
+export async function runTestsCli(defaultConfigPaths) {
+  try {
+    const userOptions = getopts(process.argv.slice(2)) || {};
+    if (userOptions.help) {
+      console.log(displayHelp());
+      return undefined;
+    }
+
+    const options = processOptions(userOptions, defaultConfigPaths);
+    await runTests(options);
+  } catch (err) {
+    console.log(chalk.red(err));
+    process.exit(1);
+  }
+}

packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js

@@ -0,0 +1,202 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { runTestsCli } from './cli';
+
+// Note: Stub the runTests function to keep testing only around the cli
+// method and arguments.
+jest.mock('../../tasks', () => ({
+  runTests: jest.fn(),
+}));
+
+describe('run tests CLI', () => {
+  describe('options', () => {
+    const originalObjects = {};
+    const exitMock = jest.fn();
+    const logMock = jest.fn(); // mock logging so we don't send output to the test results
+    const argvMock = ['foo', 'foo'];
+
+    const processMock = {
+      exit: exitMock,
+      argv: argvMock,
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      cwd: jest.fn(),
+    };
+
+    beforeAll(() => {
+      originalObjects.process = process;
+      originalObjects.console = console;
+      global.process = processMock;
+      global.console = { log: logMock };
+    });
+
+    afterAll(() => {
+      global.process = originalObjects.process;
+      global.console = originalObjects.console;
+    });
+
+    beforeEach(() => {
+      global.process.argv = [...argvMock];
+      jest.resetAllMocks();
+    });
+
+    it('rejects boolean config value', async () => {
+      global.process.argv.push('--config');
+
+      await runTestsCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('rejects empty config value if no default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await runTestsCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts empty config value if default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-boolean value for bail', async () => {
+      global.process.argv.push('--bail', 'peanut');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts string value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir', 'foo');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects boolean value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts boolean value for updateBaselines', async () => {
+      global.process.argv.push('--updateBaselines');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith();
+    });
+
+    it('accepts source value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'source');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-enum value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'butter');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts value for grep', async () => {
+      global.process.argv.push('--grep', 'management');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts debug option', async () => {
+      global.process.argv.push('--debug');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts silent option', async () => {
+      global.process.argv.push('--silent');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts quiet option', async () => {
+      global.process.argv.push('--quiet');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts verbose option', async () => {
+      global.process.argv.push('--verbose');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts extra server options', async () => {
+      global.process.argv.push('--', '--server.foo=bar');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts help option even if invalid options passed', async () => {
+      global.process.argv.push('--debug', '--aintnothang', '--help');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).not.toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('rejects invalid options even if valid options exist', async () => {
+      global.process.argv.push('--debug', '--aintnothang', '--bail');
+
+      await runTestsCli(['foo']);
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+  });
+});

packages/kbn-test/src/functional_tests/cli/run_tests_cli.js

@@ -1,103 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import dedent from 'dedent';
-import getopts from 'getopts';
-import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
-import { runTests } from '../../';
-
-/**
- * Run servers and tests for each config
- * Only cares about --config option. Other options
- * are passed directly to functional_test_runner, such as
- * --bail, --verbose, etc.
- * @param {string[]} defaultConfigPaths  Array of paths to configs to use
- *                                       if no config option is passed
- */
-export async function runTestsCli(defaultConfigPaths) {
-  const { configs, help, bail, log, installDir } = processArgs(
-    defaultConfigPaths
-  );
-
-  if (help) return displayHelp();
-
-  if (!configs || configs.length === 0) {
-    log.error(
-      `Run Tests requires at least one path to a config. Leave blank to use defaults.`
-    );
-    process.exit(9);
-  }
-
-  try {
-    await runTests(configs, { bail, log, installDir });
-  } catch (err) {
-    log.error('FATAL ERROR');
-    log.error(err);
-    process.exit(1);
-  }
-}
-
-function processArgs(defaultConfigPaths) {
-  // If no args are passed, use {}
-  const options = getopts(process.argv.slice(2)) || {};
-
-  // If --config is passed without paths, it's "true", so use default
-  const configs =
-    typeof options.config === 'string' || Array.isArray(options.config)
-      ? [].concat(options.config)
-      : defaultConfigPaths;
-
-  const log = createToolingLog(pickLevelFromFlags(options));
-  log.pipe(process.stdout);
-
-  return {
-    configs,
-    log,
-    help: options.help,
-    bail: options.bail,
-    installDir: options['kibana-install-dir'],
-    rest: options._,
-  };
-}
-
-function displayHelp() {
-  console.log(
-    dedent(`
-    Run Functional Tests
-
-    Usage:  node scripts/functional_tests [options]
-
-    --config      Option to pass in a config
-                  Can pass in multiple configs with
-                  --config file1 --config file2 --config file3
-    --kibana-install-dir
-                  Run Kibana from an existing install directory
-                  Default: run from source
-    --bail        Stop the test run at the first failure
-    --help        Display this menu and exit
-
-    Log level options:
-
-    --verbose
-    --debug
-    --quiet       Log errors
-    --silent
-    `)
-  );
-}

packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap

@@ -0,0 +1,111 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`display help for start servers CLI displays as expected 1`] = `
+"Start Functional Test Servers
+
+Usage:
+  node scripts/functional_tests_server --help
+  node scripts/functional_tests_server [--config <file>]
+  node scripts/functional_tests_server [options] [-- --<other args>]
+
+Options:
+  --help                         Display this menu and exit. 
+  --config <file>                Pass in a config 
+  --esFrom <snapshot|source>     Build Elasticsearch from source or run from snapshot. Default: snapshot
+  --kibana-install-dir <dir>     Run Kibana from existing install directory instead of from source. 
+  --verbose                      Log everything. 
+  --debug                        Run in debug mode. 
+  --quiet                        Only log errors. 
+  --silent                       Log nothing."
+`;
+
+exports[`process options for start servers CLI accepts debug option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "debug": true,
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for start servers CLI accepts empty config value if default passed 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for start servers CLI accepts extra server options 1`] = `
+Object {
+  "_": Object {
+    "server.foo": "bar",
+  },
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": Object {
+    "server.foo": "bar",
+  },
+}
+`;
+
+exports[`process options for start servers CLI accepts quiet option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "quiet": true,
+}
+`;
+
+exports[`process options for start servers CLI accepts silent option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "silent": true,
+}
+`;
+
+exports[`process options for start servers CLI accepts source value for esFrom 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "esFrom": "source",
+  "extraKbnOpts": undefined,
+}
+`;
+
+exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "kibana-install-dir": "foo",
+}
+`;
+
+exports[`process options for start servers CLI accepts verbose option 1`] = `
+Object {
+  "config": Array [
+    "foo",
+  ],
+  "createLogger": [Function],
+  "extraKbnOpts": undefined,
+  "verbose": true,
+}
+`;

packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/cli.test.js.snap

@@ -0,0 +1,57 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`start servers CLI options accepts boolean value for updateBaselines 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid option [updateBaselines]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects bail 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid option [bail]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects boolean config value 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid argument [true] to option [config]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects boolean value for kibana-install-dir 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid argument [true] to option [kibana-install-dir]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects empty config value if no default passed 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: config is required",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects invalid options even if valid options exist 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid option [grep]",
+  ],
+]
+`;
+
+exports[`start servers CLI options rejects non-enum value for esFrom 1`] = `
+Array [
+  Array [
+    "Error: functional_tests_server: invalid argument [butter] to option [esFrom]",
+  ],
+]
+`;

packages/kbn-test/src/functional_tests/cli/start_servers/args.js

@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import dedent from 'dedent';
+import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+
+const options = {
+  help: { desc: 'Display this menu and exit.' },
+  config: {
+    arg: '<file>',
+    desc: 'Pass in a config',
+  },
+  esFrom: {
+    arg: '<snapshot|source>',
+    choices: ['snapshot', 'source'],
+    desc: 'Build Elasticsearch from source or run from snapshot.',
+    default: 'snapshot',
+  },
+  'kibana-install-dir': {
+    arg: '<dir>',
+    desc: 'Run Kibana from existing install directory instead of from source.',
+  },
+  verbose: { desc: 'Log everything.' },
+  debug: { desc: 'Run in debug mode.' },
+  quiet: { desc: 'Only log errors.' },
+  silent: { desc: 'Log nothing.' },
+};
+
+export function displayHelp() {
+  const helpOptions = Object.keys(options)
+    .filter(name => name !== '_')
+    .map(name => {
+      const option = options[name];
+      return {
+        ...option,
+        usage: `${name} ${option.arg || ''}`,
+        default: option.default ? `Default: ${option.default}` : '',
+      };
+    })
+    .map(option => {
+      return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
+    })
+    .join(`\n      `);
+
+  return dedent(`
+    Start Functional Test Servers
+
+    Usage:
+      node scripts/functional_tests_server --help
+      node scripts/functional_tests_server [--config <file>]
+      node scripts/functional_tests_server [options] [-- --<other args>]
+
+    Options:
+      ${helpOptions}
+    `);
+}
+
+export function processOptions(userOptions, defaultConfigPath) {
+  validateOptions(userOptions);
+
+  const config = userOptions.config || defaultConfigPath;
+
+  if (!config) {
+    throw new Error(`functional_tests_server: config is required`);
+  }
+
+  function createLogger() {
+    const log = createToolingLog(pickLevelFromFlags(userOptions));
+    log.pipe(process.stdout);
+    return log;
+  }
+
+  return {
+    ...userOptions,
+    config,
+    createLogger,
+    extraKbnOpts: userOptions._,
+  };
+}
+
+function validateOptions(userOptions) {
+  Object.entries(userOptions).forEach(([key, val]) => {
+    if (key === '_') return;
+
+    // Validate flags passed
+    if (options[key] === undefined) {
+      throw new Error(`functional_tests_server: invalid option [${key}]`);
+    }
+
+    if (
+      // Validate boolean flags
+      (!options[key].arg && typeof val !== 'boolean') ||
+      // Validate string/array flags
+      (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
+      // Validate enum flags
+      (options[key].choices && !options[key].choices.includes(val))
+    ) {
+      throw new Error(
+        `functional_tests_server: invalid argument [${val}] to option [${key}]`
+      );
+    }
+  });
+}

packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js

@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { displayHelp, processOptions } from './args';
+
+describe('display help for start servers CLI', () => {
+  it('displays as expected', () => {
+    expect(displayHelp()).toMatchSnapshot();
+  });
+});
+
+describe('process options for start servers CLI', () => {
+  it('rejects boolean config value', () => {
+    expect(() => {
+      processOptions({ config: true });
+    }).toThrow(
+      'functional_tests_server: invalid argument [true] to option [config]'
+    );
+  });
+
+  it('rejects empty config value if no default passed', () => {
+    expect(() => {
+      processOptions({});
+    }).toThrow('functional_tests_server: config is required');
+  });
+
+  it('accepts empty config value if default passed', () => {
+    const options = processOptions({ config: '' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects invalid option', () => {
+    expect(() => {
+      processOptions({ bail: true }, ['foo']);
+    }).toThrow('functional_tests_server: invalid option [bail]');
+  });
+
+  it('accepts string value for kibana-install-dir', () => {
+    const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects boolean value for kibana-install-dir', () => {
+    expect(() => {
+      processOptions({ 'kibana-install-dir': true }, ['foo']);
+    }).toThrow(
+      'functional_tests_server: invalid argument [true] to option [kibana-install-dir]'
+    );
+  });
+
+  it('accepts source value for esFrom', () => {
+    const options = processOptions({ esFrom: 'source' }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects non-enum value for esFrom', () => {
+    expect(() => {
+      processOptions({ esFrom: 'butter' }, ['foo']);
+    }).toThrow(
+      'functional_tests_server: invalid argument [butter] to option [esFrom]'
+    );
+  });
+
+  it('accepts debug option', () => {
+    const options = processOptions({ debug: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts silent option', () => {
+    const options = processOptions({ silent: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts quiet option', () => {
+    const options = processOptions({ quiet: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts verbose option', () => {
+    const options = processOptions({ verbose: true }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('accepts extra server options', () => {
+    const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
+    expect(options).toMatchSnapshot();
+  });
+
+  it('rejects invalid options even if valid options exist', () => {
+    expect(() => {
+      processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
+    }).toThrow('functional_tests_server: invalid option [aintnothang]');
+  });
+});

packages/kbn-test/src/functional_tests/cli/start_servers/cli.js

@@ -17,16 +17,28 @@
  * under the License.
  */
 
-export const functional = {
-  options: {
-    logLevel: 'debug',
-    configFile: require.resolve('../../test/functional/config.js')
-  }
-};
+import chalk from 'chalk';
+import getopts from 'getopts';
+import { startServers } from '../../tasks';
+import { processOptions, displayHelp } from './args';
 
-export const apiIntegration = {
-  options: {
-    logLevel: 'debug',
-    configFile: require.resolve('../../test/api_integration/config.js')
+/**
+ * Start servers
+ * @param {string} defaultConfigPath Optional path to config
+ *                                   if no config option is passed
+ */
+export async function startServersCli(defaultConfigPath) {
+  try {
+    const userOptions = getopts(process.argv.slice(2)) || {};
+    if (userOptions.help) {
+      console.log(displayHelp());
+      return undefined;
+    }
+
+    const options = processOptions(userOptions, defaultConfigPath);
+    await startServers(options);
+  } catch (err) {
+    console.log(chalk.red(err));
+    process.exit(1);
   }
-};
+}

packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js

@@ -0,0 +1,194 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { startServersCli } from './cli';
+
+// Note: Stub the startServers function to keep testing only around the cli
+// method and arguments.
+jest.mock('../../tasks', () => ({
+  startServers: jest.fn(),
+}));
+
+describe('start servers CLI', () => {
+  describe('options', () => {
+    const originalObjects = {};
+    const exitMock = jest.fn();
+    const logMock = jest.fn(); // mock logging so we don't send output to the test results
+    const argvMock = ['foo', 'foo'];
+
+    const processMock = {
+      exit: exitMock,
+      argv: argvMock,
+      stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
+      cwd: jest.fn(),
+    };
+
+    beforeAll(() => {
+      originalObjects.process = process;
+      originalObjects.console = console;
+      global.process = processMock;
+      global.console = { log: logMock };
+    });
+
+    afterAll(() => {
+      global.process = originalObjects.process;
+      global.console = originalObjects.console;
+    });
+
+    beforeEach(() => {
+      global.process.argv = [...argvMock];
+      jest.resetAllMocks();
+    });
+
+    it('rejects boolean config value', async () => {
+      global.process.argv.push('--config');
+
+      await startServersCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('rejects empty config value if no default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await startServersCli();
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts empty config value if default passed', async () => {
+      global.process.argv.push('--config', '');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects bail', async () => {
+      global.process.argv.push('--bail', true);
+
+      await startServersCli('foo');
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts string value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir', 'foo');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects boolean value for kibana-install-dir', async () => {
+      global.process.argv.push('--kibana-install-dir');
+
+      await startServersCli('foo');
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts boolean value for updateBaselines', async () => {
+      global.process.argv.push('--updateBaselines');
+
+      await startServersCli('foo');
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts source value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'source');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('rejects non-enum value for esFrom', async () => {
+      global.process.argv.push('--esFrom', 'butter');
+
+      await startServersCli('foo');
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+
+    it('accepts debug option', async () => {
+      global.process.argv.push('--debug');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts silent option', async () => {
+      global.process.argv.push('--silent');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts quiet option', async () => {
+      global.process.argv.push('--quiet');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts verbose option', async () => {
+      global.process.argv.push('--verbose');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts extra server options', async () => {
+      global.process.argv.push('--', '--server.foo=bar');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalled();
+    });
+
+    it('accepts help option even if invalid options passed', async () => {
+      global.process.argv.push('--debug', '--grep', '--help');
+
+      await startServersCli('foo');
+
+      expect(exitMock).not.toHaveBeenCalledWith(1);
+    });
+
+    it('rejects invalid options even if valid options exist', async () => {
+      global.process.argv.push('--debug', '--grep', '--bail');
+
+      await startServersCli('foo');
+
+      expect(exitMock).toHaveBeenCalledWith(1);
+      expect(logMock.mock.calls).toMatchSnapshot();
+    });
+  });
+});

packages/kbn-test/src/functional_tests/cli/start_servers_cli.js

@@ -1,99 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import chalk from 'chalk';
-import dedent from 'dedent';
-import getopts from 'getopts';
-import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
-import { startServers } from '../../';
-
-/**
- * Start servers
- * @param {string} configPath path to config
- */
-export async function startServersCli(defaultConfigPath) {
-  const { config, log, help, installDir } = processArgv(defaultConfigPath);
-
-  if (help) return displayHelp();
-
-  if (!config) {
-    log.error(
-      `Start Servers requires one path to a config. Leave blank to use default.`
-    );
-    process.exit(1);
-  }
-
-  try {
-    await startServers(config, { log, installDir });
-  } catch (err) {
-    log.error('FATAL ERROR');
-    log.error(err);
-    process.exit(1);
-  }
-}
-
-function processArgv(defaultConfigPath) {
-  const options = getopts(process.argv.slice(2)) || {};
-
-  if (Array.isArray(options.config)) {
-    console.log(
-      chalk.red(
-        `Starting servers requires a single config path. Multiple were passed.`
-      )
-    );
-    process.exit(9);
-  }
-
-  const config =
-    typeof options.config === 'string' ? options.config : defaultConfigPath;
-
-  const log = createToolingLog(pickLevelFromFlags(options));
-  log.pipe(process.stdout);
-
-  return {
-    config,
-    log,
-    installDir: options.kibanaInstallDir,
-    help: options.help,
-    rest: options._,
-  };
-}
-
-function displayHelp() {
-  console.log(
-    dedent(`
-    Start Functional Test Servers
-
-    Usage:  node scripts/functional_tests_server [options]
-
-    --config      Option to pass in a config
-    --kibana-install-dir
-                  Run Kibana from an existing install directory
-                  Default: run from source
-    --help        Display this menu and exit
-
-    Log level options:
-
-    --verbose
-    --debug
-    --quiet       Log errors
-    --silent
-    `)
-  );
-}

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.js

@@ -23,7 +23,8 @@ import { createEsTestCluster } from '../../es';
 
 import { setupUsers, DEFAULT_SUPERUSER_PASS } from './auth';
 
-export async function runElasticsearch({ config, log }) {
+export async function runElasticsearch({ config, options }) {
+  const { log, esFrom } = options;
   const isOss = config.get('esTestCluster.license') === 'oss';
 
   const cluster = createEsTestCluster({
@@ -34,7 +35,7 @@ export async function runElasticsearch({ config, log }) {
     license: config.get('esTestCluster.license'),
     log,
     basePath: resolve(KIBANA_ROOT, '.es'),
-    from: config.get('esTestCluster.from'),
+    esFrom: esFrom || config.get('esTestCluster.from'),
   });
 
   const esArgs = config.get('esTestCluster.serverArgs');

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -22,15 +22,16 @@ import { KIBANA_FTR_SCRIPT, PROJECT_ROOT } from './paths';
 export async function runFtr({
   procs,
   configPath,
-  bail,
-  log,
   cwd = PROJECT_ROOT,
+  options: { log, bail, grep, updateBaselines },
 }) {
   const args = [KIBANA_FTR_SCRIPT];
 
   if (getLogFlag(log)) args.push(`--${getLogFlag(log)}`);
   if (bail) args.push('--bail');
   if (configPath) args.push('--config', configPath);
+  if (grep) args.push('--grep', grep);
+  if (updateBaselines) args.push('--updateBaselines');
 
   await procs.run('ftr', {
     cmd: 'node',

packages/kbn-test/src/functional_tests/lib/run_kibana_server.js

@@ -25,7 +25,7 @@ export async function runKibanaServer({ procs, config, options }) {
 
   await procs.run('kibana', {
     cmd: getKibanaCmd(installDir),
-    args: getCliArgs(config, options),
+    args: collectCliArgs(config, options),
     env: {
       FORCE_COLOR: 1,
       ...process.env,
@@ -45,14 +45,35 @@ function getKibanaCmd(installDir) {
   return KIBANA_EXEC;
 }
 
-function getCliArgs(config, { devMode, installDir }) {
+/* When installDir is passed, we run from a built version of Kibana,
+ * which uses different command line arguments. If installDir is not
+ * passed, we run from source code. We also allow passing in extra
+ * Kibana server options, so we tack those on here.
+ */
+function collectCliArgs(config, { installDir, extraKbnOpts }) {
   const buildArgs = config.get('kbnTestServer.buildArgs') || [];
   const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
   const serverArgs = config.get('kbnTestServer.serverArgs') || [];
 
-  if (devMode) serverArgs.push('--dev');
-
-  return installDir
-    ? [...serverArgs, ...buildArgs]
-    : [KIBANA_EXEC_PATH, ...serverArgs, ...sourceArgs];
+  return pipe(
+    serverArgs,
+    args => (installDir ? args.filter(a => a !== '--oss') : args),
+    args => {
+      return installDir
+        ? [...args, ...buildArgs]
+        : [KIBANA_EXEC_PATH, ...args, ...sourceArgs];
+    },
+    args => args.concat(extraKbnOpts || [])
+  );
+}
+
+/*
+ * Apply each function in fns to the result of the
+ * previous function. The first function's input
+ * is the arr array.
+ */
+function pipe(arr, ...fns) {
+  return fns.reduce((acc, fn) => {
+    return fn(acc);
+  }, arr);
 }

packages/kbn-test/src/functional_tests/tasks.js

@@ -42,39 +42,47 @@ in another terminal session by running this command from this directory:
 
 /**
  * Run servers and tests for each config
- * @param {string[]} configPaths         Array of paths to configs
- * @param {object}   options             Optional
- * @param {Log}      options.log         Optional logger
- * @param {string}   options.installDir  Optional installation dir
- *                                       from which to run Kibana
- * @param {boolean}  options.bail  Whether to exit test run at the first failure
+ * @param {object} options                   Optional
+ * @property {string[]} configPaths          Array of paths to configs
+ * @property {function} options.createLogger Optional logger creation function
+ * @property {string} options.installDir     Optional installation dir from which to run Kibana
+ * @property {boolean} options.bail          Whether to exit test run at the first failure
+ * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
-export async function runTests(configPaths, options) {
-  for (const configPath of configPaths) {
+export async function runTests(options) {
+  for (const configPath of options.configs) {
     await runSingleConfig(resolve(process.cwd(), configPath), options);
   }
 }
 
 /**
  * Start only servers using single config
- * @param {string}  configPath          Path to a config file
- * @param {object}  options             Optional
- * @param {Log}     options.log         Optional logger
- * @param {string}  options.installDir  Optional installation dir
- *                                      from which to run Kibana
+ * @param {object} options                   Optional
+ * @property {string} options.configPath     Path to a config file
+ * @property {function} options.createLogger Optional logger creation function
+ * @property {string} options.installDir     Optional installation dir from which to run Kibana
+ * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
-export async function startServers(configPath, options) {
-  const { log } = options;
-  configPath = resolve(process.cwd(), configPath);
+export async function startServers(options) {
+  const { config: configOption, createLogger } = options;
+  const configPath = resolve(process.cwd(), configOption);
+  const log = createLogger();
+  const opts = {
+    ...options,
+    log,
+  };
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
-    const es = await runElasticsearch({ config, log });
+    const es = await runElasticsearch({ config, options: opts });
     await runKibanaServer({
       procs,
       config,
-      options: { ...options, devMode: true },
+      options: {
+        ...opts,
+        extraKbnOpts: [...options.extraKbnOpts, '--dev'],
+      },
     });
 
     // wait for 5 seconds of silence before logging the
@@ -97,22 +105,25 @@ async function silence(milliseconds, { log }) {
  * Start servers and run tests for single config
  */
 async function runSingleConfig(configPath, options) {
-  const { bail, log } = options;
+  const log = options.createLogger();
+  const opts = {
+    ...options,
+    log,
+  };
 
   await withProcRunner(log, async procs => {
     const config = await readConfigFile(log, configPath);
 
-    const es = await runElasticsearch({ config, log });
-    await runKibanaServer({ procs, config, options });
+    const es = await runElasticsearch({ config, options: opts });
+    await runKibanaServer({ procs, config, options: opts });
 
     // Note: When solving how to incorporate functional_test_runner
     // clean this up
     await runFtr({
       procs,
       configPath,
-      bail,
-      log,
       cwd: process.cwd(),
+      options: opts,
     });
 
     await procs.stop('kibana');

tasks/config/run.js

@@ -17,12 +17,8 @@
  * under the License.
  */
 
-import { esTestConfig, kbnTestConfig } from '@kbn/test';
 import { resolve } from 'path';
 
-const SECOND = 1000;
-const MINUTE = 60 * SECOND;
-const HOUR = 60 * MINUTE;
 const PKG_VERSION = require('../../package.json').version;
 
 module.exports = function (grunt) {
@@ -58,20 +54,6 @@ module.exports = function (grunt) {
     };
   }
 
-  const apiTestServerFlags = [
-    '--optimize.enabled=false',
-    '--elasticsearch.url=' + esTestConfig.getUrl(),
-    '--elasticsearch.healthCheck.delay=' + HOUR,
-    '--server.port=' + kbnTestConfig.getPort(),
-    '--server.xsrf.disableProtection=true',
-  ];
-
-  const funcTestServerFlags = [
-    '--server.maxPayloadBytes=1648576', //default is 1048576
-    '--elasticsearch.url=' + esTestConfig.getUrl(),
-    '--server.port=' + kbnTestConfig.getPort(),
-  ];
-
   const browserTestServerFlags = [
     '--plugins.initialize=false',
     '--optimize.bundleFilter=tests',
@@ -117,55 +99,6 @@ module.exports = function (grunt) {
       ]
     },
 
-    // used by the test:api task
-    //    runs the kibana server prepared for the api_integration tests
-    apiTestServer: createKbnServerTask({
-      flags: [
-        ...apiTestServerFlags
-      ]
-    }),
-
-    // used by the test:api:server task
-    //    runs the kibana server in --dev mode, prepared for developing api_integration tests
-    //    and watching for changes so the server will restart when necessary
-    devApiTestServer: createKbnServerTask({
-      flags: [
-        '--dev',
-        '--no-base-path',
-        ...apiTestServerFlags,
-      ]
-    }),
-
-    // used by test:ui task
-    //    runs the kibana server prepared for the functional tests
-    funcTestServer: createKbnServerTask({
-      flags: [
-        ...funcTestServerFlags,
-      ]
-    }),
-
-    // used by the test:ui:server task
-    //    runs the kibana server in dev mode, prepared for the functional tests
-    devFuncTestServer: createKbnServerTask({
-      flags: [
-        ...funcTestServerFlags,
-        '--dev',
-        '--no-base-path',
-        '--optimize.watchPort=5611',
-        '--optimize.watchPrebuild=true',
-        '--optimize.bundleDir=' + resolve(__dirname, '../../optimize/testUiServer'),
-      ]
-    }),
-
-    // used by test:uiRelease task
-    //    runs the kibana server from the oss distributable prepared for the functional tests
-    ossDistFuncTestServer: createKbnServerTask({
-      runBuild: `oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
-      flags: [
-        ...funcTestServerFlags,
-      ]
-    }),
-
     // used by the test:browser task
     //    runs the kibana server to serve the browser test bundle
     browserTestServer: createKbnServerTask({
@@ -198,22 +131,6 @@ module.exports = function (grunt) {
       ]
     }),
 
-    testEsServer: {
-      options: {
-        wait: false,
-        ready: /started/,
-        quiet: false,
-      },
-      cmd: process.execPath,
-      args: [
-        'scripts/es',
-        grunt.option('from') || 'snapshot',
-        '--license', 'oss',
-        '-E', `http.port=${esTestConfig.getPort()}`,
-        '-E', `discovery.zen.ping.unicast.hosts=localhost:${esTestConfig.getPort()}`,
-      ],
-    },
-
     verifyNotice: {
       options: {
         wait: true,
@@ -223,6 +140,56 @@ module.exports = function (grunt) {
         'scripts/notice',
         '--validate'
       ]
-    }
+    },
+
+    apiIntegrationTests: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/api_integration/config.js',
+        '--esFrom', 'source',
+        '--bail',
+        '--debug',
+      ],
+    },
+
+    functionalTests: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/functional/config.js',
+        '--esFrom', 'source',
+        '--bail',
+        '--debug',
+        '--',
+        '--server.maxPayloadBytes=1648576',
+      ],
+    },
+
+    functionalTestsRelease: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/functional/config.js',
+        '--esFrom', 'source',
+        '--bail',
+        '--debug',
+        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
+        '--',
+        '--server.maxPayloadBytes=1648576',
+      ],
+    },
+
+    functionalTestsDevServer: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests_server',
+        '--config', 'test/functional/config.js',
+        '--esFrom', 'source',
+        '--debug',
+        '--',
+        '--server.maxPayloadBytes=1648576',
+      ],
+    },
   };
 };

tasks/jenkins.js

@@ -34,11 +34,10 @@ module.exports = function (grunt) {
     'test:jest_integration',
     'test:projects',
     'test:browser-ci',
-    'test:api',
+    'run:apiIntegrationTests',
     'verifyTranslations',
   ]);
 
-  grunt.config.set('functional_test_runner.functional.options.configOverrides.mochaOpts.bail', true);
   grunt.registerTask('jenkins:selenium', [
     'test:uiRelease'
   ]);

tasks/test.js

@@ -64,7 +64,7 @@ module.exports = function (grunt) {
     'test:jest_integration',
     'test:projects',
     'test:browser',
-    'test:api'
+    'run:apiIntegrationTests'
   ]);
 
   grunt.registerTask('test:dev', [
@@ -75,45 +75,19 @@ module.exports = function (grunt) {
 
   grunt.registerTask('test:ui', [
     'checkPlugins',
-    'run:testEsServer',
-    'run:funcTestServer',
-    'functional_test_runner:functional',
-    'stop:testEsServer',
-    'stop:funcTestServer'
+    'run:functionalTests',
   ]);
 
   grunt.registerTask('test:uiRelease', [
     'checkPlugins',
-    'run:testEsServer',
-    'run:ossDistFuncTestServer',
-    'functional_test_runner:functional',
-    'stop:testEsServer',
-    'stop:ossDistFuncTestServer'
+    'run:functionalTestsRelease',
   ]);
 
   grunt.registerTask('test:ui:server', [
     'checkPlugins',
-    'run:testEsServer',
-    'run:devFuncTestServer:keepalive'
+    'run:functionalTestsDevServer',
   ]);
 
-  grunt.registerTask('test:api', [
-    'run:testEsServer',
-    'run:apiTestServer',
-    'functional_test_runner:apiIntegration',
-    'stop:testEsServer',
-    'stop:apiTestServer'
-  ]);
-
-  grunt.registerTask('test:api:server', [
-    'run:testEsServer',
-    'run:devApiTestServer:keepalive'
-  ]);
-
-  grunt.registerTask('test:api:runner', () => {
-    grunt.fail.fatal('test:api:runner has moved, use: `node scripts/functional_test_runner --config test/api_integration/config.js`');
-  });
-
   grunt.registerTask('test', subTask => {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 

test/common/config.js

@@ -49,7 +49,7 @@ export default function () {
         `--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
       ],
       serverArgs: [
-        '--env=development',
+        '--env.name=development',
         '--logging.json=false',
         `--server.port=${kbnTestConfig.getPort()}`,
         `--optimize.watchPort=${kbnTestConfig.getPort() + 10}`,

test/scripts/jenkins_xpack.sh

@@ -37,6 +37,6 @@ tar -xzf "$linuxBuild" -C "$installDir" --strip=1
 
 echo " -> Running functional and api tests"
 cd "$XPACK_DIR"
-xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --es-from=source
+xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --esFrom=source
 echo ""
 echo ""