From b0d11e5607d2f950e418987cc54243e0a8e7e11b Mon Sep 17 00:00:00 2001 From: Spencer Date: Fri, 16 Nov 2018 16:30:55 -0800 Subject: [PATCH] [kbn/test] add support for using test groups (#25776) --- packages/kbn-dev-utils/index.d.ts | 1 + packages/kbn-dev-utils/src/index.js | 1 + .../serializers/absolute_path_serializer.d.ts | 22 ++++++ .../serializers/absolute_path_serializer.js} | 31 ++------ .../kbn-dev-utils/src/serializers/index.d.ts | 20 +++++ .../kbn-dev-utils/src/serializers/index.js | 20 +++++ .../run_tests/__snapshots__/args.test.js.snap | 31 +++++--- .../run_tests/__snapshots__/cli.test.js.snap | 1 + .../functional_tests/cli/run_tests/args.js | 10 ++- .../cli/run_tests/args.test.js | 3 + .../cli/run_tests/cli.test.js | 4 +- .../__snapshots__/args.test.js.snap | 32 ++------ .../cli/start_servers/args.js | 4 +- .../cli/start_servers/args.test.js | 25 +++--- .../cli/start_servers/cli.test.js | 4 +- .../src/functional_tests/lib/fun_ftr.js | 69 +++++++++++++++++ .../src/functional_tests/lib/index.js | 2 +- .../kbn-test/src/functional_tests/tasks.js | 77 +++++++++++-------- 18 files changed, 248 insertions(+), 109 deletions(-) create mode 100644 packages/kbn-dev-utils/src/serializers/absolute_path_serializer.d.ts rename packages/{kbn-test/src/functional_tests/lib/run_ftr.js => kbn-dev-utils/src/serializers/absolute_path_serializer.js} (55%) create mode 100644 packages/kbn-dev-utils/src/serializers/index.d.ts create mode 100644 packages/kbn-dev-utils/src/serializers/index.js create mode 100644 packages/kbn-test/src/functional_tests/lib/fun_ftr.js diff --git a/packages/kbn-dev-utils/index.d.ts b/packages/kbn-dev-utils/index.d.ts index 9d8fd8889b7e..dceadb9496f3 100644 --- a/packages/kbn-dev-utils/index.d.ts +++ b/packages/kbn-dev-utils/index.d.ts @@ -18,3 +18,4 @@ */ export * from './src/tooling_log'; +export * from './src/serializers'; diff --git a/packages/kbn-dev-utils/src/index.js b/packages/kbn-dev-utils/src/index.js index 4e2d1b62a92f..82492e568f4b 100644 --- a/packages/kbn-dev-utils/src/index.js +++ b/packages/kbn-dev-utils/src/index.js @@ -19,3 +19,4 @@ export { withProcRunner } from './proc_runner'; export { ToolingLog, ToolingLogTextWriter, pickLevelFromFlags } from './tooling_log'; +export { createAbsolutePathSerializer } from './serializers'; diff --git a/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.d.ts b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.d.ts new file mode 100644 index 000000000000..66e94571a230 --- /dev/null +++ b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.d.ts @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export function createAbsolutePathSerializer( + rootPath: string +): { print(...args: any[]): string; test(value: any): boolean }; diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.js similarity index 55% rename from packages/kbn-test/src/functional_tests/lib/run_ftr.js rename to packages/kbn-dev-utils/src/serializers/absolute_path_serializer.js index 779286212f5c..d944772048eb 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.js @@ -17,30 +17,9 @@ * under the License. */ -import { createFunctionalTestRunner } from '../../../../../src/functional_test_runner'; -import { CliError } from './run_cli'; - -export async function runFtr({ - configPath, - options: { log, bail, grep, updateBaselines, suiteTags }, -}) { - const ftr = createFunctionalTestRunner({ - log, - configFile: configPath, - configOverrides: { - mochaOpts: { - bail: !!bail, - grep, - }, - updateBaselines, - suiteTags, - }, - }); - - const failureCount = await ftr.run(); - if (failureCount > 0) { - throw new CliError( - `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}` - ); - } +export function createAbsolutePathSerializer(rootPath) { + return { + print: value => value.replace(rootPath, '').replace(/\\/g, '/'), + test: value => typeof value === 'string' && value.startsWith(rootPath), + }; } diff --git a/packages/kbn-dev-utils/src/serializers/index.d.ts b/packages/kbn-dev-utils/src/serializers/index.d.ts new file mode 100644 index 000000000000..3b49e243058d --- /dev/null +++ b/packages/kbn-dev-utils/src/serializers/index.d.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { createAbsolutePathSerializer } from './absolute_path_serializer'; diff --git a/packages/kbn-dev-utils/src/serializers/index.js b/packages/kbn-dev-utils/src/serializers/index.js new file mode 100644 index 000000000000..3b49e243058d --- /dev/null +++ b/packages/kbn-dev-utils/src/serializers/index.js @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { createAbsolutePathSerializer } from './absolute_path_serializer'; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap index 93d129213ff0..c55b8abf6c78 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap @@ -18,6 +18,7 @@ Options: --updateBaselines Replace baseline screenshots with whatever is generated from the test. --include-tag Tags that suites must include to be run, can be included multiple times. --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. + --assert-none-excluded Exit with 1/0 based on if any test is excluded with the current set of tags. --verbose Log everything. --debug Run in debug mode. --quiet Only log errors. @@ -26,8 +27,9 @@ Options: exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, @@ -41,8 +43,9 @@ Object { exports[`process options for run tests CLI accepts debug option 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "debug": true, @@ -56,9 +59,10 @@ Object { exports[`process options for run tests CLI accepts empty config value if default passed 1`] = ` Object { + "assertNoneExcluded": false, "config": "", "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, @@ -74,8 +78,9 @@ Object { "_": Object { "server.foo": "bar", }, + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": Object { @@ -90,8 +95,9 @@ Object { exports[`process options for run tests CLI accepts quiet option 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, @@ -105,8 +111,9 @@ Object { exports[`process options for run tests CLI accepts silent option 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, @@ -120,8 +127,9 @@ Object { exports[`process options for run tests CLI accepts source value for esFrom 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "esFrom": "source", @@ -135,8 +143,9 @@ Object { exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, @@ -150,8 +159,9 @@ Object { exports[`process options for run tests CLI accepts value for grep 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, @@ -165,8 +175,9 @@ Object { exports[`process options for run tests CLI accepts verbose option 1`] = ` Object { + "assertNoneExcluded": false, "configs": Array [ - "foo", + /foo, ], "createLogger": [Function], "extraKbnOpts": undefined, diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap index 6d4b35f754b4..e9f3dfac5674 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap @@ -18,6 +18,7 @@ Options: --updateBaselines Replace baseline screenshots with whatever is generated from the test. --include-tag Tags that suites must include to be run, can be included multiple times. --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. + --assert-none-excluded Exit with 1/0 based on if any test is excluded with the current set of tags. --verbose Log everything. --debug Run in debug mode. --quiet Only log errors. diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js index 6aa1e1aa5c31..eb7809b91ceb 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js @@ -17,6 +17,8 @@ * under the License. */ +import { resolve } from 'path'; + import dedent from 'dedent'; import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; @@ -52,6 +54,9 @@ const options = { arg: '', desc: 'Tags that suites must NOT include to be run, can be included multiple times.', }, + 'assert-none-excluded': { + desc: 'Exit with 1/0 based on if any test is excluded with the current set of tags.', + }, verbose: { desc: 'Log everything.' }, debug: { desc: 'Run in debug mode.' }, quiet: { desc: 'Only log errors.' }, @@ -113,6 +118,9 @@ export function processOptions(userOptions, defaultConfigPaths) { delete userOptions['include-tag']; delete userOptions['exclude-tag']; + userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded']; + delete userOptions['assert-none-excluded']; + function createLogger() { return new ToolingLog({ level: pickLevelFromFlags(userOptions), @@ -122,7 +130,7 @@ export function processOptions(userOptions, defaultConfigPaths) { return { ...userOptions, - configs, + configs: configs.map(c => resolve(c)), createLogger, extraKbnOpts: userOptions._, }; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js index 2e0c2fb44022..a6d5bf72ffc8 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js @@ -18,6 +18,9 @@ */ import { displayHelp, processOptions } from './args'; +import { createAbsolutePathSerializer } from '@kbn/dev-utils'; + +expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd())); describe('display help for run tests CLI', () => { it('displays as expected', () => { diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js index 7e2e6c5a40e7..90caaf2cc6ce 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js @@ -17,6 +17,8 @@ * under the License. */ +import { Writable } from 'stream'; + import { runTestsCli } from './cli'; import { checkMockConsoleLogSnapshot } from '../../test_helpers'; @@ -36,7 +38,7 @@ describe('run tests CLI', () => { const processMock = { exit: exitMock, argv: argvMock, - stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() }, + stdout: new Writable(), cwd: jest.fn(), }; diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap index 50da9f683688..106ddcb00faf 100644 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap @@ -21,9 +21,7 @@ Options: exports[`process options for start servers CLI accepts debug option 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "debug": true, "esFrom": "snapshot", @@ -33,9 +31,7 @@ Object { exports[`process options for start servers CLI accepts empty config value if default passed 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, @@ -47,9 +43,7 @@ Object { "_": Object { "server.foo": "bar", }, - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": Object { @@ -60,9 +54,7 @@ Object { exports[`process options for start servers CLI accepts quiet option 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, @@ -72,9 +64,7 @@ Object { exports[`process options for start servers CLI accepts silent option 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, @@ -84,9 +74,7 @@ Object { exports[`process options for start servers CLI accepts source value for esFrom 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "source", "extraKbnOpts": undefined, @@ -95,9 +83,7 @@ Object { exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, @@ -107,9 +93,7 @@ Object { exports[`process options for start servers CLI accepts verbose option 1`] = ` Object { - "config": Array [ - "foo", - ], + "config": /foo, "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js index 170de1942232..026c23ab46f3 100644 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js @@ -17,6 +17,8 @@ * under the License. */ +import { resolve } from 'path'; + import dedent from 'dedent'; import { ToolingLog, pickLevelFromFlags } from '@kbn/dev-utils'; @@ -97,7 +99,7 @@ export function processOptions(userOptions, defaultConfigPath) { return { ...userOptions, - config, + config: resolve(config), createLogger, extraKbnOpts: userOptions._, }; diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js index 22c06b6ae323..bc2b958d793f 100644 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js @@ -18,6 +18,9 @@ */ import { displayHelp, processOptions } from './args'; +import { createAbsolutePathSerializer } from '@kbn/dev-utils'; + +expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd())); describe('display help for start servers CLI', () => { it('displays as expected', () => { @@ -39,60 +42,60 @@ describe('process options for start servers CLI', () => { }); it('accepts empty config value if default passed', () => { - const options = processOptions({ config: '' }, ['foo']); + const options = processOptions({ config: '' }, 'foo'); expect(options).toMatchSnapshot(); }); it('rejects invalid option', () => { expect(() => { - processOptions({ bail: true }, ['foo']); + processOptions({ bail: true }, 'foo'); }).toThrow('functional_tests_server: invalid option [bail]'); }); it('accepts string value for kibana-install-dir', () => { - const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']); + const options = processOptions({ 'kibana-install-dir': 'foo' }, 'foo'); expect(options).toMatchSnapshot(); }); it('rejects boolean value for kibana-install-dir', () => { expect(() => { - processOptions({ 'kibana-install-dir': true }, ['foo']); + processOptions({ 'kibana-install-dir': true }, 'foo'); }).toThrow('functional_tests_server: invalid argument [true] to option [kibana-install-dir]'); }); it('accepts source value for esFrom', () => { - const options = processOptions({ esFrom: 'source' }, ['foo']); + const options = processOptions({ esFrom: 'source' }, 'foo'); expect(options).toMatchSnapshot(); }); it('accepts debug option', () => { - const options = processOptions({ debug: true }, ['foo']); + const options = processOptions({ debug: true }, 'foo'); expect(options).toMatchSnapshot(); }); it('accepts silent option', () => { - const options = processOptions({ silent: true }, ['foo']); + const options = processOptions({ silent: true }, 'foo'); expect(options).toMatchSnapshot(); }); it('accepts quiet option', () => { - const options = processOptions({ quiet: true }, ['foo']); + const options = processOptions({ quiet: true }, 'foo'); expect(options).toMatchSnapshot(); }); it('accepts verbose option', () => { - const options = processOptions({ verbose: true }, ['foo']); + const options = processOptions({ verbose: true }, 'foo'); expect(options).toMatchSnapshot(); }); it('accepts extra server options', () => { - const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']); + const options = processOptions({ _: { 'server.foo': 'bar' } }, 'foo'); expect(options).toMatchSnapshot(); }); it('rejects invalid options even if valid options exist', () => { expect(() => { - processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']); + processOptions({ debug: true, aintnothang: true, bail: true }, 'foo'); }).toThrow('functional_tests_server: invalid option [aintnothang]'); }); }); diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js index 43bab5a4afe8..5a195c813f39 100644 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js +++ b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.test.js @@ -17,6 +17,8 @@ * under the License. */ +import { Writable } from 'stream'; + import { startServersCli } from './cli'; import { checkMockConsoleLogSnapshot } from '../../test_helpers'; @@ -36,7 +38,7 @@ describe('start servers CLI', () => { const processMock = { exit: exitMock, argv: argvMock, - stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() }, + stdout: new Writable(), cwd: jest.fn(), }; diff --git a/packages/kbn-test/src/functional_tests/lib/fun_ftr.js b/packages/kbn-test/src/functional_tests/lib/fun_ftr.js new file mode 100644 index 000000000000..e3dbe6cef6d8 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/lib/fun_ftr.js @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as FunctionalTestRunner from '../../../../../src/functional_test_runner'; +import { CliError } from './run_cli'; + +function createFtr({ configPath, options: { log, bail, grep, updateBaselines, suiteTags } }) { + return FunctionalTestRunner.createFunctionalTestRunner({ + log, + configFile: configPath, + configOverrides: { + mochaOpts: { + bail: !!bail, + grep, + }, + updateBaselines, + suiteTags, + }, + }); +} + +export async function assertNoneExcluded({ configPath, options }) { + const ftr = createFtr({ configPath, options }); + + const stats = await ftr.getTestStats(); + if (stats.excludedTests > 0) { + throw new CliError(` + ${stats.excludedTests} tests in the ${configPath} config + are excluded when filtering by the tags run on CI. Make sure that all suites are + tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh + + ${JSON.stringify(options.suiteTags)} + + `); + } +} + +export async function runFtr({ configPath, options }) { + const ftr = createFtr({ configPath, options }); + + const failureCount = await ftr.run(); + if (failureCount > 0) { + throw new CliError( + `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}` + ); + } +} + +export async function hasTests({ configPath, options }) { + const ftr = createFtr({ configPath, options }); + const stats = await ftr.getTestStats(); + return stats.tests > 0; +} diff --git a/packages/kbn-test/src/functional_tests/lib/index.js b/packages/kbn-test/src/functional_tests/lib/index.js index d66f641c10f6..886a77ba6e2e 100644 --- a/packages/kbn-test/src/functional_tests/lib/index.js +++ b/packages/kbn-test/src/functional_tests/lib/index.js @@ -19,6 +19,6 @@ export { runKibanaServer } from './run_kibana_server'; export { runElasticsearch } from './run_elasticsearch'; -export { runFtr } from './run_ftr'; +export { runFtr, hasTests, assertNoneExcluded } from './fun_ftr'; export { KIBANA_ROOT, KIBANA_FTR_SCRIPT, FUNCTIONAL_CONFIG_PATH, API_CONFIG_PATH } from './paths'; export { runCli } from './run_cli'; diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js index 0cdcc77161a6..b7f22836ab76 100644 --- a/packages/kbn-test/src/functional_tests/tasks.js +++ b/packages/kbn-test/src/functional_tests/tasks.js @@ -17,12 +17,19 @@ * under the License. */ -import { relative, resolve } from 'path'; +import { relative } from 'path'; import * as Rx from 'rxjs'; import { startWith, switchMap, take } from 'rxjs/operators'; import { withProcRunner } from '@kbn/dev-utils'; -import { runElasticsearch, runKibanaServer, runFtr, KIBANA_FTR_SCRIPT } from './lib'; +import { + runElasticsearch, + runKibanaServer, + runFtr, + assertNoneExcluded, + hasTests, + KIBANA_FTR_SCRIPT, +} from './lib'; import { readConfigFile } from '../../../../src/functional_test_runner/lib'; @@ -38,37 +45,63 @@ in another terminal session by running this command from this directory: /** * Run servers and tests for each config * @param {object} options Optional - * @property {string[]} configPaths Array of paths to configs - * @property {function} options.createLogger Optional logger creation function + * @property {string[]} options.configs Array of paths to configs + * @property {function} options.log An instance of the ToolingLog * @property {string} options.installDir Optional installation dir from which to run Kibana * @property {boolean} options.bail Whether to exit test run at the first failure * @property {string} options.esFrom Optionally run from source instead of snapshot */ export async function runTests(options) { for (const configPath of options.configs) { - await runSingleConfig(resolve(process.cwd(), configPath), options); + const log = options.createLogger(); + const opts = { + ...options, + log, + }; + + log.info('Running', configPath); + log.indent(2); + + if (options.assertNoneExcluded) { + await assertNoneExcluded({ configPath, options: opts }); + continue; + } + + if (!(await hasTests({ configPath, options: opts }))) { + log.info('Skipping', configPath, 'since all tests are excluded'); + continue; + } + + await withProcRunner(log, async procs => { + const config = await readConfigFile(log, configPath); + + const es = await runElasticsearch({ config, options: opts }); + await runKibanaServer({ procs, config, options: opts }); + await runFtr({ configPath, options: opts }); + + await procs.stop('kibana'); + await es.cleanup(); + }); } } /** * Start only servers using single config * @param {object} options Optional - * @property {string} options.configPath Path to a config file - * @property {function} options.createLogger Optional logger creation function + * @property {string} options.config Path to a config file + * @property {function} options.log An instance of the ToolingLog * @property {string} options.installDir Optional installation dir from which to run Kibana * @property {string} options.esFrom Optionally run from source instead of snapshot */ export async function startServers(options) { - const { config: configOption, createLogger } = options; - const configPath = resolve(process.cwd(), configOption); - const log = createLogger(); + const log = options.createLogger(); const opts = { ...options, log, }; await withProcRunner(log, async procs => { - const config = await readConfigFile(log, configPath); + const config = await readConfigFile(log, options.config); const es = await runElasticsearch({ config, options: opts }); await runKibanaServer({ @@ -100,25 +133,3 @@ async function silence(milliseconds, { log }) { ) .toPromise(); } - -/* - * Start servers and run tests for single config - */ -async function runSingleConfig(configPath, options) { - const log = options.createLogger(); - const opts = { - ...options, - log, - }; - - await withProcRunner(log, async procs => { - const config = await readConfigFile(log, configPath); - - const es = await runElasticsearch({ config, options: opts }); - await runKibanaServer({ procs, config, options: opts }); - await runFtr({ configPath, options: opts }); - - await procs.stop('kibana'); - await es.cleanup(); - }); -}