From 4122705bc2c6abc1602980ca7ed42513d1e14ec8 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 26 Feb 2020 13:32:51 -0700 Subject: [PATCH] [kbn/test/ftr] add support for custom runners (#58610) * [kbn/test/ftr] add support for custom runners * remove irrelevant tests --- .../functional_test_runner.ts | 19 +++++++- .../lib/config/__tests__/read_config_file.js | 14 ------ .../lib/config/schema.ts | 9 +--- .../lib/providers/provider_collection.ts | 16 ++++--- .../src/functional_tests/cli/index.js | 2 + .../src/functional_tests/lib/run_ftr.js | 47 ++++++++++++------- packages/kbn-test/src/index.ts | 11 ++++- 7 files changed, 70 insertions(+), 48 deletions(-) diff --git a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts index e566a9a4af26..03075dd4081f 100644 --- a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts +++ b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts @@ -60,6 +60,14 @@ export class FunctionalTestRunner { await providers.loadAll(); + const customTestRunner = config.get('testRunner'); + if (customTestRunner) { + this.log.warning( + 'custom test runner defined, ignoring all mocha/suite/filtering related options' + ); + return (await providers.invokeProviderFn(customTestRunner)) || 0; + } + const mocha = await setupMocha(this.lifecycle, this.log, config, providers); await this.lifecycle.beforeTests.trigger(); this.log.info('Starting tests'); @@ -70,6 +78,10 @@ export class FunctionalTestRunner { async getTestStats() { return await this._run(async (config, coreProviders) => { + if (config.get('testRunner')) { + throw new Error('Unable to get test stats for config that uses a custom test runner'); + } + // replace the function of custom service providers so that they return // promise-like objects which never resolve, essentially disabling them // allowing us to load the test files and populate the mocha suites @@ -108,8 +120,11 @@ export class FunctionalTestRunner { const config = await readConfigFile(this.log, this.configFile, this.configOverrides); this.log.info('Config loaded'); - if (config.get('testFiles').length === 0) { - throw new Error('No test files defined.'); + if ( + (!config.get('testFiles') || config.get('testFiles').length === 0) && + !config.get('testRunner') + ) { + throw new Error('No tests defined.'); } // base level services that functional_test_runner exposes diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/__tests__/read_config_file.js b/packages/kbn-test/src/functional_test_runner/lib/config/__tests__/read_config_file.js index 325e972a6cd9..8d02e7262409 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/__tests__/read_config_file.js +++ b/packages/kbn-test/src/functional_test_runner/lib/config/__tests__/read_config_file.js @@ -55,18 +55,4 @@ describe('readConfigFile()', () => { expect(err.message).to.match(/"foo"/); } }); - - it('throws if config does not define testFiles', async () => { - try { - await readConfigFile(log, require.resolve('./fixtures/config.4')); - throw new Error('expected readConfigFile() to fail'); - } catch (err) { - expect(err.message).to.match(/"testFiles"/); - } - }); - - it('does not throw if child config file does not have any testFiles', async () => { - const config = await readConfigFile(log, require.resolve('./fixtures/config.3')); - expect(config.get('screenshots.directory')).to.be('bar'); - }); }); diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 4530b6142362..75623d6c0889 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -61,13 +61,8 @@ const defaultRelativeToConfigPath = (path: string) => { export const schema = Joi.object() .keys({ - testFiles: Joi.array() - .items(Joi.string()) - .when('$primary', { - is: true, - then: Joi.required(), - otherwise: Joi.any().default([]), - }), + testFiles: Joi.array().items(Joi.string()), + testRunner: Joi.func(), excludeTestFiles: Joi.array() .items(Joi.string()) diff --git a/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts b/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts index a5d192a1f720..f9ad86be634f 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts @@ -68,6 +68,15 @@ export class ProviderCollection { } } + public invokeProviderFn(provider: (args: any) => any) { + return provider({ + getService: this.getService, + hasService: this.hasService, + getPageObject: this.getPageObject, + getPageObjects: this.getPageObjects, + }); + } + private findProvider(type: string, name: string) { return this.providers.find(p => p.type === type && p.name === name); } @@ -89,12 +98,7 @@ export class ProviderCollection { } if (!instances.has(provider)) { - let instance = provider({ - getService: this.getService, - hasService: this.hasService, - getPageObject: this.getPageObject, - getPageObjects: this.getPageObjects, - }); + let instance = this.invokeProviderFn(provider); if (instance && typeof instance.then === 'function') { instance = createAsyncInstance(type, name, instance); diff --git a/packages/kbn-test/src/functional_tests/cli/index.js b/packages/kbn-test/src/functional_tests/cli/index.js index 3c6280d91f58..4671a4f652af 100644 --- a/packages/kbn-test/src/functional_tests/cli/index.js +++ b/packages/kbn-test/src/functional_tests/cli/index.js @@ -18,4 +18,6 @@ */ export { runTestsCli } from './run_tests/cli'; +export { processOptions as processRunTestsCliOptions } from './run_tests/args'; export { startServersCli } from './start_servers/cli'; +export { processOptions as processStartServersCliOptions } from './start_servers/args'; diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js index aeda84f9524e..9b631e33f3b2 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -26,24 +26,32 @@ async function createFtr({ }) { const config = await readConfigFile(log, configPath); - return new FunctionalTestRunner(log, configPath, { - mochaOpts: { - bail: !!bail, - grep, - }, - kbnTestServer: { - installDir, - }, - updateBaselines, - suiteTags: { - include: [...suiteTags.include, ...config.get('suiteTags.include')], - exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')], - }, - }); + return { + config, + ftr: new FunctionalTestRunner(log, configPath, { + mochaOpts: { + bail: !!bail, + grep, + }, + kbnTestServer: { + installDir, + }, + updateBaselines, + suiteTags: { + include: [...suiteTags.include, ...config.get('suiteTags.include')], + exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')], + }, + }), + }; } export async function assertNoneExcluded({ configPath, options }) { - const ftr = await createFtr({ configPath, options }); + const { config, ftr } = await createFtr({ configPath, options }); + + if (config.get('testRunner')) { + // tests with custom test runners are not included in this check + return; + } const stats = await ftr.getTestStats(); if (stats.excludedTests.length > 0) { @@ -60,7 +68,7 @@ export async function assertNoneExcluded({ configPath, options }) { } export async function runFtr({ configPath, options }) { - const ftr = await createFtr({ configPath, options }); + const { ftr } = await createFtr({ configPath, options }); const failureCount = await ftr.run(); if (failureCount > 0) { @@ -71,7 +79,12 @@ export async function runFtr({ configPath, options }) { } export async function hasTests({ configPath, options }) { - const ftr = await createFtr({ configPath, options }); + const { ftr, config } = await createFtr({ configPath, options }); + + if (config.get('testRunner')) { + // configs with custom test runners are assumed to always have tests + return true; + } const stats = await ftr.getTestStats(); return stats.testCount > 0; } diff --git a/packages/kbn-test/src/index.ts b/packages/kbn-test/src/index.ts index 06a83cdd8b1d..cfbd1ee0fe64 100644 --- a/packages/kbn-test/src/index.ts +++ b/packages/kbn-test/src/index.ts @@ -17,8 +17,15 @@ * under the License. */ -// @ts-ignore not typed yet -export { runTestsCli, startServersCli } from './functional_tests/cli'; +import { + runTestsCli, + processRunTestsCliOptions, + startServersCli, + processStartServersCliOptions, + // @ts-ignore not typed yet +} from './functional_tests/cli'; + +export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions }; // @ts-ignore not typed yet export { runTests, startServers } from './functional_tests/tasks';