diff --git a/src/functional_test_runner/cli.js b/src/functional_test_runner/cli.js index 6998a1b8bfb7..9414dcfe1cfd 100644 --- a/src/functional_test_runner/cli.js +++ b/src/functional_test_runner/cli.js @@ -48,6 +48,7 @@ cmd .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), []) .option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), []) .option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), []) + .option('--test-stats', 'Print the number of tests (included and excluded) to STDERR', false) .option('--verbose', 'Log everything', false) .option('--quiet', 'Only log errors', false) .option('--silent', 'Log nothing', false) @@ -86,8 +87,16 @@ const functionalTestRunner = createFunctionalTestRunner({ async function run() { try { - const failureCount = await functionalTestRunner.run(); - process.exitCode = failureCount ? 1 : 0; + if (cmd.testStats) { + process.stderr.write(JSON.stringify( + await functionalTestRunner.getTestStats(), + null, + 2 + ) + '\n'); + } else { + const failureCount = await functionalTestRunner.run(); + process.exitCode = failureCount ? 1 : 0; + } } catch (err) { await teardown(err); } finally { diff --git a/src/functional_test_runner/functional_test_runner.js b/src/functional_test_runner/functional_test_runner.js index 65d347fbee45..d32ad172d99b 100644 --- a/src/functional_test_runner/functional_test_runner.js +++ b/src/functional_test_runner/functional_test_runner.js @@ -20,7 +20,8 @@ import { createLifecycle, readConfigFile, - createProviderCollection, + ProviderCollection, + readProviderSpec, setupMocha, runTests, } from './lib'; @@ -36,8 +37,61 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides }) log.verbose('ending %j lifecycle phase', name); }); + class FunctionalTestRunner { async run() { + return await this._run(async (config, coreProviders) => { + const providers = new ProviderCollection(log, [ + ...coreProviders, + ...readProviderSpec('Service', config.get('services')), + ...readProviderSpec('PageObject', config.get('pageObjects')) + ]); + + await providers.loadAll(); + + const mocha = await setupMocha(lifecycle, log, config, providers); + await lifecycle.trigger('beforeTests'); + log.info('Starting tests'); + + return await runTests(lifecycle, log, mocha); + }); + } + + async getTestStats() { + return await this._run(async (config, coreProviders) => { + // replace the function of a provider so that it returns a promise-like object which + // never "resolves", essentially disabling all the services while still allowing us + // to load the test files and populate the mocha suites + const stubProvider = provider => ({ + ...provider, + fn: () => ({ + then: () => {} + }) + }); + + const providers = new ProviderCollection(log, [ + ...coreProviders, + ...readProviderSpec('Service', config.get('services')), + ...readProviderSpec('PageObject', config.get('pageObjects')) + ].map(stubProvider)); + + const mocha = await setupMocha(lifecycle, log, config, providers); + + const countTests = suite => ( + suite.suites.reduce( + (sum, suite) => sum + countTests(suite), + suite.tests.length + ) + ); + + return { + tests: countTests(mocha.suite), + excludedTests: mocha.excludedTests.length + }; + }); + } + + async _run(handler) { let runErrorOccurred = false; try { @@ -49,14 +103,14 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides }) return; } - const providers = createProviderCollection(lifecycle, log, config); - await providers.loadAll(); - - const mocha = await setupMocha(lifecycle, log, config, providers); - await lifecycle.trigger('beforeTests'); - log.info('Starting tests'); - return await runTests(lifecycle, log, mocha); + // base level services that functional_test_runner exposes + const coreProviders = readProviderSpec('Service', { + lifecycle: () => lifecycle, + log: () => log, + config: () => config, + }); + return await handler(config, coreProviders); } catch (runError) { runErrorOccurred = true; throw runError; diff --git a/src/functional_test_runner/lib/create_provider_collection.js b/src/functional_test_runner/lib/create_provider_collection.js deleted file mode 100644 index e91f473290ec..000000000000 --- a/src/functional_test_runner/lib/create_provider_collection.js +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { - ProviderCollection, - readProviderSpec -} from './providers'; - -/** - * Create a ProviderCollection that includes the Service - * providers and PageObject providers from config, as well - * providers for the default services, lifecycle, log, and - * config - * - * @param {Lifecycle} lifecycle - * @param {ToolingLog} log - * @param {Config} config [description] - * @return {ProviderCollection} - */ -export function createProviderCollection(lifecycle, log, config) { - return new ProviderCollection(log, [ - ...readProviderSpec('Service', { - // base level services that functional_test_runner exposes - lifecycle: () => lifecycle, - log: () => log, - config: () => config, - - ...config.get('services'), - }), - ...readProviderSpec('PageObject', { - ...config.get('pageObjects') - }) - ]); -} diff --git a/src/functional_test_runner/lib/index.js b/src/functional_test_runner/lib/index.js index 1a933447bb0f..e1f93d8ac627 100644 --- a/src/functional_test_runner/lib/index.js +++ b/src/functional_test_runner/lib/index.js @@ -19,5 +19,5 @@ export { createLifecycle } from './lifecycle'; export { readConfigFile } from './config'; -export { createProviderCollection } from './create_provider_collection'; export { setupMocha, runTests } from './mocha'; +export { readProviderSpec, ProviderCollection } from './providers'; diff --git a/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js index fd15f936c762..597be9bab32a 100644 --- a/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js +++ b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js @@ -28,6 +28,14 @@ * @param options.exclude an array of tags that will be used to exclude suites from the run */ export function filterSuitesByTags({ log, mocha, include, exclude }) { + mocha.excludedTests = []; + // collect all the tests from some suite, including it's children + const collectTests = (suite) => + suite.suites.reduce( + (acc, s) => acc.concat(collectTests(s)), + suite.tests + ); + // if include tags were provided, filter the tree once to // only include branches that are included at some point if (include.length) { @@ -47,14 +55,18 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) { continue; } + // this suite has an included child but is not included // itself, so strip out its tests and recurse to filter // out child suites which are not included if (isChildIncluded(child)) { + mocha.excludedTests = mocha.excludedTests.concat(child.tests); child.tests = []; parentSuite.suites.push(child); recurse(child); continue; + } else { + mocha.excludedTests = mocha.excludedTests.concat(collectTests(child)); } } }(mocha.suite)); @@ -78,6 +90,8 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) { if (isNotExcluded(child)) { parentSuite.suites.push(child); recurse(child); + } else { + mocha.excludedTests = mocha.excludedTests.concat(collectTests(child)); } } }(mocha.suite));