maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ftr] add ability to get included/excluded test counts (#25760)

Browse source 
## Summary

This pr adds a `--test-stats` flag to the functional test runner that will allow extracting information from a specific config/tags combo about how many tests will run and how many are excluded. This is necessary as part of https://github.com/elastic/kibana/pull/22359 so we can effectively ignore specific configs (avoiding starting es and kibana) when there aren't any tests that will be run.

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~
~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~
~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~
~~- [ ] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios~~
~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~
~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~
This commit is contained in:
Spencer 2018-11-16 15:48:07 -08:00 committed by GitHub
parent df8514fa09
commit f7832a514c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 88 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/functional_test_runner/cli.js
View file

@ -48,6 +48,7 @@ cmd
.option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), [])
.option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), [])
.option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), [])
.option('--test-stats', 'Print the number of tests (included and excluded) to STDERR', false)
.option('--verbose', 'Log everything', false)
.option('--quiet', 'Only log errors', false)
.option('--silent', 'Log nothing', false)
@ -86,8 +87,16 @@ const functionalTestRunner = createFunctionalTestRunner({
async function run() {
try {
const failureCount = await functionalTestRunner.run();
process.exitCode = failureCount ? 1 : 0;
if (cmd.testStats) {
process.stderr.write(JSON.stringify(
await functionalTestRunner.getTestStats(),
null,
2
) + '\n');
} else {
const failureCount = await functionalTestRunner.run();
process.exitCode = failureCount ? 1 : 0;
}
} catch (err) {
await teardown(err);
} finally {

70
src/functional_test_runner/functional_test_runner.js
View file

@ -20,7 +20,8 @@
import {
createLifecycle,
readConfigFile,
createProviderCollection,
ProviderCollection,
readProviderSpec,
setupMocha,
runTests,
} from './lib';
@ -36,8 +37,61 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
log.verbose('ending %j lifecycle phase', name);
});
class FunctionalTestRunner {
async run() {
return await this._run(async (config, coreProviders) => {
const providers = new ProviderCollection(log, [
...coreProviders,
...readProviderSpec('Service', config.get('services')),
...readProviderSpec('PageObject', config.get('pageObjects'))
]);
await providers.loadAll();
const mocha = await setupMocha(lifecycle, log, config, providers);
await lifecycle.trigger('beforeTests');
log.info('Starting tests');
return await runTests(lifecycle, log, mocha);
});
}
async getTestStats() {
return await this._run(async (config, coreProviders) => {
// replace the function of a provider so that it returns a promise-like object which
// never "resolves", essentially disabling all the services while still allowing us
// to load the test files and populate the mocha suites
const stubProvider = provider => ({
...provider,
fn: () => ({
then: () => {}
})
});
const providers = new ProviderCollection(log, [
...coreProviders,
...readProviderSpec('Service', config.get('services')),
...readProviderSpec('PageObject', config.get('pageObjects'))
].map(stubProvider));
const mocha = await setupMocha(lifecycle, log, config, providers);
const countTests = suite => (
suite.suites.reduce(
(sum, suite) => sum + countTests(suite),
suite.tests.length
)
);
return {
tests: countTests(mocha.suite),
excludedTests: mocha.excludedTests.length
};
});
}
async _run(handler) {
let runErrorOccurred = false;
try {
@ -49,14 +103,14 @@ export function createFunctionalTestRunner({ log, configFile, configOverrides })
return;
}
const providers = createProviderCollection(lifecycle, log, config);
await providers.loadAll();
const mocha = await setupMocha(lifecycle, log, config, providers);
await lifecycle.trigger('beforeTests');
log.info('Starting tests');
return await runTests(lifecycle, log, mocha);
// base level services that functional_test_runner exposes
const coreProviders = readProviderSpec('Service', {
lifecycle: () => lifecycle,
log: () => log,
config: () => config,
});
return await handler(config, coreProviders);
} catch (runError) {
runErrorOccurred = true;
throw runError;

50
src/functional_test_runner/lib/create_provider_collection.js
View file

@ -1,50 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import {
ProviderCollection,
readProviderSpec
} from './providers';
/**
* Create a ProviderCollection that includes the Service
* providers and PageObject providers from config, as well
* providers for the default services, lifecycle, log, and
* config
*
* @param {Lifecycle} lifecycle
* @param {ToolingLog} log
* @param {Config} config [description]
* @return {ProviderCollection}
*/
export function createProviderCollection(lifecycle, log, config) {
return new ProviderCollection(log, [
...readProviderSpec('Service', {
// base level services that functional_test_runner exposes
lifecycle: () => lifecycle,
log: () => log,
config: () => config,
...config.get('services'),
}),
...readProviderSpec('PageObject', {
...config.get('pageObjects')
})
]);
}

2
src/functional_test_runner/lib/index.js
View file

@ -19,5 +19,5 @@
export { createLifecycle } from './lifecycle';
export { readConfigFile } from './config';
export { createProviderCollection } from './create_provider_collection';
export { setupMocha, runTests } from './mocha';
export { readProviderSpec, ProviderCollection } from './providers';

14
src/functional_test_runner/lib/mocha/filter_suites_by_tags.js
View file

@ -28,6 +28,14 @@
* @param options.exclude an array of tags that will be used to exclude suites from the run
*/
export function filterSuitesByTags({ log, mocha, include, exclude }) {
mocha.excludedTests = [];
// collect all the tests from some suite, including it's children
const collectTests = (suite) =>
suite.suites.reduce(
(acc, s) => acc.concat(collectTests(s)),
suite.tests
);
// if include tags were provided, filter the tree once to
// only include branches that are included at some point
if (include.length) {
@ -47,14 +55,18 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
continue;
}
// this suite has an included child but is not included
// itself, so strip out its tests and recurse to filter
// out child suites which are not included
if (isChildIncluded(child)) {
mocha.excludedTests = mocha.excludedTests.concat(child.tests);
child.tests = [];
parentSuite.suites.push(child);
recurse(child);
continue;
} else {
mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
}
}
}(mocha.suite));
@ -78,6 +90,8 @@ export function filterSuitesByTags({ log, mocha, include, exclude }) {
if (isNotExcluded(child)) {
parentSuite.suites.push(child);
recurse(child);
} else {
mocha.excludedTests = mocha.excludedTests.concat(collectTests(child));
}
}
}(mocha.suite));