[kbn/test/ftr] add support for custom runners (#58610)
* [kbn/test/ftr] add support for custom runners * remove irrelevant tests
This commit is contained in:
parent
71982069ee
commit
4122705bc2
|
@ -60,6 +60,14 @@ export class FunctionalTestRunner {
|
|||
|
||||
await providers.loadAll();
|
||||
|
||||
const customTestRunner = config.get('testRunner');
|
||||
if (customTestRunner) {
|
||||
this.log.warning(
|
||||
'custom test runner defined, ignoring all mocha/suite/filtering related options'
|
||||
);
|
||||
return (await providers.invokeProviderFn(customTestRunner)) || 0;
|
||||
}
|
||||
|
||||
const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
|
||||
await this.lifecycle.beforeTests.trigger();
|
||||
this.log.info('Starting tests');
|
||||
|
@ -70,6 +78,10 @@ export class FunctionalTestRunner {
|
|||
|
||||
async getTestStats() {
|
||||
return await this._run(async (config, coreProviders) => {
|
||||
if (config.get('testRunner')) {
|
||||
throw new Error('Unable to get test stats for config that uses a custom test runner');
|
||||
}
|
||||
|
||||
// replace the function of custom service providers so that they return
|
||||
// promise-like objects which never resolve, essentially disabling them
|
||||
// allowing us to load the test files and populate the mocha suites
|
||||
|
@ -108,8 +120,11 @@ export class FunctionalTestRunner {
|
|||
const config = await readConfigFile(this.log, this.configFile, this.configOverrides);
|
||||
this.log.info('Config loaded');
|
||||
|
||||
if (config.get('testFiles').length === 0) {
|
||||
throw new Error('No test files defined.');
|
||||
if (
|
||||
(!config.get('testFiles') || config.get('testFiles').length === 0) &&
|
||||
!config.get('testRunner')
|
||||
) {
|
||||
throw new Error('No tests defined.');
|
||||
}
|
||||
|
||||
// base level services that functional_test_runner exposes
|
||||
|
|
|
@ -55,18 +55,4 @@ describe('readConfigFile()', () => {
|
|||
expect(err.message).to.match(/"foo"/);
|
||||
}
|
||||
});
|
||||
|
||||
it('throws if config does not define testFiles', async () => {
|
||||
try {
|
||||
await readConfigFile(log, require.resolve('./fixtures/config.4'));
|
||||
throw new Error('expected readConfigFile() to fail');
|
||||
} catch (err) {
|
||||
expect(err.message).to.match(/"testFiles"/);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not throw if child config file does not have any testFiles', async () => {
|
||||
const config = await readConfigFile(log, require.resolve('./fixtures/config.3'));
|
||||
expect(config.get('screenshots.directory')).to.be('bar');
|
||||
});
|
||||
});
|
||||
|
|
|
@ -61,13 +61,8 @@ const defaultRelativeToConfigPath = (path: string) => {
|
|||
|
||||
export const schema = Joi.object()
|
||||
.keys({
|
||||
testFiles: Joi.array()
|
||||
.items(Joi.string())
|
||||
.when('$primary', {
|
||||
is: true,
|
||||
then: Joi.required(),
|
||||
otherwise: Joi.any().default([]),
|
||||
}),
|
||||
testFiles: Joi.array().items(Joi.string()),
|
||||
testRunner: Joi.func(),
|
||||
|
||||
excludeTestFiles: Joi.array()
|
||||
.items(Joi.string())
|
||||
|
|
|
@ -68,6 +68,15 @@ export class ProviderCollection {
|
|||
}
|
||||
}
|
||||
|
||||
public invokeProviderFn(provider: (args: any) => any) {
|
||||
return provider({
|
||||
getService: this.getService,
|
||||
hasService: this.hasService,
|
||||
getPageObject: this.getPageObject,
|
||||
getPageObjects: this.getPageObjects,
|
||||
});
|
||||
}
|
||||
|
||||
private findProvider(type: string, name: string) {
|
||||
return this.providers.find(p => p.type === type && p.name === name);
|
||||
}
|
||||
|
@ -89,12 +98,7 @@ export class ProviderCollection {
|
|||
}
|
||||
|
||||
if (!instances.has(provider)) {
|
||||
let instance = provider({
|
||||
getService: this.getService,
|
||||
hasService: this.hasService,
|
||||
getPageObject: this.getPageObject,
|
||||
getPageObjects: this.getPageObjects,
|
||||
});
|
||||
let instance = this.invokeProviderFn(provider);
|
||||
|
||||
if (instance && typeof instance.then === 'function') {
|
||||
instance = createAsyncInstance(type, name, instance);
|
||||
|
|
|
@ -18,4 +18,6 @@
|
|||
*/
|
||||
|
||||
export { runTestsCli } from './run_tests/cli';
|
||||
export { processOptions as processRunTestsCliOptions } from './run_tests/args';
|
||||
export { startServersCli } from './start_servers/cli';
|
||||
export { processOptions as processStartServersCliOptions } from './start_servers/args';
|
||||
|
|
|
@ -26,24 +26,32 @@ async function createFtr({
|
|||
}) {
|
||||
const config = await readConfigFile(log, configPath);
|
||||
|
||||
return new FunctionalTestRunner(log, configPath, {
|
||||
mochaOpts: {
|
||||
bail: !!bail,
|
||||
grep,
|
||||
},
|
||||
kbnTestServer: {
|
||||
installDir,
|
||||
},
|
||||
updateBaselines,
|
||||
suiteTags: {
|
||||
include: [...suiteTags.include, ...config.get('suiteTags.include')],
|
||||
exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],
|
||||
},
|
||||
});
|
||||
return {
|
||||
config,
|
||||
ftr: new FunctionalTestRunner(log, configPath, {
|
||||
mochaOpts: {
|
||||
bail: !!bail,
|
||||
grep,
|
||||
},
|
||||
kbnTestServer: {
|
||||
installDir,
|
||||
},
|
||||
updateBaselines,
|
||||
suiteTags: {
|
||||
include: [...suiteTags.include, ...config.get('suiteTags.include')],
|
||||
exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function assertNoneExcluded({ configPath, options }) {
|
||||
const ftr = await createFtr({ configPath, options });
|
||||
const { config, ftr } = await createFtr({ configPath, options });
|
||||
|
||||
if (config.get('testRunner')) {
|
||||
// tests with custom test runners are not included in this check
|
||||
return;
|
||||
}
|
||||
|
||||
const stats = await ftr.getTestStats();
|
||||
if (stats.excludedTests.length > 0) {
|
||||
|
@ -60,7 +68,7 @@ export async function assertNoneExcluded({ configPath, options }) {
|
|||
}
|
||||
|
||||
export async function runFtr({ configPath, options }) {
|
||||
const ftr = await createFtr({ configPath, options });
|
||||
const { ftr } = await createFtr({ configPath, options });
|
||||
|
||||
const failureCount = await ftr.run();
|
||||
if (failureCount > 0) {
|
||||
|
@ -71,7 +79,12 @@ export async function runFtr({ configPath, options }) {
|
|||
}
|
||||
|
||||
export async function hasTests({ configPath, options }) {
|
||||
const ftr = await createFtr({ configPath, options });
|
||||
const { ftr, config } = await createFtr({ configPath, options });
|
||||
|
||||
if (config.get('testRunner')) {
|
||||
// configs with custom test runners are assumed to always have tests
|
||||
return true;
|
||||
}
|
||||
const stats = await ftr.getTestStats();
|
||||
return stats.testCount > 0;
|
||||
}
|
||||
|
|
|
@ -17,8 +17,15 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
// @ts-ignore not typed yet
|
||||
export { runTestsCli, startServersCli } from './functional_tests/cli';
|
||||
import {
|
||||
runTestsCli,
|
||||
processRunTestsCliOptions,
|
||||
startServersCli,
|
||||
processStartServersCliOptions,
|
||||
// @ts-ignore not typed yet
|
||||
} from './functional_tests/cli';
|
||||
|
||||
export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions };
|
||||
|
||||
// @ts-ignore not typed yet
|
||||
export { runTests, startServers } from './functional_tests/tasks';
|
||||
|
|
Loading…
Reference in a new issue