[kbn/test/ftr] add support for custom runners (#58610)

* [kbn/test/ftr] add support for custom runners

* remove irrelevant tests

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -60,6 +60,14 @@ export class FunctionalTestRunner {
 
       await providers.loadAll();
 
+      const customTestRunner = config.get('testRunner');
+      if (customTestRunner) {
+        this.log.warning(
+          'custom test runner defined, ignoring all mocha/suite/filtering related options'
+        );
+        return (await providers.invokeProviderFn(customTestRunner)) || 0;
+      }
+
       const mocha = await setupMocha(this.lifecycle, this.log, config, providers);
       await this.lifecycle.beforeTests.trigger();
       this.log.info('Starting tests');
@@ -70,6 +78,10 @@ export class FunctionalTestRunner {
 
   async getTestStats() {
     return await this._run(async (config, coreProviders) => {
+      if (config.get('testRunner')) {
+        throw new Error('Unable to get test stats for config that uses a custom test runner');
+      }
+
       // replace the function of custom service providers so that they return
       // promise-like objects which never resolve, essentially disabling them
       // allowing us to load the test files and populate the mocha suites
@@ -108,8 +120,11 @@ export class FunctionalTestRunner {
       const config = await readConfigFile(this.log, this.configFile, this.configOverrides);
       this.log.info('Config loaded');
 
-      if (config.get('testFiles').length === 0) {
-        throw new Error('No test files defined.');
+      if (
+        (!config.get('testFiles') || config.get('testFiles').length === 0) &&
+        !config.get('testRunner')
+      ) {
+        throw new Error('No tests defined.');
       }
 
       // base level services that functional_test_runner exposes

packages/kbn-test/src/functional_test_runner/lib/config/__tests__/read_config_file.js

@@ -55,18 +55,4 @@ describe('readConfigFile()', () => {
       expect(err.message).to.match(/"foo"/);
     }
   });
-
-  it('throws if config does not define testFiles', async () => {
-    try {
-      await readConfigFile(log, require.resolve('./fixtures/config.4'));
-      throw new Error('expected readConfigFile() to fail');
-    } catch (err) {
-      expect(err.message).to.match(/"testFiles"/);
-    }
-  });
-
-  it('does not throw if child config file does not have any testFiles', async () => {
-    const config = await readConfigFile(log, require.resolve('./fixtures/config.3'));
-    expect(config.get('screenshots.directory')).to.be('bar');
-  });
 });

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -61,13 +61,8 @@ const defaultRelativeToConfigPath = (path: string) => {
 
 export const schema = Joi.object()
   .keys({
-    testFiles: Joi.array()
-      .items(Joi.string())
-      .when('$primary', {
-        is: true,
-        then: Joi.required(),
-        otherwise: Joi.any().default([]),
-      }),
+    testFiles: Joi.array().items(Joi.string()),
+    testRunner: Joi.func(),
 
     excludeTestFiles: Joi.array()
       .items(Joi.string())

packages/kbn-test/src/functional_test_runner/lib/providers/provider_collection.ts

@@ -68,6 +68,15 @@ export class ProviderCollection {
     }
   }
 
+  public invokeProviderFn(provider: (args: any) => any) {
+    return provider({
+      getService: this.getService,
+      hasService: this.hasService,
+      getPageObject: this.getPageObject,
+      getPageObjects: this.getPageObjects,
+    });
+  }
+
   private findProvider(type: string, name: string) {
     return this.providers.find(p => p.type === type && p.name === name);
   }
@@ -89,12 +98,7 @@ export class ProviderCollection {
       }
 
       if (!instances.has(provider)) {
-        let instance = provider({
-          getService: this.getService,
-          hasService: this.hasService,
-          getPageObject: this.getPageObject,
-          getPageObjects: this.getPageObjects,
-        });
+        let instance = this.invokeProviderFn(provider);
 
         if (instance && typeof instance.then === 'function') {
           instance = createAsyncInstance(type, name, instance);

packages/kbn-test/src/functional_tests/cli/index.js

@@ -18,4 +18,6 @@
  */
 
 export { runTestsCli } from './run_tests/cli';
+export { processOptions as processRunTestsCliOptions } from './run_tests/args';
 export { startServersCli } from './start_servers/cli';
+export { processOptions as processStartServersCliOptions } from './start_servers/args';

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -26,24 +26,32 @@ async function createFtr({
 }) {
   const config = await readConfigFile(log, configPath);
 
-  return new FunctionalTestRunner(log, configPath, {
-    mochaOpts: {
-      bail: !!bail,
-      grep,
-    },
-    kbnTestServer: {
-      installDir,
-    },
-    updateBaselines,
-    suiteTags: {
-      include: [...suiteTags.include, ...config.get('suiteTags.include')],
-      exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],
-    },
-  });
+  return {
+    config,
+    ftr: new FunctionalTestRunner(log, configPath, {
+      mochaOpts: {
+        bail: !!bail,
+        grep,
+      },
+      kbnTestServer: {
+        installDir,
+      },
+      updateBaselines,
+      suiteTags: {
+        include: [...suiteTags.include, ...config.get('suiteTags.include')],
+        exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],
+      },
+    }),
+  };
 }
 
 export async function assertNoneExcluded({ configPath, options }) {
-  const ftr = await createFtr({ configPath, options });
+  const { config, ftr } = await createFtr({ configPath, options });
+
+  if (config.get('testRunner')) {
+    // tests with custom test runners are not included in this check
+    return;
+  }
 
   const stats = await ftr.getTestStats();
   if (stats.excludedTests.length > 0) {
@@ -60,7 +68,7 @@ export async function assertNoneExcluded({ configPath, options }) {
 }
 
 export async function runFtr({ configPath, options }) {
-  const ftr = await createFtr({ configPath, options });
+  const { ftr } = await createFtr({ configPath, options });
 
   const failureCount = await ftr.run();
   if (failureCount > 0) {
@@ -71,7 +79,12 @@ export async function runFtr({ configPath, options }) {
 }
 
 export async function hasTests({ configPath, options }) {
-  const ftr = await createFtr({ configPath, options });
+  const { ftr, config } = await createFtr({ configPath, options });
+
+  if (config.get('testRunner')) {
+    // configs with custom test runners are assumed to always have tests
+    return true;
+  }
   const stats = await ftr.getTestStats();
   return stats.testCount > 0;
 }

packages/kbn-test/src/index.ts

@@ -17,8 +17,15 @@
  * under the License.
  */
 
-// @ts-ignore not typed yet
-export { runTestsCli, startServersCli } from './functional_tests/cli';
+import {
+  runTestsCli,
+  processRunTestsCliOptions,
+  startServersCli,
+  processStartServersCliOptions,
+  // @ts-ignore not typed yet
+} from './functional_tests/cli';
+
+export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions };
 
 // @ts-ignore not typed yet
 export { runTests, startServers } from './functional_tests/tasks';