[ftr] allow filtering suites by tag (#25021)

Closes #22840

In the functional tests we want a better way to include/exclude certain tests, especially as we move forward with #22359. This PR allows us to decorate suite objects with "tags", which won't clutter up the test names and can be used to filter out specific tests within a single test config. The functional test runner supports defining `--include-tag` and `--exclude-tag` CLI arguments, and multiple can be defined.

The method of filtering out tests for running against cloud has been updated to use this approach and I plan to do the same to #22359 once this is merged.

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -16,6 +16,8 @@ Options:
   --bail                         Stop the test run at the first failure. 
   --grep <pattern>               Pattern to select which tests to run. 
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
+  --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 
@@ -29,6 +31,10 @@ Object {
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
   "updateBaselines": true,
 }
 `;
@@ -41,6 +47,10 @@ Object {
   "createLogger": [Function],
   "debug": true,
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -52,6 +62,10 @@ Object {
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -67,6 +81,10 @@ Object {
   "extraKbnOpts": Object {
     "server.foo": "bar",
   },
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -78,6 +96,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "quiet": true,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -89,6 +111,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "silent": true,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -100,6 +126,10 @@ Object {
   "createLogger": [Function],
   "esFrom": "source",
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -111,6 +141,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "installDir": "foo",
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -122,6 +156,10 @@ Object {
   "createLogger": [Function],
   "extraKbnOpts": undefined,
   "grep": "management",
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
 }
 `;
 
@@ -132,6 +170,10 @@ Object {
   ],
   "createLogger": [Function],
   "extraKbnOpts": undefined,
+  "suiteTags": Object {
+    "exclude": Array [],
+    "include": Array [],
+  },
   "verbose": true,
 }
 `;

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -16,6 +16,8 @@ Options:
   --bail                         Stop the test run at the first failure. 
   --grep <pattern>               Pattern to select which tests to run. 
   --updateBaselines              Replace baseline screenshots with whatever is generated from the test. 
+  --include-tag <tag>            Tags that suites must include to be run, can be included multiple times. 
+  --exclude-tag <tag>            Tags that suites must NOT include to be run, can be included multiple times. 
   --verbose                      Log everything. 
   --debug                        Run in debug mode. 
   --quiet                        Only log errors. 

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -44,6 +44,14 @@ const options = {
   updateBaselines: {
     desc: 'Replace baseline screenshots with whatever is generated from the test.',
   },
+  'include-tag': {
+    arg: '<tag>',
+    desc: 'Tags that suites must include to be run, can be included multiple times.',
+  },
+  'exclude-tag': {
+    arg: '<tag>',
+    desc: 'Tags that suites must NOT include to be run, can be included multiple times.',
+  },
   verbose: { desc: 'Log everything.' },
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
@@ -98,6 +106,13 @@ export function processOptions(userOptions, defaultConfigPaths) {
     delete userOptions['kibana-install-dir'];
   }
 
+  userOptions.suiteTags = {
+    include: [].concat(userOptions['include-tag'] || []),
+    exclude: [].concat(userOptions['exclude-tag'] || []),
+  };
+  delete userOptions['include-tag'];
+  delete userOptions['exclude-tag'];
+
   function createLogger() {
     return new ToolingLog({
       level: pickLevelFromFlags(userOptions),
@@ -115,7 +130,9 @@ export function processOptions(userOptions, defaultConfigPaths) {
 
 function validateOptions(userOptions) {
   Object.entries(userOptions).forEach(([key, val]) => {
-    if (key === '_') return;
+    if (key === '_' || key === 'suiteTags') {
+      return;
+    }
 
     // Validate flags passed
     if (options[key] === undefined) {

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -20,7 +20,10 @@
 import { createFunctionalTestRunner } from '../../../../../src/functional_test_runner';
 import { CliError } from './run_cli';
 
-export async function runFtr({ configPath, options: { log, bail, grep, updateBaselines } }) {
+export async function runFtr({
+  configPath,
+  options: { log, bail, grep, updateBaselines, suiteTags },
+}) {
   const ftr = createFunctionalTestRunner({
     log,
     configFile: configPath,
@@ -30,6 +33,7 @@ export async function runFtr({ configPath, options: { log, bail, grep, updateBas
         grep,
       },
       updateBaselines,
+      suiteTags,
     },
   });
 

src/dev/jest/config.js

@@ -27,6 +27,7 @@ export default {
     '<rootDir>/src/cli',
     '<rootDir>/src/cli_keystore',
     '<rootDir>/src/cli_plugin',
+    '<rootDir>/src/functional_test_runner',
     '<rootDir>/src/dev',
     '<rootDir>/src/utils',
     '<rootDir>/src/setup_node_env',

src/functional_test_runner/cli.js

@@ -28,20 +28,26 @@ const cmd = new Command('node scripts/functional_test_runner');
 const resolveConfigPath = v => resolve(process.cwd(), v);
 const defaultConfigPath = resolveConfigPath('test/functional/config.js');
 
-const collectExcludePaths = () => {
+const createMultiArgCollector = (map) => () => {
   const paths = [];
   return (arg) => {
-    paths.push(resolve(arg));
+    paths.push(map ? map(arg) : arg);
     return paths;
   };
 };
 
+const collectExcludePaths = createMultiArgCollector(a => resolve(a));
+const collectIncludeTags = createMultiArgCollector();
+const collectExcludeTags = createMultiArgCollector();
+
 cmd
   .option('--config [path]', 'Path to a config file', resolveConfigPath, defaultConfigPath)
   .option('--bail', 'stop tests after the first failure', false)
   .option('--grep <pattern>', 'pattern used to select which tests to run')
   .option('--invert', 'invert grep to exclude tests', false)
   .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), [])
+  .option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), [])
+  .option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), [])
   .option('--verbose', 'Log everything', false)
   .option('--quiet', 'Only log errors', false)
   .option('--silent', 'Log nothing', false)
@@ -69,6 +75,10 @@ const functionalTestRunner = createFunctionalTestRunner({
       grep: cmd.grep,
       invert: cmd.invert,
     },
+    suiteTags: {
+      include: cmd.includeTag,
+      exclude: cmd.excludeTag,
+    },
     updateBaselines: cmd.updateBaselines,
     excludeTestFiles: cmd.exclude
   }

src/functional_test_runner/lib/config/schema.js

@@ -62,6 +62,11 @@ export const schema = Joi.object().keys({
 
   excludeTestFiles: Joi.array().items(Joi.string()).default([]),
 
+  suiteTags: Joi.object().keys({
+    include: Joi.array().items(Joi.string()).default([]),
+    exclude: Joi.array().items(Joi.string()).default([]),
+  }).default(),
+
   services: Joi.object().pattern(
     ID_PATTERN,
     Joi.func().required()

src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -61,6 +61,10 @@ export function decorateMochaUi(lifecycle, context) {
             await lifecycle.trigger('beforeTestSuite', this);
           });
 
+          this.tags = (tags) => {
+            this._tags = [].concat(this._tags || [], tags);
+          };
+
           provider.call(this);
 
           after(async () => {

src/functional_test_runner/lib/mocha/filter_suites_by_tags.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Given a mocha instance that has already loaded all of its suites, filter out
+ * the suites based on the include/exclude tags. If there are include tags then
+ * only suites which include the tag will be run, and if there are exclude tags
+ * then any suite with that tag will not be run.
+ *
+ * @param options.mocha instance of mocha that we are going to be running
+ * @param options.include an array of tags that suites must be tagged with to be run
+ * @param options.exclude an array of tags that will be used to exclude suites from the run
+ */
+export function filterSuitesByTags({ log, mocha, include, exclude }) {
+  // if include tags were provided, filter the tree once to
+  // only include branches that are included at some point
+  if (include.length) {
+    log.info('Only running suites (and their sub-suites) if they include the tag(s):', include);
+
+    const isIncluded = suite => !suite._tags ? false : suite._tags.some(t => include.includes(t));
+    const isChildIncluded = suite => suite.suites.some(s => isIncluded(s) || isChildIncluded(s));
+
+    (function recurse(parentSuite) {
+      const children = parentSuite.suites;
+      parentSuite.suites = [];
+
+      for (const child of children) {
+        // this suite is explicitly included
+        if (isIncluded(child)) {
+          parentSuite.suites.push(child);
+          continue;
+        }
+
+        // this suite has an included child but is not included
+        // itself, so strip out its tests and recurse to filter
+        // out child suites which are not included
+        if (isChildIncluded(child)) {
+          child.tests = [];
+          parentSuite.suites.push(child);
+          recurse(child);
+          continue;
+        }
+      }
+    }(mocha.suite));
+  }
+
+
+  // if exclude tags were provided, filter the possibly already
+  // filtered tree to remove branches that are excluded
+  if (exclude.length) {
+    log.info('Filtering out any suites that include the tag(s):', exclude);
+
+    const isNotExcluded = suite => !suite._tags || !suite._tags.some(t => exclude.includes(t));
+
+    (function recurse(parentSuite) {
+      const children = parentSuite.suites;
+      parentSuite.suites = [];
+
+      for (const child of children) {
+        // keep suites that are not explicitly excluded but
+        // recurse to remove excluded children
+        if (isNotExcluded(child)) {
+          parentSuite.suites.push(child);
+          recurse(child);
+        }
+      }
+    }(mocha.suite));
+  }
+}

src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js

@@ -0,0 +1,221 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { format } from 'util';
+
+import Mocha from 'mocha';
+import { create as createSuite } from 'mocha/lib/suite';
+import Test from 'mocha/lib/test';
+
+import { filterSuitesByTags } from './filter_suites_by_tags';
+
+function setup({ include, exclude }) {
+  return new Promise(resolve => {
+    const history = [];
+
+    const mocha = new Mocha({
+      reporter: class {
+        constructor(runner) {
+          runner.on('hook', hook => {
+            history.push(`hook:  ${hook.fullTitle()}`);
+          });
+
+          runner.on('pass', test => {
+            history.push(`test:  ${test.fullTitle()}`);
+          });
+
+          runner.on('suite', suite => {
+            history.push(`suite: ${suite.fullTitle()}`);
+          });
+        }
+
+        done() {
+          resolve({
+            history,
+            mocha,
+          });
+        }
+      },
+    });
+
+    mocha.suite.beforeEach(function rootBeforeEach() {});
+
+    const level1 = createSuite(mocha.suite, 'level 1');
+    level1.beforeEach(function level1BeforeEach() {});
+    level1._tags = ['level1'];
+
+    const level1a = createSuite(level1, 'level 1a');
+    level1a._tags = ['level1a'];
+    level1a.addTest(new Test('test 1a', () => {}));
+
+    const level1b = createSuite(level1, 'level 1b');
+    level1b._tags = ['level1b'];
+    level1b.addTest(new Test('test 1b', () => {}));
+
+    const level2 = createSuite(mocha.suite, 'level 2');
+    const level2a = createSuite(level2, 'level 2a');
+    level2a._tags = ['level2a'];
+    level2a.addTest(new Test('test 2a', () => {}));
+
+    filterSuitesByTags({
+      log: {
+        info(...args) {
+          history.push(`info: ${format(...args)}`);
+        },
+      },
+      mocha,
+      include,
+      exclude,
+    });
+
+    mocha.run();
+  });
+}
+
+it('only runs hooks of parents and tests in level1a', async () => {
+  const { history } = await setup({
+    include: ['level1a'],
+    exclude: [],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1a' ]",
+  "suite: ",
+  "suite: level 1",
+  "suite: level 1 level 1a",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "hook:  level 1 \\"before each\\" hook: level1BeforeEach",
+  "test:  level 1 level 1a test 1a",
+]
+`);
+});
+
+it('only runs hooks of parents and tests in level1b', async () => {
+  const { history } = await setup({
+    include: ['level1b'],
+    exclude: [],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1b' ]",
+  "suite: ",
+  "suite: level 1",
+  "suite: level 1 level 1b",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "hook:  level 1 \\"before each\\" hook: level1BeforeEach",
+  "test:  level 1 level 1b test 1b",
+]
+`);
+});
+
+it('only runs hooks of parents and tests in level1a and level1b', async () => {
+  const { history } = await setup({
+    include: ['level1a', 'level1b'],
+    exclude: [],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1a', 'level1b' ]",
+  "suite: ",
+  "suite: level 1",
+  "suite: level 1 level 1a",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "hook:  level 1 \\"before each\\" hook: level1BeforeEach",
+  "test:  level 1 level 1a test 1a",
+  "suite: level 1 level 1b",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "hook:  level 1 \\"before each\\" hook: level1BeforeEach",
+  "test:  level 1 level 1b test 1b",
+]
+`);
+});
+
+it('only runs level1a if including level1 and excluding level1b', async () => {
+  const { history } = await setup({
+    include: ['level1'],
+    exclude: ['level1b'],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1' ]",
+  "info: Filtering out any suites that include the tag(s): [ 'level1b' ]",
+  "suite: ",
+  "suite: level 1",
+  "suite: level 1 level 1a",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "hook:  level 1 \\"before each\\" hook: level1BeforeEach",
+  "test:  level 1 level 1a test 1a",
+]
+`);
+});
+
+it('only runs level1b if including level1 and excluding level1a', async () => {
+  const { history } = await setup({
+    include: ['level1'],
+    exclude: ['level1a'],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1' ]",
+  "info: Filtering out any suites that include the tag(s): [ 'level1a' ]",
+  "suite: ",
+  "suite: level 1",
+  "suite: level 1 level 1b",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "hook:  level 1 \\"before each\\" hook: level1BeforeEach",
+  "test:  level 1 level 1b test 1b",
+]
+`);
+});
+
+it('only runs level2 if excluding level1', async () => {
+  const { history } = await setup({
+    include: [],
+    exclude: ['level1'],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Filtering out any suites that include the tag(s): [ 'level1' ]",
+  "suite: ",
+  "suite: level 2",
+  "suite: level 2 level 2a",
+  "hook:   \\"before each\\" hook: rootBeforeEach",
+  "test:  level 2 level 2a test 2a",
+]
+`);
+});
+
+it('does nothing if everything excluded', async () => {
+  const { history } = await setup({
+    include: [],
+    exclude: ['level1', 'level2a'],
+  });
+
+  expect(history).toMatchInlineSnapshot(`
+Array [
+  "info: Filtering out any suites that include the tag(s): [ 'level1', 'level2a' ]",
+]
+`);
+});

src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -20,6 +20,7 @@
 import Mocha from 'mocha';
 
 import { loadTestFiles } from './load_test_files';
+import { filterSuitesByTags } from './filter_suites_by_tags';
 import { MochaReporterProvider } from './reporter';
 
 /**
@@ -55,5 +56,13 @@ export async function setupMocha(lifecycle, log, config, providers) {
     excludePaths: config.get('excludeTestFiles'),
     updateBaselines: config.get('updateBaselines'),
   });
+
+  filterSuitesByTags({
+    log,
+    mocha,
+    include: config.get('suiteTags.include'),
+    exclude: config.get('suiteTags.exclude'),
+  });
+
   return mocha;
 }

test/functional/apps/management/_test_huge_fields.js

@@ -24,7 +24,9 @@ export default function ({ getService, getPageObjects }) {
   const log = getService('log');
   const PageObjects = getPageObjects(['common', 'home', 'settings']);
 
-  describe('test large number of fields @skipcloud', function () {
+  describe('test large number of fields', function () {
+    this.tags(['skipCloud']);
+
     const EXPECTED_FIELD_COUNT = '10006';
     before(async function () {
       await esArchiver.loadIfNeeded('large_fields');

test/scripts/jenkins_cloud.sh

@@ -22,4 +22,4 @@ set -e
 
 source "$(dirname $0)/../../src/dev/ci_setup/setup.sh"
 
-xvfb-run node scripts/functional_test_runner --debug --grep @skipcloud --invert $@
+xvfb-run node scripts/functional_test_runner --debug --exclude-tag skipCloud $@