diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap index 04122fbf55f9..93d129213ff0 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap @@ -16,6 +16,8 @@ Options: --bail Stop the test run at the first failure. --grep Pattern to select which tests to run. --updateBaselines Replace baseline screenshots with whatever is generated from the test. + --include-tag Tags that suites must include to be run, can be included multiple times. + --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. --verbose Log everything. --debug Run in debug mode. --quiet Only log errors. @@ -29,6 +31,10 @@ Object { ], "createLogger": [Function], "extraKbnOpts": undefined, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, "updateBaselines": true, } `; @@ -41,6 +47,10 @@ Object { "createLogger": [Function], "debug": true, "extraKbnOpts": undefined, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -52,6 +62,10 @@ Object { ], "createLogger": [Function], "extraKbnOpts": undefined, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -67,6 +81,10 @@ Object { "extraKbnOpts": Object { "server.foo": "bar", }, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -78,6 +96,10 @@ Object { "createLogger": [Function], "extraKbnOpts": undefined, "quiet": true, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -89,6 +111,10 @@ Object { "createLogger": [Function], "extraKbnOpts": undefined, "silent": true, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -100,6 +126,10 @@ Object { "createLogger": [Function], "esFrom": "source", "extraKbnOpts": undefined, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -111,6 +141,10 @@ Object { "createLogger": [Function], "extraKbnOpts": undefined, "installDir": "foo", + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -122,6 +156,10 @@ Object { "createLogger": [Function], "extraKbnOpts": undefined, "grep": "management", + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, } `; @@ -132,6 +170,10 @@ Object { ], "createLogger": [Function], "extraKbnOpts": undefined, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, "verbose": true, } `; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap index 854415c734e0..6d4b35f754b4 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap @@ -16,6 +16,8 @@ Options: --bail Stop the test run at the first failure. --grep Pattern to select which tests to run. --updateBaselines Replace baseline screenshots with whatever is generated from the test. + --include-tag Tags that suites must include to be run, can be included multiple times. + --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. --verbose Log everything. --debug Run in debug mode. --quiet Only log errors. diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js index c03649a1167e..6aa1e1aa5c31 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js @@ -44,6 +44,14 @@ const options = { updateBaselines: { desc: 'Replace baseline screenshots with whatever is generated from the test.', }, + 'include-tag': { + arg: '', + desc: 'Tags that suites must include to be run, can be included multiple times.', + }, + 'exclude-tag': { + arg: '', + desc: 'Tags that suites must NOT include to be run, can be included multiple times.', + }, verbose: { desc: 'Log everything.' }, debug: { desc: 'Run in debug mode.' }, quiet: { desc: 'Only log errors.' }, @@ -98,6 +106,13 @@ export function processOptions(userOptions, defaultConfigPaths) { delete userOptions['kibana-install-dir']; } + userOptions.suiteTags = { + include: [].concat(userOptions['include-tag'] || []), + exclude: [].concat(userOptions['exclude-tag'] || []), + }; + delete userOptions['include-tag']; + delete userOptions['exclude-tag']; + function createLogger() { return new ToolingLog({ level: pickLevelFromFlags(userOptions), @@ -115,7 +130,9 @@ export function processOptions(userOptions, defaultConfigPaths) { function validateOptions(userOptions) { Object.entries(userOptions).forEach(([key, val]) => { - if (key === '_') return; + if (key === '_' || key === 'suiteTags') { + return; + } // Validate flags passed if (options[key] === undefined) { diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js index 6fe02ec39d9a..779286212f5c 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -20,7 +20,10 @@ import { createFunctionalTestRunner } from '../../../../../src/functional_test_runner'; import { CliError } from './run_cli'; -export async function runFtr({ configPath, options: { log, bail, grep, updateBaselines } }) { +export async function runFtr({ + configPath, + options: { log, bail, grep, updateBaselines, suiteTags }, +}) { const ftr = createFunctionalTestRunner({ log, configFile: configPath, @@ -30,6 +33,7 @@ export async function runFtr({ configPath, options: { log, bail, grep, updateBas grep, }, updateBaselines, + suiteTags, }, }); diff --git a/src/dev/jest/config.js b/src/dev/jest/config.js index 99b22b14c2d7..f171655c9c36 100644 --- a/src/dev/jest/config.js +++ b/src/dev/jest/config.js @@ -27,6 +27,7 @@ export default { '/src/cli', '/src/cli_keystore', '/src/cli_plugin', + '/src/functional_test_runner', '/src/dev', '/src/utils', '/src/setup_node_env', diff --git a/src/functional_test_runner/cli.js b/src/functional_test_runner/cli.js index 10e096da777c..6998a1b8bfb7 100644 --- a/src/functional_test_runner/cli.js +++ b/src/functional_test_runner/cli.js @@ -28,20 +28,26 @@ const cmd = new Command('node scripts/functional_test_runner'); const resolveConfigPath = v => resolve(process.cwd(), v); const defaultConfigPath = resolveConfigPath('test/functional/config.js'); -const collectExcludePaths = () => { +const createMultiArgCollector = (map) => () => { const paths = []; return (arg) => { - paths.push(resolve(arg)); + paths.push(map ? map(arg) : arg); return paths; }; }; +const collectExcludePaths = createMultiArgCollector(a => resolve(a)); +const collectIncludeTags = createMultiArgCollector(); +const collectExcludeTags = createMultiArgCollector(); + cmd .option('--config [path]', 'Path to a config file', resolveConfigPath, defaultConfigPath) .option('--bail', 'stop tests after the first failure', false) .option('--grep ', 'pattern used to select which tests to run') .option('--invert', 'invert grep to exclude tests', false) .option('--exclude [file]', 'Path to a test file that should not be loaded', collectExcludePaths(), []) + .option('--include-tag [tag]', 'A tag to be included, pass multiple times for multiple tags', collectIncludeTags(), []) + .option('--exclude-tag [tag]', 'A tag to be excluded, pass multiple times for multiple tags', collectExcludeTags(), []) .option('--verbose', 'Log everything', false) .option('--quiet', 'Only log errors', false) .option('--silent', 'Log nothing', false) @@ -69,6 +75,10 @@ const functionalTestRunner = createFunctionalTestRunner({ grep: cmd.grep, invert: cmd.invert, }, + suiteTags: { + include: cmd.includeTag, + exclude: cmd.excludeTag, + }, updateBaselines: cmd.updateBaselines, excludeTestFiles: cmd.exclude } diff --git a/src/functional_test_runner/lib/config/schema.js b/src/functional_test_runner/lib/config/schema.js index 4225895a605b..ae0c0087258d 100644 --- a/src/functional_test_runner/lib/config/schema.js +++ b/src/functional_test_runner/lib/config/schema.js @@ -62,6 +62,11 @@ export const schema = Joi.object().keys({ excludeTestFiles: Joi.array().items(Joi.string()).default([]), + suiteTags: Joi.object().keys({ + include: Joi.array().items(Joi.string()).default([]), + exclude: Joi.array().items(Joi.string()).default([]), + }).default(), + services: Joi.object().pattern( ID_PATTERN, Joi.func().required() diff --git a/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js b/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js index e438999962b6..a8272ae67717 100644 --- a/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js +++ b/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js @@ -61,6 +61,10 @@ export function decorateMochaUi(lifecycle, context) { await lifecycle.trigger('beforeTestSuite', this); }); + this.tags = (tags) => { + this._tags = [].concat(this._tags || [], tags); + }; + provider.call(this); after(async () => { diff --git a/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js new file mode 100644 index 000000000000..fd15f936c762 --- /dev/null +++ b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.js @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Given a mocha instance that has already loaded all of its suites, filter out + * the suites based on the include/exclude tags. If there are include tags then + * only suites which include the tag will be run, and if there are exclude tags + * then any suite with that tag will not be run. + * + * @param options.mocha instance of mocha that we are going to be running + * @param options.include an array of tags that suites must be tagged with to be run + * @param options.exclude an array of tags that will be used to exclude suites from the run + */ +export function filterSuitesByTags({ log, mocha, include, exclude }) { + // if include tags were provided, filter the tree once to + // only include branches that are included at some point + if (include.length) { + log.info('Only running suites (and their sub-suites) if they include the tag(s):', include); + + const isIncluded = suite => !suite._tags ? false : suite._tags.some(t => include.includes(t)); + const isChildIncluded = suite => suite.suites.some(s => isIncluded(s) || isChildIncluded(s)); + + (function recurse(parentSuite) { + const children = parentSuite.suites; + parentSuite.suites = []; + + for (const child of children) { + // this suite is explicitly included + if (isIncluded(child)) { + parentSuite.suites.push(child); + continue; + } + + // this suite has an included child but is not included + // itself, so strip out its tests and recurse to filter + // out child suites which are not included + if (isChildIncluded(child)) { + child.tests = []; + parentSuite.suites.push(child); + recurse(child); + continue; + } + } + }(mocha.suite)); + } + + + // if exclude tags were provided, filter the possibly already + // filtered tree to remove branches that are excluded + if (exclude.length) { + log.info('Filtering out any suites that include the tag(s):', exclude); + + const isNotExcluded = suite => !suite._tags || !suite._tags.some(t => exclude.includes(t)); + + (function recurse(parentSuite) { + const children = parentSuite.suites; + parentSuite.suites = []; + + for (const child of children) { + // keep suites that are not explicitly excluded but + // recurse to remove excluded children + if (isNotExcluded(child)) { + parentSuite.suites.push(child); + recurse(child); + } + } + }(mocha.suite)); + } +} diff --git a/src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js new file mode 100644 index 000000000000..fb1ca192c5fd --- /dev/null +++ b/src/functional_test_runner/lib/mocha/filter_suites_by_tags.test.js @@ -0,0 +1,221 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { format } from 'util'; + +import Mocha from 'mocha'; +import { create as createSuite } from 'mocha/lib/suite'; +import Test from 'mocha/lib/test'; + +import { filterSuitesByTags } from './filter_suites_by_tags'; + +function setup({ include, exclude }) { + return new Promise(resolve => { + const history = []; + + const mocha = new Mocha({ + reporter: class { + constructor(runner) { + runner.on('hook', hook => { + history.push(`hook: ${hook.fullTitle()}`); + }); + + runner.on('pass', test => { + history.push(`test: ${test.fullTitle()}`); + }); + + runner.on('suite', suite => { + history.push(`suite: ${suite.fullTitle()}`); + }); + } + + done() { + resolve({ + history, + mocha, + }); + } + }, + }); + + mocha.suite.beforeEach(function rootBeforeEach() {}); + + const level1 = createSuite(mocha.suite, 'level 1'); + level1.beforeEach(function level1BeforeEach() {}); + level1._tags = ['level1']; + + const level1a = createSuite(level1, 'level 1a'); + level1a._tags = ['level1a']; + level1a.addTest(new Test('test 1a', () => {})); + + const level1b = createSuite(level1, 'level 1b'); + level1b._tags = ['level1b']; + level1b.addTest(new Test('test 1b', () => {})); + + const level2 = createSuite(mocha.suite, 'level 2'); + const level2a = createSuite(level2, 'level 2a'); + level2a._tags = ['level2a']; + level2a.addTest(new Test('test 2a', () => {})); + + filterSuitesByTags({ + log: { + info(...args) { + history.push(`info: ${format(...args)}`); + }, + }, + mocha, + include, + exclude, + }); + + mocha.run(); + }); +} + +it('only runs hooks of parents and tests in level1a', async () => { + const { history } = await setup({ + include: ['level1a'], + exclude: [], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1a' ]", + "suite: ", + "suite: level 1", + "suite: level 1 level 1a", + "hook: \\"before each\\" hook: rootBeforeEach", + "hook: level 1 \\"before each\\" hook: level1BeforeEach", + "test: level 1 level 1a test 1a", +] +`); +}); + +it('only runs hooks of parents and tests in level1b', async () => { + const { history } = await setup({ + include: ['level1b'], + exclude: [], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1b' ]", + "suite: ", + "suite: level 1", + "suite: level 1 level 1b", + "hook: \\"before each\\" hook: rootBeforeEach", + "hook: level 1 \\"before each\\" hook: level1BeforeEach", + "test: level 1 level 1b test 1b", +] +`); +}); + +it('only runs hooks of parents and tests in level1a and level1b', async () => { + const { history } = await setup({ + include: ['level1a', 'level1b'], + exclude: [], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1a', 'level1b' ]", + "suite: ", + "suite: level 1", + "suite: level 1 level 1a", + "hook: \\"before each\\" hook: rootBeforeEach", + "hook: level 1 \\"before each\\" hook: level1BeforeEach", + "test: level 1 level 1a test 1a", + "suite: level 1 level 1b", + "hook: \\"before each\\" hook: rootBeforeEach", + "hook: level 1 \\"before each\\" hook: level1BeforeEach", + "test: level 1 level 1b test 1b", +] +`); +}); + +it('only runs level1a if including level1 and excluding level1b', async () => { + const { history } = await setup({ + include: ['level1'], + exclude: ['level1b'], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1' ]", + "info: Filtering out any suites that include the tag(s): [ 'level1b' ]", + "suite: ", + "suite: level 1", + "suite: level 1 level 1a", + "hook: \\"before each\\" hook: rootBeforeEach", + "hook: level 1 \\"before each\\" hook: level1BeforeEach", + "test: level 1 level 1a test 1a", +] +`); +}); + +it('only runs level1b if including level1 and excluding level1a', async () => { + const { history } = await setup({ + include: ['level1'], + exclude: ['level1a'], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Only running suites (and their sub-suites) if they include the tag(s): [ 'level1' ]", + "info: Filtering out any suites that include the tag(s): [ 'level1a' ]", + "suite: ", + "suite: level 1", + "suite: level 1 level 1b", + "hook: \\"before each\\" hook: rootBeforeEach", + "hook: level 1 \\"before each\\" hook: level1BeforeEach", + "test: level 1 level 1b test 1b", +] +`); +}); + +it('only runs level2 if excluding level1', async () => { + const { history } = await setup({ + include: [], + exclude: ['level1'], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Filtering out any suites that include the tag(s): [ 'level1' ]", + "suite: ", + "suite: level 2", + "suite: level 2 level 2a", + "hook: \\"before each\\" hook: rootBeforeEach", + "test: level 2 level 2a test 2a", +] +`); +}); + +it('does nothing if everything excluded', async () => { + const { history } = await setup({ + include: [], + exclude: ['level1', 'level2a'], + }); + + expect(history).toMatchInlineSnapshot(` +Array [ + "info: Filtering out any suites that include the tag(s): [ 'level1', 'level2a' ]", +] +`); +}); diff --git a/src/functional_test_runner/lib/mocha/setup_mocha.js b/src/functional_test_runner/lib/mocha/setup_mocha.js index 1b822d83a07e..a7a302f222a5 100644 --- a/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -20,6 +20,7 @@ import Mocha from 'mocha'; import { loadTestFiles } from './load_test_files'; +import { filterSuitesByTags } from './filter_suites_by_tags'; import { MochaReporterProvider } from './reporter'; /** @@ -55,5 +56,13 @@ export async function setupMocha(lifecycle, log, config, providers) { excludePaths: config.get('excludeTestFiles'), updateBaselines: config.get('updateBaselines'), }); + + filterSuitesByTags({ + log, + mocha, + include: config.get('suiteTags.include'), + exclude: config.get('suiteTags.exclude'), + }); + return mocha; } diff --git a/test/functional/apps/management/_test_huge_fields.js b/test/functional/apps/management/_test_huge_fields.js index 4b137def5b8e..7f39838971ab 100644 --- a/test/functional/apps/management/_test_huge_fields.js +++ b/test/functional/apps/management/_test_huge_fields.js @@ -24,7 +24,9 @@ export default function ({ getService, getPageObjects }) { const log = getService('log'); const PageObjects = getPageObjects(['common', 'home', 'settings']); - describe('test large number of fields @skipcloud', function () { + describe('test large number of fields', function () { + this.tags(['skipCloud']); + const EXPECTED_FIELD_COUNT = '10006'; before(async function () { await esArchiver.loadIfNeeded('large_fields'); diff --git a/test/scripts/jenkins_cloud.sh b/test/scripts/jenkins_cloud.sh index 52d3a021e412..413a19fb5c74 100755 --- a/test/scripts/jenkins_cloud.sh +++ b/test/scripts/jenkins_cloud.sh @@ -22,4 +22,4 @@ set -e source "$(dirname $0)/../../src/dev/ci_setup/setup.sh" -xvfb-run node scripts/functional_test_runner --debug --grep @skipcloud --invert $@ +xvfb-run node scripts/functional_test_runner --debug --exclude-tag skipCloud $@