[Functional Tests] Use @kbn/test on Kibana CI (#18967)
* Replace test:api with @kbn/test runTests
* Improve CLI help menu 🆘
* Use --es-from
* Replace jenkins:selenium with kbn-test
* Validate cli args, fixing test in the process
* Clean up some stuff
* Code review fixes
* Explanation for collectCliArgs
* Remove exit codes, they're useless anyway.
* Make markdown vis test pass with dev_mode setting
* Tests
* Remove unneeded export
* Code review: move console logging up to cli.js
* Code review: refactor startServers and runTests to take single options collection
* Code review: Remove all things I am sure we do not use
* Improve tests
* Code review fixes
* Pass created log to runFtr, runElasticsearch, runKibanaServer
* Update --es-from option to --esFrom
This commit is contained in:
parent
068c02f1c9
commit
b03243f5d8
|
@ -34,8 +34,7 @@ export function createEsTestCluster(options = {}) {
|
||||||
license = 'oss',
|
license = 'oss',
|
||||||
log,
|
log,
|
||||||
basePath = resolve(KIBANA_ROOT, '.es'),
|
basePath = resolve(KIBANA_ROOT, '.es'),
|
||||||
// Use source when running on CI
|
esFrom = esTestConfig.getBuildFrom(),
|
||||||
from = esTestConfig.getBuildFrom(),
|
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
const randomHash = Math.random()
|
const randomHash = Math.random()
|
||||||
|
@ -58,12 +57,12 @@ export function createEsTestCluster(options = {}) {
|
||||||
const second = 1000;
|
const second = 1000;
|
||||||
const minute = second * 60;
|
const minute = second * 60;
|
||||||
|
|
||||||
return from === 'snapshot' ? minute : minute * 6;
|
return esFrom === 'snapshot' ? minute : minute * 6;
|
||||||
}
|
}
|
||||||
|
|
||||||
async start(esArgs = []) {
|
async start(esArgs = []) {
|
||||||
const { installPath } =
|
const { installPath } =
|
||||||
from === 'source'
|
esFrom === 'source'
|
||||||
? await cluster.installSource(config)
|
? await cluster.installSource(config)
|
||||||
: await cluster.installSnapshot(config);
|
: await cluster.installSnapshot(config);
|
||||||
|
|
||||||
|
|
|
@ -17,5 +17,5 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export { runTestsCli } from './run_tests_cli';
|
export { runTestsCli } from './run_tests/cli';
|
||||||
export { startServersCli } from './start_servers_cli';
|
export { startServersCli } from './start_servers/cli';
|
||||||
|
|
|
@ -0,0 +1,137 @@
|
||||||
|
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||||
|
|
||||||
|
exports[`display help for run tests CLI displays as expected 1`] = `
|
||||||
|
"Run Functional Tests
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
node scripts/functional_tests --help
|
||||||
|
node scripts/functional_tests [--config <file1> [--config <file2> ...]]
|
||||||
|
node scripts/functional_tests [options] [-- --<other args>]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--help Display this menu and exit.
|
||||||
|
--config <file> Pass in a config. Can pass in multiple configs.
|
||||||
|
--esFrom <snapshot|source> Build Elasticsearch from source or run from snapshot. Default: snapshot
|
||||||
|
--kibana-install-dir <dir> Run Kibana from existing install directory instead of from source.
|
||||||
|
--bail Stop the test run at the first failure.
|
||||||
|
--grep <pattern> Pattern to select which tests to run.
|
||||||
|
--updateBaselines Replace baseline screenshots with whatever is generated from the test.
|
||||||
|
--verbose Log everything.
|
||||||
|
--debug Run in debug mode.
|
||||||
|
--quiet Only log errors.
|
||||||
|
--silent Log nothing."
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"updateBaselines": true,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts debug option 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"debug": true,
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": "",
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts extra server options 1`] = `
|
||||||
|
Object {
|
||||||
|
"_": Object {
|
||||||
|
"server.foo": "bar",
|
||||||
|
},
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": Object {
|
||||||
|
"server.foo": "bar",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts quiet option 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"quiet": true,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts silent option 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"silent": true,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"esFrom": "source",
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"kibana-install-dir": "foo",
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts value for grep 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"grep": "management",
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for run tests CLI accepts verbose option 1`] = `
|
||||||
|
Object {
|
||||||
|
"configs": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"verbose": true,
|
||||||
|
}
|
||||||
|
`;
|
|
@ -0,0 +1,75 @@
|
||||||
|
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||||
|
|
||||||
|
exports[`run tests CLI options accepts help option even if invalid options passed 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"Run Functional Tests
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
node scripts/functional_tests --help
|
||||||
|
node scripts/functional_tests [--config <file1> [--config <file2> ...]]
|
||||||
|
node scripts/functional_tests [options] [-- --<other args>]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--help Display this menu and exit.
|
||||||
|
--config <file> Pass in a config. Can pass in multiple configs.
|
||||||
|
--esFrom <snapshot|source> Build Elasticsearch from source or run from snapshot. Default: snapshot
|
||||||
|
--kibana-install-dir <dir> Run Kibana from existing install directory instead of from source.
|
||||||
|
--bail Stop the test run at the first failure.
|
||||||
|
--grep <pattern> Pattern to select which tests to run.
|
||||||
|
--updateBaselines Replace baseline screenshots with whatever is generated from the test.
|
||||||
|
--verbose Log everything.
|
||||||
|
--debug Run in debug mode.
|
||||||
|
--quiet Only log errors.
|
||||||
|
--silent Log nothing.",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`run tests CLI options rejects boolean config value 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests: invalid argument [true] to option [config][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`run tests CLI options rejects boolean value for kibana-install-dir 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests: invalid argument [true] to option [kibana-install-dir][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`run tests CLI options rejects empty config value if no default passed 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests: config is required[39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`run tests CLI options rejects invalid options even if valid options exist 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests: invalid option [aintnothang][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`run tests CLI options rejects non-boolean value for bail 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests: invalid argument [peanut] to option [bail][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`run tests CLI options rejects non-enum value for esFrom 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests: invalid argument [butter] to option [esFrom][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
133
packages/kbn-test/src/functional_tests/cli/run_tests/args.js
Normal file
133
packages/kbn-test/src/functional_tests/cli/run_tests/args.js
Normal file
|
@ -0,0 +1,133 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import dedent from 'dedent';
|
||||||
|
import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
|
||||||
|
|
||||||
|
const options = {
|
||||||
|
help: { desc: 'Display this menu and exit.' },
|
||||||
|
config: {
|
||||||
|
arg: '<file>',
|
||||||
|
desc: 'Pass in a config. Can pass in multiple configs.',
|
||||||
|
},
|
||||||
|
esFrom: {
|
||||||
|
arg: '<snapshot|source>',
|
||||||
|
choices: ['snapshot', 'source'],
|
||||||
|
desc: 'Build Elasticsearch from source or run from snapshot.',
|
||||||
|
default: 'snapshot',
|
||||||
|
},
|
||||||
|
'kibana-install-dir': {
|
||||||
|
arg: '<dir>',
|
||||||
|
desc: 'Run Kibana from existing install directory instead of from source.',
|
||||||
|
},
|
||||||
|
bail: { desc: 'Stop the test run at the first failure.' },
|
||||||
|
grep: {
|
||||||
|
arg: '<pattern>',
|
||||||
|
desc: 'Pattern to select which tests to run.',
|
||||||
|
},
|
||||||
|
updateBaselines: {
|
||||||
|
desc:
|
||||||
|
'Replace baseline screenshots with whatever is generated from the test.',
|
||||||
|
},
|
||||||
|
verbose: { desc: 'Log everything.' },
|
||||||
|
debug: { desc: 'Run in debug mode.' },
|
||||||
|
quiet: { desc: 'Only log errors.' },
|
||||||
|
silent: { desc: 'Log nothing.' },
|
||||||
|
};
|
||||||
|
|
||||||
|
export function displayHelp() {
|
||||||
|
const helpOptions = Object.keys(options)
|
||||||
|
.filter(name => name !== '_')
|
||||||
|
.map(name => {
|
||||||
|
const option = options[name];
|
||||||
|
return {
|
||||||
|
...option,
|
||||||
|
usage: `${name} ${option.arg || ''}`,
|
||||||
|
default: option.default ? `Default: ${option.default}` : '',
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.map(option => {
|
||||||
|
return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
|
||||||
|
})
|
||||||
|
.join(`\n `);
|
||||||
|
|
||||||
|
return dedent(`
|
||||||
|
Run Functional Tests
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
node scripts/functional_tests --help
|
||||||
|
node scripts/functional_tests [--config <file1> [--config <file2> ...]]
|
||||||
|
node scripts/functional_tests [options] [-- --<other args>]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
${helpOptions}
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function processOptions(userOptions, defaultConfigPaths) {
|
||||||
|
validateOptions(userOptions);
|
||||||
|
|
||||||
|
let configs;
|
||||||
|
if (userOptions.config) {
|
||||||
|
configs = [].concat(userOptions.config);
|
||||||
|
} else {
|
||||||
|
if (!defaultConfigPaths || defaultConfigPaths.length === 0) {
|
||||||
|
throw new Error(`functional_tests: config is required`);
|
||||||
|
} else {
|
||||||
|
configs = defaultConfigPaths;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createLogger() {
|
||||||
|
const log = createToolingLog(pickLevelFromFlags(userOptions));
|
||||||
|
log.pipe(process.stdout);
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
...userOptions,
|
||||||
|
configs,
|
||||||
|
createLogger,
|
||||||
|
extraKbnOpts: userOptions._,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateOptions(userOptions) {
|
||||||
|
Object.entries(userOptions).forEach(([key, val]) => {
|
||||||
|
if (key === '_') return;
|
||||||
|
|
||||||
|
// Validate flags passed
|
||||||
|
if (options[key] === undefined) {
|
||||||
|
throw new Error(`functional_tests: invalid option [${key}]`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
// Validate boolean flags
|
||||||
|
(!options[key].arg && typeof val !== 'boolean') ||
|
||||||
|
// Validate string/array flags
|
||||||
|
(options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
|
||||||
|
// Validate enum flags
|
||||||
|
(options[key].choices && !options[key].choices.includes(val))
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`functional_tests: invalid argument [${val}] to option [${key}]`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
|
@ -0,0 +1,118 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { displayHelp, processOptions } from './args';
|
||||||
|
|
||||||
|
describe('display help for run tests CLI', () => {
|
||||||
|
it('displays as expected', () => {
|
||||||
|
expect(displayHelp()).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('process options for run tests CLI', () => {
|
||||||
|
it('rejects boolean config value', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ config: true });
|
||||||
|
}).toThrow('functional_tests: invalid argument [true] to option [config]');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects empty config value if no default passed', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({});
|
||||||
|
}).toThrow('functional_tests: config is required');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts empty config value if default passed', () => {
|
||||||
|
const options = processOptions({ config: '' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-boolean value for bail', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ bail: 'peanut' }, ['foo']);
|
||||||
|
}).toThrow('functional_tests: invalid argument [peanut] to option [bail]');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts string value for kibana-install-dir', () => {
|
||||||
|
const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects boolean value for kibana-install-dir', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ 'kibana-install-dir': true }, ['foo']);
|
||||||
|
}).toThrow(
|
||||||
|
'functional_tests: invalid argument [true] to option [kibana-install-dir]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts boolean value for updateBaselines', () => {
|
||||||
|
const options = processOptions({ updateBaselines: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts source value for esFrom', () => {
|
||||||
|
const options = processOptions({ esFrom: 'source' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-enum value for esFrom', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ esFrom: 'butter' }, ['foo']);
|
||||||
|
}).toThrow(
|
||||||
|
'functional_tests: invalid argument [butter] to option [esFrom]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts value for grep', () => {
|
||||||
|
const options = processOptions({ grep: 'management' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts debug option', () => {
|
||||||
|
const options = processOptions({ debug: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts silent option', () => {
|
||||||
|
const options = processOptions({ silent: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts quiet option', () => {
|
||||||
|
const options = processOptions({ quiet: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts verbose option', () => {
|
||||||
|
const options = processOptions({ verbose: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts extra server options', () => {
|
||||||
|
const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid options even if valid options exist', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
|
||||||
|
}).toThrow('functional_tests: invalid option [aintnothang]');
|
||||||
|
});
|
||||||
|
});
|
47
packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
Normal file
47
packages/kbn-test/src/functional_tests/cli/run_tests/cli.js
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import chalk from 'chalk';
|
||||||
|
import getopts from 'getopts';
|
||||||
|
import { runTests } from '../../tasks';
|
||||||
|
import { processOptions, displayHelp } from './args';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run servers and tests for each config
|
||||||
|
* Only cares about --config option. Other options
|
||||||
|
* are passed directly to functional_test_runner, such as
|
||||||
|
* --bail, --verbose, etc.
|
||||||
|
* @param {string[]} defaultConfigPaths Optional paths to configs
|
||||||
|
* if no config option is passed
|
||||||
|
*/
|
||||||
|
export async function runTestsCli(defaultConfigPaths) {
|
||||||
|
try {
|
||||||
|
const userOptions = getopts(process.argv.slice(2)) || {};
|
||||||
|
if (userOptions.help) {
|
||||||
|
console.log(displayHelp());
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const options = processOptions(userOptions, defaultConfigPaths);
|
||||||
|
await runTests(options);
|
||||||
|
} catch (err) {
|
||||||
|
console.log(chalk.red(err));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
202
packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
Normal file
202
packages/kbn-test/src/functional_tests/cli/run_tests/cli.test.js
Normal file
|
@ -0,0 +1,202 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { runTestsCli } from './cli';
|
||||||
|
|
||||||
|
// Note: Stub the runTests function to keep testing only around the cli
|
||||||
|
// method and arguments.
|
||||||
|
jest.mock('../../tasks', () => ({
|
||||||
|
runTests: jest.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('run tests CLI', () => {
|
||||||
|
describe('options', () => {
|
||||||
|
const originalObjects = {};
|
||||||
|
const exitMock = jest.fn();
|
||||||
|
const logMock = jest.fn(); // mock logging so we don't send output to the test results
|
||||||
|
const argvMock = ['foo', 'foo'];
|
||||||
|
|
||||||
|
const processMock = {
|
||||||
|
exit: exitMock,
|
||||||
|
argv: argvMock,
|
||||||
|
stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
|
||||||
|
cwd: jest.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
originalObjects.process = process;
|
||||||
|
originalObjects.console = console;
|
||||||
|
global.process = processMock;
|
||||||
|
global.console = { log: logMock };
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
global.process = originalObjects.process;
|
||||||
|
global.console = originalObjects.console;
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
global.process.argv = [...argvMock];
|
||||||
|
jest.resetAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects boolean config value', async () => {
|
||||||
|
global.process.argv.push('--config');
|
||||||
|
|
||||||
|
await runTestsCli();
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects empty config value if no default passed', async () => {
|
||||||
|
global.process.argv.push('--config', '');
|
||||||
|
|
||||||
|
await runTestsCli();
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts empty config value if default passed', async () => {
|
||||||
|
global.process.argv.push('--config', '');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-boolean value for bail', async () => {
|
||||||
|
global.process.argv.push('--bail', 'peanut');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts string value for kibana-install-dir', async () => {
|
||||||
|
global.process.argv.push('--kibana-install-dir', 'foo');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects boolean value for kibana-install-dir', async () => {
|
||||||
|
global.process.argv.push('--kibana-install-dir');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts boolean value for updateBaselines', async () => {
|
||||||
|
global.process.argv.push('--updateBaselines');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalledWith();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts source value for esFrom', async () => {
|
||||||
|
global.process.argv.push('--esFrom', 'source');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-enum value for esFrom', async () => {
|
||||||
|
global.process.argv.push('--esFrom', 'butter');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts value for grep', async () => {
|
||||||
|
global.process.argv.push('--grep', 'management');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts debug option', async () => {
|
||||||
|
global.process.argv.push('--debug');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts silent option', async () => {
|
||||||
|
global.process.argv.push('--silent');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts quiet option', async () => {
|
||||||
|
global.process.argv.push('--quiet');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts verbose option', async () => {
|
||||||
|
global.process.argv.push('--verbose');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts extra server options', async () => {
|
||||||
|
global.process.argv.push('--', '--server.foo=bar');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts help option even if invalid options passed', async () => {
|
||||||
|
global.process.argv.push('--debug', '--aintnothang', '--help');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid options even if valid options exist', async () => {
|
||||||
|
global.process.argv.push('--debug', '--aintnothang', '--bail');
|
||||||
|
|
||||||
|
await runTestsCli(['foo']);
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -1,103 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
|
||||||
* license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright
|
|
||||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
|
||||||
* the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
* not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import dedent from 'dedent';
|
|
||||||
import getopts from 'getopts';
|
|
||||||
import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
|
|
||||||
import { runTests } from '../../';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run servers and tests for each config
|
|
||||||
* Only cares about --config option. Other options
|
|
||||||
* are passed directly to functional_test_runner, such as
|
|
||||||
* --bail, --verbose, etc.
|
|
||||||
* @param {string[]} defaultConfigPaths Array of paths to configs to use
|
|
||||||
* if no config option is passed
|
|
||||||
*/
|
|
||||||
export async function runTestsCli(defaultConfigPaths) {
|
|
||||||
const { configs, help, bail, log, installDir } = processArgs(
|
|
||||||
defaultConfigPaths
|
|
||||||
);
|
|
||||||
|
|
||||||
if (help) return displayHelp();
|
|
||||||
|
|
||||||
if (!configs || configs.length === 0) {
|
|
||||||
log.error(
|
|
||||||
`Run Tests requires at least one path to a config. Leave blank to use defaults.`
|
|
||||||
);
|
|
||||||
process.exit(9);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await runTests(configs, { bail, log, installDir });
|
|
||||||
} catch (err) {
|
|
||||||
log.error('FATAL ERROR');
|
|
||||||
log.error(err);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function processArgs(defaultConfigPaths) {
|
|
||||||
// If no args are passed, use {}
|
|
||||||
const options = getopts(process.argv.slice(2)) || {};
|
|
||||||
|
|
||||||
// If --config is passed without paths, it's "true", so use default
|
|
||||||
const configs =
|
|
||||||
typeof options.config === 'string' || Array.isArray(options.config)
|
|
||||||
? [].concat(options.config)
|
|
||||||
: defaultConfigPaths;
|
|
||||||
|
|
||||||
const log = createToolingLog(pickLevelFromFlags(options));
|
|
||||||
log.pipe(process.stdout);
|
|
||||||
|
|
||||||
return {
|
|
||||||
configs,
|
|
||||||
log,
|
|
||||||
help: options.help,
|
|
||||||
bail: options.bail,
|
|
||||||
installDir: options['kibana-install-dir'],
|
|
||||||
rest: options._,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function displayHelp() {
|
|
||||||
console.log(
|
|
||||||
dedent(`
|
|
||||||
Run Functional Tests
|
|
||||||
|
|
||||||
Usage: node scripts/functional_tests [options]
|
|
||||||
|
|
||||||
--config Option to pass in a config
|
|
||||||
Can pass in multiple configs with
|
|
||||||
--config file1 --config file2 --config file3
|
|
||||||
--kibana-install-dir
|
|
||||||
Run Kibana from an existing install directory
|
|
||||||
Default: run from source
|
|
||||||
--bail Stop the test run at the first failure
|
|
||||||
--help Display this menu and exit
|
|
||||||
|
|
||||||
Log level options:
|
|
||||||
|
|
||||||
--verbose
|
|
||||||
--debug
|
|
||||||
--quiet Log errors
|
|
||||||
--silent
|
|
||||||
`)
|
|
||||||
);
|
|
||||||
}
|
|
|
@ -0,0 +1,111 @@
|
||||||
|
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||||
|
|
||||||
|
exports[`display help for start servers CLI displays as expected 1`] = `
|
||||||
|
"Start Functional Test Servers
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
node scripts/functional_tests_server --help
|
||||||
|
node scripts/functional_tests_server [--config <file>]
|
||||||
|
node scripts/functional_tests_server [options] [-- --<other args>]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--help Display this menu and exit.
|
||||||
|
--config <file> Pass in a config
|
||||||
|
--esFrom <snapshot|source> Build Elasticsearch from source or run from snapshot. Default: snapshot
|
||||||
|
--kibana-install-dir <dir> Run Kibana from existing install directory instead of from source.
|
||||||
|
--verbose Log everything.
|
||||||
|
--debug Run in debug mode.
|
||||||
|
--quiet Only log errors.
|
||||||
|
--silent Log nothing."
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts debug option 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"debug": true,
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts empty config value if default passed 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts extra server options 1`] = `
|
||||||
|
Object {
|
||||||
|
"_": Object {
|
||||||
|
"server.foo": "bar",
|
||||||
|
},
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": Object {
|
||||||
|
"server.foo": "bar",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts quiet option 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"quiet": true,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts silent option 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"silent": true,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts source value for esFrom 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"esFrom": "source",
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"kibana-install-dir": "foo",
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`process options for start servers CLI accepts verbose option 1`] = `
|
||||||
|
Object {
|
||||||
|
"config": Array [
|
||||||
|
"foo",
|
||||||
|
],
|
||||||
|
"createLogger": [Function],
|
||||||
|
"extraKbnOpts": undefined,
|
||||||
|
"verbose": true,
|
||||||
|
}
|
||||||
|
`;
|
|
@ -0,0 +1,57 @@
|
||||||
|
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||||
|
|
||||||
|
exports[`start servers CLI options accepts boolean value for updateBaselines 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: invalid option [updateBaselines][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`start servers CLI options rejects bail 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: invalid option [bail][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`start servers CLI options rejects boolean config value 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: invalid argument [true] to option [config][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`start servers CLI options rejects boolean value for kibana-install-dir 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: invalid argument [true] to option [kibana-install-dir][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`start servers CLI options rejects empty config value if no default passed 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: config is required[39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`start servers CLI options rejects invalid options even if valid options exist 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: invalid option [grep][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
||||||
|
|
||||||
|
exports[`start servers CLI options rejects non-enum value for esFrom 1`] = `
|
||||||
|
Array [
|
||||||
|
Array [
|
||||||
|
"[31mError: functional_tests_server: invalid argument [butter] to option [esFrom][39m",
|
||||||
|
],
|
||||||
|
]
|
||||||
|
`;
|
119
packages/kbn-test/src/functional_tests/cli/start_servers/args.js
Normal file
119
packages/kbn-test/src/functional_tests/cli/start_servers/args.js
Normal file
|
@ -0,0 +1,119 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import dedent from 'dedent';
|
||||||
|
import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
|
||||||
|
|
||||||
|
const options = {
|
||||||
|
help: { desc: 'Display this menu and exit.' },
|
||||||
|
config: {
|
||||||
|
arg: '<file>',
|
||||||
|
desc: 'Pass in a config',
|
||||||
|
},
|
||||||
|
esFrom: {
|
||||||
|
arg: '<snapshot|source>',
|
||||||
|
choices: ['snapshot', 'source'],
|
||||||
|
desc: 'Build Elasticsearch from source or run from snapshot.',
|
||||||
|
default: 'snapshot',
|
||||||
|
},
|
||||||
|
'kibana-install-dir': {
|
||||||
|
arg: '<dir>',
|
||||||
|
desc: 'Run Kibana from existing install directory instead of from source.',
|
||||||
|
},
|
||||||
|
verbose: { desc: 'Log everything.' },
|
||||||
|
debug: { desc: 'Run in debug mode.' },
|
||||||
|
quiet: { desc: 'Only log errors.' },
|
||||||
|
silent: { desc: 'Log nothing.' },
|
||||||
|
};
|
||||||
|
|
||||||
|
export function displayHelp() {
|
||||||
|
const helpOptions = Object.keys(options)
|
||||||
|
.filter(name => name !== '_')
|
||||||
|
.map(name => {
|
||||||
|
const option = options[name];
|
||||||
|
return {
|
||||||
|
...option,
|
||||||
|
usage: `${name} ${option.arg || ''}`,
|
||||||
|
default: option.default ? `Default: ${option.default}` : '',
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.map(option => {
|
||||||
|
return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`;
|
||||||
|
})
|
||||||
|
.join(`\n `);
|
||||||
|
|
||||||
|
return dedent(`
|
||||||
|
Start Functional Test Servers
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
node scripts/functional_tests_server --help
|
||||||
|
node scripts/functional_tests_server [--config <file>]
|
||||||
|
node scripts/functional_tests_server [options] [-- --<other args>]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
${helpOptions}
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function processOptions(userOptions, defaultConfigPath) {
|
||||||
|
validateOptions(userOptions);
|
||||||
|
|
||||||
|
const config = userOptions.config || defaultConfigPath;
|
||||||
|
|
||||||
|
if (!config) {
|
||||||
|
throw new Error(`functional_tests_server: config is required`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createLogger() {
|
||||||
|
const log = createToolingLog(pickLevelFromFlags(userOptions));
|
||||||
|
log.pipe(process.stdout);
|
||||||
|
return log;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
...userOptions,
|
||||||
|
config,
|
||||||
|
createLogger,
|
||||||
|
extraKbnOpts: userOptions._,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateOptions(userOptions) {
|
||||||
|
Object.entries(userOptions).forEach(([key, val]) => {
|
||||||
|
if (key === '_') return;
|
||||||
|
|
||||||
|
// Validate flags passed
|
||||||
|
if (options[key] === undefined) {
|
||||||
|
throw new Error(`functional_tests_server: invalid option [${key}]`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
// Validate boolean flags
|
||||||
|
(!options[key].arg && typeof val !== 'boolean') ||
|
||||||
|
// Validate string/array flags
|
||||||
|
(options[key].arg && typeof val !== 'string' && !Array.isArray(val)) ||
|
||||||
|
// Validate enum flags
|
||||||
|
(options[key].choices && !options[key].choices.includes(val))
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`functional_tests_server: invalid argument [${val}] to option [${key}]`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
|
@ -0,0 +1,110 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { displayHelp, processOptions } from './args';
|
||||||
|
|
||||||
|
describe('display help for start servers CLI', () => {
|
||||||
|
it('displays as expected', () => {
|
||||||
|
expect(displayHelp()).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('process options for start servers CLI', () => {
|
||||||
|
it('rejects boolean config value', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ config: true });
|
||||||
|
}).toThrow(
|
||||||
|
'functional_tests_server: invalid argument [true] to option [config]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects empty config value if no default passed', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({});
|
||||||
|
}).toThrow('functional_tests_server: config is required');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts empty config value if default passed', () => {
|
||||||
|
const options = processOptions({ config: '' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid option', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ bail: true }, ['foo']);
|
||||||
|
}).toThrow('functional_tests_server: invalid option [bail]');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts string value for kibana-install-dir', () => {
|
||||||
|
const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects boolean value for kibana-install-dir', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ 'kibana-install-dir': true }, ['foo']);
|
||||||
|
}).toThrow(
|
||||||
|
'functional_tests_server: invalid argument [true] to option [kibana-install-dir]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts source value for esFrom', () => {
|
||||||
|
const options = processOptions({ esFrom: 'source' }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-enum value for esFrom', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ esFrom: 'butter' }, ['foo']);
|
||||||
|
}).toThrow(
|
||||||
|
'functional_tests_server: invalid argument [butter] to option [esFrom]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts debug option', () => {
|
||||||
|
const options = processOptions({ debug: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts silent option', () => {
|
||||||
|
const options = processOptions({ silent: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts quiet option', () => {
|
||||||
|
const options = processOptions({ quiet: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts verbose option', () => {
|
||||||
|
const options = processOptions({ verbose: true }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts extra server options', () => {
|
||||||
|
const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']);
|
||||||
|
expect(options).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid options even if valid options exist', () => {
|
||||||
|
expect(() => {
|
||||||
|
processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']);
|
||||||
|
}).toThrow('functional_tests_server: invalid option [aintnothang]');
|
||||||
|
});
|
||||||
|
});
|
|
@ -17,16 +17,28 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const functional = {
|
import chalk from 'chalk';
|
||||||
options: {
|
import getopts from 'getopts';
|
||||||
logLevel: 'debug',
|
import { startServers } from '../../tasks';
|
||||||
configFile: require.resolve('../../test/functional/config.js')
|
import { processOptions, displayHelp } from './args';
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const apiIntegration = {
|
/**
|
||||||
options: {
|
* Start servers
|
||||||
logLevel: 'debug',
|
* @param {string} defaultConfigPath Optional path to config
|
||||||
configFile: require.resolve('../../test/api_integration/config.js')
|
* if no config option is passed
|
||||||
|
*/
|
||||||
|
export async function startServersCli(defaultConfigPath) {
|
||||||
|
try {
|
||||||
|
const userOptions = getopts(process.argv.slice(2)) || {};
|
||||||
|
if (userOptions.help) {
|
||||||
|
console.log(displayHelp());
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const options = processOptions(userOptions, defaultConfigPath);
|
||||||
|
await startServers(options);
|
||||||
|
} catch (err) {
|
||||||
|
console.log(chalk.red(err));
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
|
|
@ -0,0 +1,194 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { startServersCli } from './cli';
|
||||||
|
|
||||||
|
// Note: Stub the startServers function to keep testing only around the cli
|
||||||
|
// method and arguments.
|
||||||
|
jest.mock('../../tasks', () => ({
|
||||||
|
startServers: jest.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('start servers CLI', () => {
|
||||||
|
describe('options', () => {
|
||||||
|
const originalObjects = {};
|
||||||
|
const exitMock = jest.fn();
|
||||||
|
const logMock = jest.fn(); // mock logging so we don't send output to the test results
|
||||||
|
const argvMock = ['foo', 'foo'];
|
||||||
|
|
||||||
|
const processMock = {
|
||||||
|
exit: exitMock,
|
||||||
|
argv: argvMock,
|
||||||
|
stdout: { on: jest.fn(), once: jest.fn(), emit: jest.fn() },
|
||||||
|
cwd: jest.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
originalObjects.process = process;
|
||||||
|
originalObjects.console = console;
|
||||||
|
global.process = processMock;
|
||||||
|
global.console = { log: logMock };
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
global.process = originalObjects.process;
|
||||||
|
global.console = originalObjects.console;
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
global.process.argv = [...argvMock];
|
||||||
|
jest.resetAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects boolean config value', async () => {
|
||||||
|
global.process.argv.push('--config');
|
||||||
|
|
||||||
|
await startServersCli();
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects empty config value if no default passed', async () => {
|
||||||
|
global.process.argv.push('--config', '');
|
||||||
|
|
||||||
|
await startServersCli();
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts empty config value if default passed', async () => {
|
||||||
|
global.process.argv.push('--config', '');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects bail', async () => {
|
||||||
|
global.process.argv.push('--bail', true);
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts string value for kibana-install-dir', async () => {
|
||||||
|
global.process.argv.push('--kibana-install-dir', 'foo');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects boolean value for kibana-install-dir', async () => {
|
||||||
|
global.process.argv.push('--kibana-install-dir');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts boolean value for updateBaselines', async () => {
|
||||||
|
global.process.argv.push('--updateBaselines');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts source value for esFrom', async () => {
|
||||||
|
global.process.argv.push('--esFrom', 'source');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-enum value for esFrom', async () => {
|
||||||
|
global.process.argv.push('--esFrom', 'butter');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts debug option', async () => {
|
||||||
|
global.process.argv.push('--debug');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts silent option', async () => {
|
||||||
|
global.process.argv.push('--silent');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts quiet option', async () => {
|
||||||
|
global.process.argv.push('--quiet');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts verbose option', async () => {
|
||||||
|
global.process.argv.push('--verbose');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts extra server options', async () => {
|
||||||
|
global.process.argv.push('--', '--server.foo=bar');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts help option even if invalid options passed', async () => {
|
||||||
|
global.process.argv.push('--debug', '--grep', '--help');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).not.toHaveBeenCalledWith(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid options even if valid options exist', async () => {
|
||||||
|
global.process.argv.push('--debug', '--grep', '--bail');
|
||||||
|
|
||||||
|
await startServersCli('foo');
|
||||||
|
|
||||||
|
expect(exitMock).toHaveBeenCalledWith(1);
|
||||||
|
expect(logMock.mock.calls).toMatchSnapshot();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -1,99 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
|
||||||
* license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright
|
|
||||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
|
||||||
* the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
* not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import chalk from 'chalk';
|
|
||||||
import dedent from 'dedent';
|
|
||||||
import getopts from 'getopts';
|
|
||||||
import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
|
|
||||||
import { startServers } from '../../';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start servers
|
|
||||||
* @param {string} configPath path to config
|
|
||||||
*/
|
|
||||||
export async function startServersCli(defaultConfigPath) {
|
|
||||||
const { config, log, help, installDir } = processArgv(defaultConfigPath);
|
|
||||||
|
|
||||||
if (help) return displayHelp();
|
|
||||||
|
|
||||||
if (!config) {
|
|
||||||
log.error(
|
|
||||||
`Start Servers requires one path to a config. Leave blank to use default.`
|
|
||||||
);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await startServers(config, { log, installDir });
|
|
||||||
} catch (err) {
|
|
||||||
log.error('FATAL ERROR');
|
|
||||||
log.error(err);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function processArgv(defaultConfigPath) {
|
|
||||||
const options = getopts(process.argv.slice(2)) || {};
|
|
||||||
|
|
||||||
if (Array.isArray(options.config)) {
|
|
||||||
console.log(
|
|
||||||
chalk.red(
|
|
||||||
`Starting servers requires a single config path. Multiple were passed.`
|
|
||||||
)
|
|
||||||
);
|
|
||||||
process.exit(9);
|
|
||||||
}
|
|
||||||
|
|
||||||
const config =
|
|
||||||
typeof options.config === 'string' ? options.config : defaultConfigPath;
|
|
||||||
|
|
||||||
const log = createToolingLog(pickLevelFromFlags(options));
|
|
||||||
log.pipe(process.stdout);
|
|
||||||
|
|
||||||
return {
|
|
||||||
config,
|
|
||||||
log,
|
|
||||||
installDir: options.kibanaInstallDir,
|
|
||||||
help: options.help,
|
|
||||||
rest: options._,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function displayHelp() {
|
|
||||||
console.log(
|
|
||||||
dedent(`
|
|
||||||
Start Functional Test Servers
|
|
||||||
|
|
||||||
Usage: node scripts/functional_tests_server [options]
|
|
||||||
|
|
||||||
--config Option to pass in a config
|
|
||||||
--kibana-install-dir
|
|
||||||
Run Kibana from an existing install directory
|
|
||||||
Default: run from source
|
|
||||||
--help Display this menu and exit
|
|
||||||
|
|
||||||
Log level options:
|
|
||||||
|
|
||||||
--verbose
|
|
||||||
--debug
|
|
||||||
--quiet Log errors
|
|
||||||
--silent
|
|
||||||
`)
|
|
||||||
);
|
|
||||||
}
|
|
|
@ -23,7 +23,8 @@ import { createEsTestCluster } from '../../es';
|
||||||
|
|
||||||
import { setupUsers, DEFAULT_SUPERUSER_PASS } from './auth';
|
import { setupUsers, DEFAULT_SUPERUSER_PASS } from './auth';
|
||||||
|
|
||||||
export async function runElasticsearch({ config, log }) {
|
export async function runElasticsearch({ config, options }) {
|
||||||
|
const { log, esFrom } = options;
|
||||||
const isOss = config.get('esTestCluster.license') === 'oss';
|
const isOss = config.get('esTestCluster.license') === 'oss';
|
||||||
|
|
||||||
const cluster = createEsTestCluster({
|
const cluster = createEsTestCluster({
|
||||||
|
@ -34,7 +35,7 @@ export async function runElasticsearch({ config, log }) {
|
||||||
license: config.get('esTestCluster.license'),
|
license: config.get('esTestCluster.license'),
|
||||||
log,
|
log,
|
||||||
basePath: resolve(KIBANA_ROOT, '.es'),
|
basePath: resolve(KIBANA_ROOT, '.es'),
|
||||||
from: config.get('esTestCluster.from'),
|
esFrom: esFrom || config.get('esTestCluster.from'),
|
||||||
});
|
});
|
||||||
|
|
||||||
const esArgs = config.get('esTestCluster.serverArgs');
|
const esArgs = config.get('esTestCluster.serverArgs');
|
||||||
|
|
|
@ -22,15 +22,16 @@ import { KIBANA_FTR_SCRIPT, PROJECT_ROOT } from './paths';
|
||||||
export async function runFtr({
|
export async function runFtr({
|
||||||
procs,
|
procs,
|
||||||
configPath,
|
configPath,
|
||||||
bail,
|
|
||||||
log,
|
|
||||||
cwd = PROJECT_ROOT,
|
cwd = PROJECT_ROOT,
|
||||||
|
options: { log, bail, grep, updateBaselines },
|
||||||
}) {
|
}) {
|
||||||
const args = [KIBANA_FTR_SCRIPT];
|
const args = [KIBANA_FTR_SCRIPT];
|
||||||
|
|
||||||
if (getLogFlag(log)) args.push(`--${getLogFlag(log)}`);
|
if (getLogFlag(log)) args.push(`--${getLogFlag(log)}`);
|
||||||
if (bail) args.push('--bail');
|
if (bail) args.push('--bail');
|
||||||
if (configPath) args.push('--config', configPath);
|
if (configPath) args.push('--config', configPath);
|
||||||
|
if (grep) args.push('--grep', grep);
|
||||||
|
if (updateBaselines) args.push('--updateBaselines');
|
||||||
|
|
||||||
await procs.run('ftr', {
|
await procs.run('ftr', {
|
||||||
cmd: 'node',
|
cmd: 'node',
|
||||||
|
|
|
@ -25,7 +25,7 @@ export async function runKibanaServer({ procs, config, options }) {
|
||||||
|
|
||||||
await procs.run('kibana', {
|
await procs.run('kibana', {
|
||||||
cmd: getKibanaCmd(installDir),
|
cmd: getKibanaCmd(installDir),
|
||||||
args: getCliArgs(config, options),
|
args: collectCliArgs(config, options),
|
||||||
env: {
|
env: {
|
||||||
FORCE_COLOR: 1,
|
FORCE_COLOR: 1,
|
||||||
...process.env,
|
...process.env,
|
||||||
|
@ -45,14 +45,35 @@ function getKibanaCmd(installDir) {
|
||||||
return KIBANA_EXEC;
|
return KIBANA_EXEC;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getCliArgs(config, { devMode, installDir }) {
|
/* When installDir is passed, we run from a built version of Kibana,
|
||||||
|
* which uses different command line arguments. If installDir is not
|
||||||
|
* passed, we run from source code. We also allow passing in extra
|
||||||
|
* Kibana server options, so we tack those on here.
|
||||||
|
*/
|
||||||
|
function collectCliArgs(config, { installDir, extraKbnOpts }) {
|
||||||
const buildArgs = config.get('kbnTestServer.buildArgs') || [];
|
const buildArgs = config.get('kbnTestServer.buildArgs') || [];
|
||||||
const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
|
const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
|
||||||
const serverArgs = config.get('kbnTestServer.serverArgs') || [];
|
const serverArgs = config.get('kbnTestServer.serverArgs') || [];
|
||||||
|
|
||||||
if (devMode) serverArgs.push('--dev');
|
return pipe(
|
||||||
|
serverArgs,
|
||||||
|
args => (installDir ? args.filter(a => a !== '--oss') : args),
|
||||||
|
args => {
|
||||||
return installDir
|
return installDir
|
||||||
? [...serverArgs, ...buildArgs]
|
? [...args, ...buildArgs]
|
||||||
: [KIBANA_EXEC_PATH, ...serverArgs, ...sourceArgs];
|
: [KIBANA_EXEC_PATH, ...args, ...sourceArgs];
|
||||||
|
},
|
||||||
|
args => args.concat(extraKbnOpts || [])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Apply each function in fns to the result of the
|
||||||
|
* previous function. The first function's input
|
||||||
|
* is the arr array.
|
||||||
|
*/
|
||||||
|
function pipe(arr, ...fns) {
|
||||||
|
return fns.reduce((acc, fn) => {
|
||||||
|
return fn(acc);
|
||||||
|
}, arr);
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,39 +42,47 @@ in another terminal session by running this command from this directory:
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run servers and tests for each config
|
* Run servers and tests for each config
|
||||||
* @param {string[]} configPaths Array of paths to configs
|
|
||||||
* @param {object} options Optional
|
* @param {object} options Optional
|
||||||
* @param {Log} options.log Optional logger
|
* @property {string[]} configPaths Array of paths to configs
|
||||||
* @param {string} options.installDir Optional installation dir
|
* @property {function} options.createLogger Optional logger creation function
|
||||||
* from which to run Kibana
|
* @property {string} options.installDir Optional installation dir from which to run Kibana
|
||||||
* @param {boolean} options.bail Whether to exit test run at the first failure
|
* @property {boolean} options.bail Whether to exit test run at the first failure
|
||||||
|
* @property {string} options.esFrom Optionally run from source instead of snapshot
|
||||||
*/
|
*/
|
||||||
export async function runTests(configPaths, options) {
|
export async function runTests(options) {
|
||||||
for (const configPath of configPaths) {
|
for (const configPath of options.configs) {
|
||||||
await runSingleConfig(resolve(process.cwd(), configPath), options);
|
await runSingleConfig(resolve(process.cwd(), configPath), options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start only servers using single config
|
* Start only servers using single config
|
||||||
* @param {string} configPath Path to a config file
|
|
||||||
* @param {object} options Optional
|
* @param {object} options Optional
|
||||||
* @param {Log} options.log Optional logger
|
* @property {string} options.configPath Path to a config file
|
||||||
* @param {string} options.installDir Optional installation dir
|
* @property {function} options.createLogger Optional logger creation function
|
||||||
* from which to run Kibana
|
* @property {string} options.installDir Optional installation dir from which to run Kibana
|
||||||
|
* @property {string} options.esFrom Optionally run from source instead of snapshot
|
||||||
*/
|
*/
|
||||||
export async function startServers(configPath, options) {
|
export async function startServers(options) {
|
||||||
const { log } = options;
|
const { config: configOption, createLogger } = options;
|
||||||
configPath = resolve(process.cwd(), configPath);
|
const configPath = resolve(process.cwd(), configOption);
|
||||||
|
const log = createLogger();
|
||||||
|
const opts = {
|
||||||
|
...options,
|
||||||
|
log,
|
||||||
|
};
|
||||||
|
|
||||||
await withProcRunner(log, async procs => {
|
await withProcRunner(log, async procs => {
|
||||||
const config = await readConfigFile(log, configPath);
|
const config = await readConfigFile(log, configPath);
|
||||||
|
|
||||||
const es = await runElasticsearch({ config, log });
|
const es = await runElasticsearch({ config, options: opts });
|
||||||
await runKibanaServer({
|
await runKibanaServer({
|
||||||
procs,
|
procs,
|
||||||
config,
|
config,
|
||||||
options: { ...options, devMode: true },
|
options: {
|
||||||
|
...opts,
|
||||||
|
extraKbnOpts: [...options.extraKbnOpts, '--dev'],
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// wait for 5 seconds of silence before logging the
|
// wait for 5 seconds of silence before logging the
|
||||||
|
@ -97,22 +105,25 @@ async function silence(milliseconds, { log }) {
|
||||||
* Start servers and run tests for single config
|
* Start servers and run tests for single config
|
||||||
*/
|
*/
|
||||||
async function runSingleConfig(configPath, options) {
|
async function runSingleConfig(configPath, options) {
|
||||||
const { bail, log } = options;
|
const log = options.createLogger();
|
||||||
|
const opts = {
|
||||||
|
...options,
|
||||||
|
log,
|
||||||
|
};
|
||||||
|
|
||||||
await withProcRunner(log, async procs => {
|
await withProcRunner(log, async procs => {
|
||||||
const config = await readConfigFile(log, configPath);
|
const config = await readConfigFile(log, configPath);
|
||||||
|
|
||||||
const es = await runElasticsearch({ config, log });
|
const es = await runElasticsearch({ config, options: opts });
|
||||||
await runKibanaServer({ procs, config, options });
|
await runKibanaServer({ procs, config, options: opts });
|
||||||
|
|
||||||
// Note: When solving how to incorporate functional_test_runner
|
// Note: When solving how to incorporate functional_test_runner
|
||||||
// clean this up
|
// clean this up
|
||||||
await runFtr({
|
await runFtr({
|
||||||
procs,
|
procs,
|
||||||
configPath,
|
configPath,
|
||||||
bail,
|
|
||||||
log,
|
|
||||||
cwd: process.cwd(),
|
cwd: process.cwd(),
|
||||||
|
options: opts,
|
||||||
});
|
});
|
||||||
|
|
||||||
await procs.stop('kibana');
|
await procs.stop('kibana');
|
||||||
|
|
|
@ -17,12 +17,8 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { esTestConfig, kbnTestConfig } from '@kbn/test';
|
|
||||||
import { resolve } from 'path';
|
import { resolve } from 'path';
|
||||||
|
|
||||||
const SECOND = 1000;
|
|
||||||
const MINUTE = 60 * SECOND;
|
|
||||||
const HOUR = 60 * MINUTE;
|
|
||||||
const PKG_VERSION = require('../../package.json').version;
|
const PKG_VERSION = require('../../package.json').version;
|
||||||
|
|
||||||
module.exports = function (grunt) {
|
module.exports = function (grunt) {
|
||||||
|
@ -58,20 +54,6 @@ module.exports = function (grunt) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const apiTestServerFlags = [
|
|
||||||
'--optimize.enabled=false',
|
|
||||||
'--elasticsearch.url=' + esTestConfig.getUrl(),
|
|
||||||
'--elasticsearch.healthCheck.delay=' + HOUR,
|
|
||||||
'--server.port=' + kbnTestConfig.getPort(),
|
|
||||||
'--server.xsrf.disableProtection=true',
|
|
||||||
];
|
|
||||||
|
|
||||||
const funcTestServerFlags = [
|
|
||||||
'--server.maxPayloadBytes=1648576', //default is 1048576
|
|
||||||
'--elasticsearch.url=' + esTestConfig.getUrl(),
|
|
||||||
'--server.port=' + kbnTestConfig.getPort(),
|
|
||||||
];
|
|
||||||
|
|
||||||
const browserTestServerFlags = [
|
const browserTestServerFlags = [
|
||||||
'--plugins.initialize=false',
|
'--plugins.initialize=false',
|
||||||
'--optimize.bundleFilter=tests',
|
'--optimize.bundleFilter=tests',
|
||||||
|
@ -117,55 +99,6 @@ module.exports = function (grunt) {
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|
||||||
// used by the test:api task
|
|
||||||
// runs the kibana server prepared for the api_integration tests
|
|
||||||
apiTestServer: createKbnServerTask({
|
|
||||||
flags: [
|
|
||||||
...apiTestServerFlags
|
|
||||||
]
|
|
||||||
}),
|
|
||||||
|
|
||||||
// used by the test:api:server task
|
|
||||||
// runs the kibana server in --dev mode, prepared for developing api_integration tests
|
|
||||||
// and watching for changes so the server will restart when necessary
|
|
||||||
devApiTestServer: createKbnServerTask({
|
|
||||||
flags: [
|
|
||||||
'--dev',
|
|
||||||
'--no-base-path',
|
|
||||||
...apiTestServerFlags,
|
|
||||||
]
|
|
||||||
}),
|
|
||||||
|
|
||||||
// used by test:ui task
|
|
||||||
// runs the kibana server prepared for the functional tests
|
|
||||||
funcTestServer: createKbnServerTask({
|
|
||||||
flags: [
|
|
||||||
...funcTestServerFlags,
|
|
||||||
]
|
|
||||||
}),
|
|
||||||
|
|
||||||
// used by the test:ui:server task
|
|
||||||
// runs the kibana server in dev mode, prepared for the functional tests
|
|
||||||
devFuncTestServer: createKbnServerTask({
|
|
||||||
flags: [
|
|
||||||
...funcTestServerFlags,
|
|
||||||
'--dev',
|
|
||||||
'--no-base-path',
|
|
||||||
'--optimize.watchPort=5611',
|
|
||||||
'--optimize.watchPrebuild=true',
|
|
||||||
'--optimize.bundleDir=' + resolve(__dirname, '../../optimize/testUiServer'),
|
|
||||||
]
|
|
||||||
}),
|
|
||||||
|
|
||||||
// used by test:uiRelease task
|
|
||||||
// runs the kibana server from the oss distributable prepared for the functional tests
|
|
||||||
ossDistFuncTestServer: createKbnServerTask({
|
|
||||||
runBuild: `oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
|
|
||||||
flags: [
|
|
||||||
...funcTestServerFlags,
|
|
||||||
]
|
|
||||||
}),
|
|
||||||
|
|
||||||
// used by the test:browser task
|
// used by the test:browser task
|
||||||
// runs the kibana server to serve the browser test bundle
|
// runs the kibana server to serve the browser test bundle
|
||||||
browserTestServer: createKbnServerTask({
|
browserTestServer: createKbnServerTask({
|
||||||
|
@ -198,22 +131,6 @@ module.exports = function (grunt) {
|
||||||
]
|
]
|
||||||
}),
|
}),
|
||||||
|
|
||||||
testEsServer: {
|
|
||||||
options: {
|
|
||||||
wait: false,
|
|
||||||
ready: /started/,
|
|
||||||
quiet: false,
|
|
||||||
},
|
|
||||||
cmd: process.execPath,
|
|
||||||
args: [
|
|
||||||
'scripts/es',
|
|
||||||
grunt.option('from') || 'snapshot',
|
|
||||||
'--license', 'oss',
|
|
||||||
'-E', `http.port=${esTestConfig.getPort()}`,
|
|
||||||
'-E', `discovery.zen.ping.unicast.hosts=localhost:${esTestConfig.getPort()}`,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
|
|
||||||
verifyNotice: {
|
verifyNotice: {
|
||||||
options: {
|
options: {
|
||||||
wait: true,
|
wait: true,
|
||||||
|
@ -223,6 +140,56 @@ module.exports = function (grunt) {
|
||||||
'scripts/notice',
|
'scripts/notice',
|
||||||
'--validate'
|
'--validate'
|
||||||
]
|
]
|
||||||
}
|
},
|
||||||
|
|
||||||
|
apiIntegrationTests: {
|
||||||
|
cmd: process.execPath,
|
||||||
|
args: [
|
||||||
|
'scripts/functional_tests',
|
||||||
|
'--config', 'test/api_integration/config.js',
|
||||||
|
'--esFrom', 'source',
|
||||||
|
'--bail',
|
||||||
|
'--debug',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
functionalTests: {
|
||||||
|
cmd: process.execPath,
|
||||||
|
args: [
|
||||||
|
'scripts/functional_tests',
|
||||||
|
'--config', 'test/functional/config.js',
|
||||||
|
'--esFrom', 'source',
|
||||||
|
'--bail',
|
||||||
|
'--debug',
|
||||||
|
'--',
|
||||||
|
'--server.maxPayloadBytes=1648576',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
functionalTestsRelease: {
|
||||||
|
cmd: process.execPath,
|
||||||
|
args: [
|
||||||
|
'scripts/functional_tests',
|
||||||
|
'--config', 'test/functional/config.js',
|
||||||
|
'--esFrom', 'source',
|
||||||
|
'--bail',
|
||||||
|
'--debug',
|
||||||
|
'--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
|
||||||
|
'--',
|
||||||
|
'--server.maxPayloadBytes=1648576',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
functionalTestsDevServer: {
|
||||||
|
cmd: process.execPath,
|
||||||
|
args: [
|
||||||
|
'scripts/functional_tests_server',
|
||||||
|
'--config', 'test/functional/config.js',
|
||||||
|
'--esFrom', 'source',
|
||||||
|
'--debug',
|
||||||
|
'--',
|
||||||
|
'--server.maxPayloadBytes=1648576',
|
||||||
|
],
|
||||||
|
},
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
@ -34,11 +34,10 @@ module.exports = function (grunt) {
|
||||||
'test:jest_integration',
|
'test:jest_integration',
|
||||||
'test:projects',
|
'test:projects',
|
||||||
'test:browser-ci',
|
'test:browser-ci',
|
||||||
'test:api',
|
'run:apiIntegrationTests',
|
||||||
'verifyTranslations',
|
'verifyTranslations',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
grunt.config.set('functional_test_runner.functional.options.configOverrides.mochaOpts.bail', true);
|
|
||||||
grunt.registerTask('jenkins:selenium', [
|
grunt.registerTask('jenkins:selenium', [
|
||||||
'test:uiRelease'
|
'test:uiRelease'
|
||||||
]);
|
]);
|
||||||
|
|
|
@ -64,7 +64,7 @@ module.exports = function (grunt) {
|
||||||
'test:jest_integration',
|
'test:jest_integration',
|
||||||
'test:projects',
|
'test:projects',
|
||||||
'test:browser',
|
'test:browser',
|
||||||
'test:api'
|
'run:apiIntegrationTests'
|
||||||
]);
|
]);
|
||||||
|
|
||||||
grunt.registerTask('test:dev', [
|
grunt.registerTask('test:dev', [
|
||||||
|
@ -75,45 +75,19 @@ module.exports = function (grunt) {
|
||||||
|
|
||||||
grunt.registerTask('test:ui', [
|
grunt.registerTask('test:ui', [
|
||||||
'checkPlugins',
|
'checkPlugins',
|
||||||
'run:testEsServer',
|
'run:functionalTests',
|
||||||
'run:funcTestServer',
|
|
||||||
'functional_test_runner:functional',
|
|
||||||
'stop:testEsServer',
|
|
||||||
'stop:funcTestServer'
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
grunt.registerTask('test:uiRelease', [
|
grunt.registerTask('test:uiRelease', [
|
||||||
'checkPlugins',
|
'checkPlugins',
|
||||||
'run:testEsServer',
|
'run:functionalTestsRelease',
|
||||||
'run:ossDistFuncTestServer',
|
|
||||||
'functional_test_runner:functional',
|
|
||||||
'stop:testEsServer',
|
|
||||||
'stop:ossDistFuncTestServer'
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
grunt.registerTask('test:ui:server', [
|
grunt.registerTask('test:ui:server', [
|
||||||
'checkPlugins',
|
'checkPlugins',
|
||||||
'run:testEsServer',
|
'run:functionalTestsDevServer',
|
||||||
'run:devFuncTestServer:keepalive'
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
grunt.registerTask('test:api', [
|
|
||||||
'run:testEsServer',
|
|
||||||
'run:apiTestServer',
|
|
||||||
'functional_test_runner:apiIntegration',
|
|
||||||
'stop:testEsServer',
|
|
||||||
'stop:apiTestServer'
|
|
||||||
]);
|
|
||||||
|
|
||||||
grunt.registerTask('test:api:server', [
|
|
||||||
'run:testEsServer',
|
|
||||||
'run:devApiTestServer:keepalive'
|
|
||||||
]);
|
|
||||||
|
|
||||||
grunt.registerTask('test:api:runner', () => {
|
|
||||||
grunt.fail.fatal('test:api:runner has moved, use: `node scripts/functional_test_runner --config test/api_integration/config.js`');
|
|
||||||
});
|
|
||||||
|
|
||||||
grunt.registerTask('test', subTask => {
|
grunt.registerTask('test', subTask => {
|
||||||
if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
|
if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
|
||||||
|
|
||||||
|
|
|
@ -49,7 +49,7 @@ export default function () {
|
||||||
`--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
|
`--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
|
||||||
],
|
],
|
||||||
serverArgs: [
|
serverArgs: [
|
||||||
'--env=development',
|
'--env.name=development',
|
||||||
'--logging.json=false',
|
'--logging.json=false',
|
||||||
`--server.port=${kbnTestConfig.getPort()}`,
|
`--server.port=${kbnTestConfig.getPort()}`,
|
||||||
`--optimize.watchPort=${kbnTestConfig.getPort() + 10}`,
|
`--optimize.watchPort=${kbnTestConfig.getPort() + 10}`,
|
||||||
|
|
|
@ -37,6 +37,6 @@ tar -xzf "$linuxBuild" -C "$installDir" --strip=1
|
||||||
|
|
||||||
echo " -> Running functional and api tests"
|
echo " -> Running functional and api tests"
|
||||||
cd "$XPACK_DIR"
|
cd "$XPACK_DIR"
|
||||||
xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --es-from=source
|
xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --esFrom=source
|
||||||
echo ""
|
echo ""
|
||||||
echo ""
|
echo ""
|
||||||
|
|
Loading…
Reference in a new issue