[FTR] Add test suite metrics tracking/output (#62515)
This commit is contained in:
parent
53a0752841
commit
3ce3da8a65
|
@ -30,6 +30,7 @@ import {
|
||||||
setupMocha,
|
setupMocha,
|
||||||
runTests,
|
runTests,
|
||||||
Config,
|
Config,
|
||||||
|
SuiteTracker,
|
||||||
} from './lib';
|
} from './lib';
|
||||||
|
|
||||||
export class FunctionalTestRunner {
|
export class FunctionalTestRunner {
|
||||||
|
@ -52,6 +53,8 @@ export class FunctionalTestRunner {
|
||||||
|
|
||||||
async run() {
|
async run() {
|
||||||
return await this._run(async (config, coreProviders) => {
|
return await this._run(async (config, coreProviders) => {
|
||||||
|
SuiteTracker.startTracking(this.lifecycle, this.configFile);
|
||||||
|
|
||||||
const providers = new ProviderCollection(this.log, [
|
const providers = new ProviderCollection(this.log, [
|
||||||
...coreProviders,
|
...coreProviders,
|
||||||
...readProviderSpec('Service', config.get('services')),
|
...readProviderSpec('Service', config.get('services')),
|
||||||
|
|
|
@ -23,3 +23,4 @@ export { readConfigFile, Config } from './config';
|
||||||
export { readProviderSpec, ProviderCollection, Provider } from './providers';
|
export { readProviderSpec, ProviderCollection, Provider } from './providers';
|
||||||
export { runTests, setupMocha } from './mocha';
|
export { runTests, setupMocha } from './mocha';
|
||||||
export { FailureMetadata } from './failure_metadata';
|
export { FailureMetadata } from './failure_metadata';
|
||||||
|
export { SuiteTracker } from './suite_tracker';
|
||||||
|
|
|
@ -0,0 +1,197 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
import { join, resolve } from 'path';
|
||||||
|
|
||||||
|
jest.mock('fs');
|
||||||
|
jest.mock('@kbn/dev-utils', () => {
|
||||||
|
return { REPO_ROOT: '/dev/null/root' };
|
||||||
|
});
|
||||||
|
|
||||||
|
import { REPO_ROOT } from '@kbn/dev-utils';
|
||||||
|
import { Lifecycle } from './lifecycle';
|
||||||
|
import { SuiteTracker } from './suite_tracker';
|
||||||
|
|
||||||
|
const DEFAULT_TEST_METADATA_PATH = join(REPO_ROOT, 'target', 'test_metadata.json');
|
||||||
|
const MOCK_CONFIG_PATH = join('test', 'config.js');
|
||||||
|
const MOCK_TEST_PATH = join('test', 'apps', 'test.js');
|
||||||
|
const ENVS_TO_RESET = ['TEST_METADATA_PATH'];
|
||||||
|
|
||||||
|
describe('SuiteTracker', () => {
|
||||||
|
const originalEnvs: Record<string, string> = {};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
for (const env of ENVS_TO_RESET) {
|
||||||
|
if (env in process.env) {
|
||||||
|
originalEnvs[env] = process.env[env] || '';
|
||||||
|
delete process.env[env];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const env of ENVS_TO_RESET) {
|
||||||
|
delete process.env[env];
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const env of Object.keys(originalEnvs)) {
|
||||||
|
process.env[env] = originalEnvs[env];
|
||||||
|
}
|
||||||
|
|
||||||
|
jest.resetAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
let MOCKS: Record<string, object>;
|
||||||
|
|
||||||
|
const createMock = (overrides = {}) => {
|
||||||
|
return {
|
||||||
|
file: resolve(REPO_ROOT, MOCK_TEST_PATH),
|
||||||
|
title: 'A Test',
|
||||||
|
suiteTag: MOCK_TEST_PATH,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const runLifecycleWithMocks = async (mocks: object[], fn: (objs: any) => any = () => {}) => {
|
||||||
|
const lifecycle = new Lifecycle();
|
||||||
|
const suiteTracker = SuiteTracker.startTracking(
|
||||||
|
lifecycle,
|
||||||
|
resolve(REPO_ROOT, MOCK_CONFIG_PATH)
|
||||||
|
);
|
||||||
|
|
||||||
|
const ret = { lifecycle, suiteTracker };
|
||||||
|
|
||||||
|
for (const mock of mocks) {
|
||||||
|
await lifecycle.beforeTestSuite.trigger(mock);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fn) {
|
||||||
|
fn(ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const mock of mocks.reverse()) {
|
||||||
|
await lifecycle.afterTestSuite.trigger(mock);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
MOCKS = {
|
||||||
|
WITH_TESTS: createMock({ tests: [{}] }), // i.e. a describe with tests in it
|
||||||
|
WITHOUT_TESTS: createMock(), // i.e. a describe with only other describes in it
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
it('collects metadata for a single suite with multiple describe()s', async () => {
|
||||||
|
const { suiteTracker } = await runLifecycleWithMocks([MOCKS.WITHOUT_TESTS, MOCKS.WITH_TESTS]);
|
||||||
|
|
||||||
|
const suites = suiteTracker.getAllFinishedSuites();
|
||||||
|
expect(suites.length).toBe(1);
|
||||||
|
const suite = suites[0];
|
||||||
|
|
||||||
|
expect(suite).toMatchObject({
|
||||||
|
config: MOCK_CONFIG_PATH,
|
||||||
|
file: MOCK_TEST_PATH,
|
||||||
|
tag: MOCK_TEST_PATH,
|
||||||
|
hasTests: true,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('writes metadata to a file when cleanup is triggered', async () => {
|
||||||
|
const { lifecycle, suiteTracker } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]);
|
||||||
|
await lifecycle.cleanup.trigger();
|
||||||
|
|
||||||
|
const suites = suiteTracker.getAllFinishedSuites();
|
||||||
|
|
||||||
|
const call = (fs.writeFileSync as jest.Mock).mock.calls[0];
|
||||||
|
expect(call[0]).toEqual(DEFAULT_TEST_METADATA_PATH);
|
||||||
|
expect(call[1]).toEqual(JSON.stringify(suites, null, 2));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('respects TEST_METADATA_PATH env var for metadata target override', async () => {
|
||||||
|
process.env.TEST_METADATA_PATH = resolve(REPO_ROOT, '../fake-test-path');
|
||||||
|
const { lifecycle } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]);
|
||||||
|
await lifecycle.cleanup.trigger();
|
||||||
|
|
||||||
|
expect((fs.writeFileSync as jest.Mock).mock.calls[0][0]).toEqual(
|
||||||
|
process.env.TEST_METADATA_PATH
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('identifies suites with tests as leaf suites', async () => {
|
||||||
|
const root = createMock({ title: 'root', file: join(REPO_ROOT, 'root.js') });
|
||||||
|
const parent = createMock({ parent: root });
|
||||||
|
const withTests = createMock({ parent, tests: [{}] });
|
||||||
|
|
||||||
|
const { suiteTracker } = await runLifecycleWithMocks([root, parent, withTests]);
|
||||||
|
const suites = suiteTracker.getAllFinishedSuites();
|
||||||
|
|
||||||
|
const finishedRoot = suites.find(s => s.title === 'root');
|
||||||
|
const finishedWithTests = suites.find(s => s.title !== 'root');
|
||||||
|
|
||||||
|
expect(finishedRoot).toBeTruthy();
|
||||||
|
expect(finishedRoot?.hasTests).toBeFalsy();
|
||||||
|
expect(finishedWithTests?.hasTests).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('with a failing suite', () => {
|
||||||
|
let root: any;
|
||||||
|
let parent: any;
|
||||||
|
let failed: any;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = createMock({ file: join(REPO_ROOT, 'root.js') });
|
||||||
|
parent = createMock({ parent: root });
|
||||||
|
failed = createMock({ parent, tests: [{}] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('marks parent suites as not successful when a test fails', async () => {
|
||||||
|
const { suiteTracker } = await runLifecycleWithMocks(
|
||||||
|
[root, parent, failed],
|
||||||
|
async ({ lifecycle }) => {
|
||||||
|
await lifecycle.testFailure.trigger(Error('test'), { parent: failed });
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const suites = suiteTracker.getAllFinishedSuites();
|
||||||
|
expect(suites.length).toBe(2);
|
||||||
|
for (const suite of suites) {
|
||||||
|
expect(suite.success).toBeFalsy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('marks parent suites as not successful when a test hook fails', async () => {
|
||||||
|
const { suiteTracker } = await runLifecycleWithMocks(
|
||||||
|
[root, parent, failed],
|
||||||
|
async ({ lifecycle }) => {
|
||||||
|
await lifecycle.testHookFailure.trigger(Error('test'), { parent: failed });
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const suites = suiteTracker.getAllFinishedSuites();
|
||||||
|
expect(suites.length).toBe(2);
|
||||||
|
for (const suite of suites) {
|
||||||
|
expect(suite.success).toBeFalsy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
|
@ -0,0 +1,147 @@
|
||||||
|
/*
|
||||||
|
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||||
|
* license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright
|
||||||
|
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||||
|
* the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
* not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
import fs from 'fs';
|
||||||
|
import { dirname, relative, resolve } from 'path';
|
||||||
|
|
||||||
|
import { REPO_ROOT } from '@kbn/dev-utils';
|
||||||
|
|
||||||
|
import { Lifecycle } from './lifecycle';
|
||||||
|
|
||||||
|
export interface SuiteInProgress {
|
||||||
|
startTime?: Date;
|
||||||
|
endTime?: Date;
|
||||||
|
success?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SuiteWithMetadata {
|
||||||
|
config: string;
|
||||||
|
file: string;
|
||||||
|
tag: string;
|
||||||
|
title: string;
|
||||||
|
startTime: Date;
|
||||||
|
endTime: Date;
|
||||||
|
duration: number;
|
||||||
|
success: boolean;
|
||||||
|
hasTests: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const getTestMetadataPath = () => {
|
||||||
|
return process.env.TEST_METADATA_PATH || resolve(REPO_ROOT, 'target', 'test_metadata.json');
|
||||||
|
};
|
||||||
|
|
||||||
|
export class SuiteTracker {
|
||||||
|
finishedSuitesByConfig: Record<string, Record<string, SuiteWithMetadata>> = {};
|
||||||
|
inProgressSuites: Map<object, SuiteInProgress> = new Map<object, SuiteInProgress>();
|
||||||
|
|
||||||
|
static startTracking(lifecycle: Lifecycle, configPath: string): SuiteTracker {
|
||||||
|
const suiteTracker = new SuiteTracker(lifecycle, configPath);
|
||||||
|
return suiteTracker;
|
||||||
|
}
|
||||||
|
|
||||||
|
getTracked(suite: object): SuiteInProgress {
|
||||||
|
if (!this.inProgressSuites.has(suite)) {
|
||||||
|
this.inProgressSuites.set(suite, { success: undefined } as SuiteInProgress);
|
||||||
|
}
|
||||||
|
return this.inProgressSuites.get(suite)!;
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(lifecycle: Lifecycle, configPathAbsolute: string) {
|
||||||
|
if (fs.existsSync(getTestMetadataPath())) {
|
||||||
|
fs.unlinkSync(getTestMetadataPath());
|
||||||
|
} else {
|
||||||
|
fs.mkdirSync(dirname(getTestMetadataPath()), { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const config = relative(REPO_ROOT, configPathAbsolute);
|
||||||
|
|
||||||
|
lifecycle.beforeTestSuite.add(suite => {
|
||||||
|
const tracked = this.getTracked(suite);
|
||||||
|
tracked.startTime = new Date();
|
||||||
|
});
|
||||||
|
|
||||||
|
// If a test fails, we want to make sure all of the ancestors, all the way up to the root, get marked as failed
|
||||||
|
// This information is not available on the mocha objects without traversing all descendants of a given node
|
||||||
|
const handleFailure = (_: any, test: any) => {
|
||||||
|
let parent = test.parent;
|
||||||
|
|
||||||
|
// Infinite loop protection, just in case
|
||||||
|
for (let i = 0; i < 500 && parent; i++) {
|
||||||
|
if (this.inProgressSuites.has(parent)) {
|
||||||
|
this.getTracked(parent).success = false;
|
||||||
|
}
|
||||||
|
parent = parent.parent;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
lifecycle.testFailure.add(handleFailure);
|
||||||
|
lifecycle.testHookFailure.add(handleFailure);
|
||||||
|
|
||||||
|
lifecycle.afterTestSuite.add(suite => {
|
||||||
|
const tracked = this.getTracked(suite);
|
||||||
|
tracked.endTime = new Date();
|
||||||
|
|
||||||
|
// The suite ended without any children failing, so we can mark it as successful
|
||||||
|
if (typeof tracked.success === 'undefined') {
|
||||||
|
tracked.success = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let duration = tracked.endTime.getTime() - (tracked.startTime || new Date()).getTime();
|
||||||
|
duration = Math.floor(duration / 1000);
|
||||||
|
|
||||||
|
const file = relative(REPO_ROOT, suite.file);
|
||||||
|
|
||||||
|
this.finishedSuitesByConfig[config] = this.finishedSuitesByConfig[config] || {};
|
||||||
|
|
||||||
|
// This will get called multiple times for a test file that has multiple describes in it or similar
|
||||||
|
// This is okay, because the last one that fires is always the root of the file, which is is the one we ultimately want
|
||||||
|
this.finishedSuitesByConfig[config][file] = {
|
||||||
|
...tracked,
|
||||||
|
duration,
|
||||||
|
config,
|
||||||
|
file,
|
||||||
|
tag: suite.suiteTag,
|
||||||
|
title: suite.title,
|
||||||
|
hasTests: !!(
|
||||||
|
(suite.tests && suite.tests.length) ||
|
||||||
|
// The below statement is so that `hasTests` will bubble up nested describes in the same file
|
||||||
|
(this.finishedSuitesByConfig[config][file] &&
|
||||||
|
this.finishedSuitesByConfig[config][file].hasTests)
|
||||||
|
),
|
||||||
|
} as SuiteWithMetadata;
|
||||||
|
});
|
||||||
|
|
||||||
|
lifecycle.cleanup.add(() => {
|
||||||
|
const suites = this.getAllFinishedSuites();
|
||||||
|
|
||||||
|
fs.writeFileSync(getTestMetadataPath(), JSON.stringify(suites, null, 2));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
getAllFinishedSuites() {
|
||||||
|
const flattened: SuiteWithMetadata[] = [];
|
||||||
|
for (const byFile of Object.values(this.finishedSuitesByConfig)) {
|
||||||
|
for (const suite of Object.values(byFile)) {
|
||||||
|
flattened.push(suite);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
flattened.sort((a, b) => b.duration - a.duration);
|
||||||
|
return flattened;
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue