Co-authored-by: Brian Seeders <brian.seeders@elastic.co>
This commit is contained in:
parent
2ec7d50c6d
commit
6ba24fc82a
|
@ -91,13 +91,5 @@ steps:
|
|||
- wait: ~
|
||||
continue_on_failure: true
|
||||
|
||||
- plugins:
|
||||
- junit-annotate#v1.9.0:
|
||||
artifacts: target/junit/**/*.xml
|
||||
job-uuid-file-pattern: '-bk__(.*).xml'
|
||||
|
||||
- wait: ~
|
||||
continue_on_failure: true
|
||||
|
||||
- command: .buildkite/scripts/lifecycle/post_build.sh
|
||||
label: Post-Build
|
||||
|
|
|
@ -159,13 +159,5 @@ steps:
|
|||
- wait: ~
|
||||
continue_on_failure: true
|
||||
|
||||
- plugins:
|
||||
- junit-annotate#v1.9.0:
|
||||
artifacts: target/junit/**/*.xml
|
||||
job-uuid-file-pattern: '-bk__(.*).xml'
|
||||
|
||||
- wait: ~
|
||||
continue_on_failure: true
|
||||
|
||||
- command: .buildkite/scripts/lifecycle/post_build.sh
|
||||
label: Post-Build
|
||||
|
|
14
.buildkite/scripts/lifecycle/annotate_test_failures.js
Normal file
14
.buildkite/scripts/lifecycle/annotate_test_failures.js
Normal file
|
@ -0,0 +1,14 @@
|
|||
const { TestFailures } = require('kibana-buildkite-library');
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
await TestFailures.annotateTestFailures();
|
||||
} catch (ex) {
|
||||
console.error('Annotate test failures error', ex.message);
|
||||
if (ex.response) {
|
||||
console.error('HTTP Error Response Status', ex.response.status);
|
||||
console.error('HTTP Error Response Body', ex.response.data);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
|
@ -23,4 +23,9 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
|
|||
buildkite-agent artifact upload '.es/**/*.hprof'
|
||||
|
||||
node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
|
||||
|
||||
if [[ -d 'target/test_failures' ]]; then
|
||||
buildkite-agent artifact upload 'target/test_failures/**/*'
|
||||
node .buildkite/scripts/lifecycle/annotate_test_failures.js
|
||||
fi
|
||||
fi
|
||||
|
|
|
@ -50,6 +50,7 @@ RUNTIME_DEPS = [
|
|||
"@npm//exit-hook",
|
||||
"@npm//form-data",
|
||||
"@npm//globby",
|
||||
"@npm//he",
|
||||
"@npm//history",
|
||||
"@npm//jest",
|
||||
"@npm//jest-cli",
|
||||
|
@ -85,6 +86,7 @@ TYPES_DEPS = [
|
|||
"@npm//xmlbuilder",
|
||||
"@npm//@types/chance",
|
||||
"@npm//@types/enzyme",
|
||||
"@npm//@types/he",
|
||||
"@npm//@types/history",
|
||||
"@npm//@types/jest",
|
||||
"@npm//@types/joi",
|
||||
|
|
|
@ -0,0 +1,138 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs';
|
||||
import { join, basename, resolve } from 'path';
|
||||
|
||||
import { ToolingLog } from '@kbn/dev-utils';
|
||||
import { REPO_ROOT } from '@kbn/utils';
|
||||
import { escape } from 'he';
|
||||
|
||||
import { TestFailure } from './get_failures';
|
||||
|
||||
const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => {
|
||||
const files = readdirSync(dirPath);
|
||||
|
||||
for (const file of files) {
|
||||
if (statSync(join(dirPath, file)).isDirectory()) {
|
||||
if (file.match(/node_modules/)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
allScreenshots = findScreenshots(join(dirPath, file), allScreenshots);
|
||||
} else {
|
||||
const fullPath = join(dirPath, file);
|
||||
if (fullPath.match(/screenshots\/failure\/.+\.png$/)) {
|
||||
allScreenshots.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allScreenshots;
|
||||
};
|
||||
|
||||
export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) {
|
||||
if (!failures?.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
let screenshots: string[];
|
||||
try {
|
||||
screenshots = [
|
||||
...findScreenshots(join(REPO_ROOT, 'test', 'functional')),
|
||||
...findScreenshots(join(REPO_ROOT, 'x-pack', 'test', 'functional')),
|
||||
];
|
||||
} catch (e) {
|
||||
log.error(e as Error);
|
||||
screenshots = [];
|
||||
}
|
||||
|
||||
const screenshotsByName: Record<string, string> = {};
|
||||
for (const screenshot of screenshots) {
|
||||
const [name] = basename(screenshot).split('.');
|
||||
screenshotsByName[name] = screenshot;
|
||||
}
|
||||
|
||||
// Jest could, in theory, fail 1000s of tests and write 1000s of failures
|
||||
// So let's just write files for the first 20
|
||||
for (const failure of failures.slice(0, 20)) {
|
||||
const hash = createHash('md5').update(failure.name).digest('hex');
|
||||
const filenameBase = `${
|
||||
process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : ''
|
||||
}${hash}`;
|
||||
const dir = join('target', 'test_failures');
|
||||
|
||||
const failureLog = [
|
||||
['Test:', '-----', failure.classname, failure.name, ''],
|
||||
['Failure:', '--------', failure.failure],
|
||||
failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [],
|
||||
]
|
||||
.flat()
|
||||
.join('\n');
|
||||
|
||||
const failureJSON = JSON.stringify(
|
||||
{
|
||||
...failure,
|
||||
hash,
|
||||
buildId: process.env.BUJILDKITE_BUILD_ID || '',
|
||||
jobId: process.env.BUILDKITE_JOB_ID || '',
|
||||
url: process.env.BUILDKITE_BUILD_URL || '',
|
||||
jobName: process.env.BUILDKITE_LABEL
|
||||
? `${process.env.BUILDKITE_LABEL}${
|
||||
process.env.BUILDKITE_PARALLEL_JOB ? ` #${process.env.BUILDKITE_PARALLEL_JOB}` : ''
|
||||
}`
|
||||
: '',
|
||||
},
|
||||
null,
|
||||
2
|
||||
);
|
||||
|
||||
let screenshot = '';
|
||||
const screenshotName = `${failure.name.replace(/([^ a-zA-Z0-9-]+)/g, '_')}`;
|
||||
if (screenshotsByName[screenshotName]) {
|
||||
try {
|
||||
screenshot = readFileSync(screenshotsByName[screenshotName]).toString('base64');
|
||||
} catch (e) {
|
||||
log.error(e as Error);
|
||||
}
|
||||
}
|
||||
|
||||
const screenshotHtml = screenshot
|
||||
? `<img class="screenshot img-fluid img-thumbnail" src="data:image/png;base64,${screenshot}" />`
|
||||
: '';
|
||||
|
||||
const failureHTML = readFileSync(
|
||||
resolve(
|
||||
REPO_ROOT,
|
||||
'packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html'
|
||||
)
|
||||
)
|
||||
.toString()
|
||||
.replace('$TITLE', escape(failure.name))
|
||||
.replace(
|
||||
'$MAIN',
|
||||
`
|
||||
${failure.classname
|
||||
.split('.')
|
||||
.map((part) => `<h5>${escape(part.replace('·', '.'))}</h5>`)
|
||||
.join('')}
|
||||
<hr />
|
||||
<p><strong>${escape(failure.name)}</strong></p>
|
||||
<pre>${escape(failure.failure)}</pre>
|
||||
${screenshotHtml}
|
||||
<pre>${escape(failure['system-out'] || '')}</pre>
|
||||
`
|
||||
);
|
||||
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, `${filenameBase}.log`), failureLog, 'utf8');
|
||||
writeFileSync(join(dir, `${filenameBase}.html`), failureHTML, 'utf8');
|
||||
writeFileSync(join(dir, `${filenameBase}.json`), failureJSON, 'utf8');
|
||||
}
|
||||
}
|
|
@ -0,0 +1,41 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link
|
||||
href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.1/dist/css/bootstrap.min.css"
|
||||
rel="stylesheet"
|
||||
integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU"
|
||||
crossorigin="anonymous"
|
||||
/>
|
||||
<style type="text/css">
|
||||
pre {
|
||||
font-size: 0.75em !important;
|
||||
}
|
||||
|
||||
img.screenshot {
|
||||
cursor: pointer;
|
||||
height: 200px;
|
||||
margin: 5px 0;
|
||||
}
|
||||
|
||||
img.screenshot.expanded {
|
||||
height: auto;
|
||||
}
|
||||
</style>
|
||||
<title>$TITLE</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="col-lg-10 mx-auto p-3 py-md-5">
|
||||
<main>$MAIN</main>
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
for (const img of document.getElementsByTagName('img')) {
|
||||
img.addEventListener('click', () => {
|
||||
img.classList.toggle('expanded');
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
|
@ -21,6 +21,7 @@ import { readTestReport } from './test_report';
|
|||
import { addMessagesToReport } from './add_messages_to_report';
|
||||
import { getReportMessageIter } from './report_metadata';
|
||||
import { reportFailuresToEs } from './report_failures_to_es';
|
||||
import { reportFailuresToFile } from './report_failures_to_file';
|
||||
|
||||
const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
|
||||
|
||||
|
@ -98,6 +99,8 @@ export function runFailedTestsReporterCli() {
|
|||
const messages = Array.from(getReportMessageIter(report));
|
||||
const failures = await getFailures(report);
|
||||
|
||||
reportFailuresToFile(log, failures);
|
||||
|
||||
if (indexInEs) {
|
||||
await reportFailuresToEs(log, failures);
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue