From 6ba24fc82add26d42074fc4be5fd9a7dc83266c2 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 30 Sep 2021 20:59:17 -0400 Subject: [PATCH] [buildkite] Improve failed test experience (#113483) (#113580) Co-authored-by: Brian Seeders --- .buildkite/pipelines/es_snapshots/verify.yml | 8 - .buildkite/pipelines/hourly.yml | 8 - .../lifecycle/annotate_test_failures.js | 14 ++ .buildkite/scripts/lifecycle/post_command.sh | 5 + packages/kbn-test/BUILD.bazel | 2 + .../report_failures_to_file.ts | 138 ++++++++++++++++++ ...report_failures_to_file_html_template.html | 41 ++++++ .../run_failed_tests_reporter_cli.ts | 3 + 8 files changed, 203 insertions(+), 16 deletions(-) create mode 100644 .buildkite/scripts/lifecycle/annotate_test_failures.js create mode 100644 packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts create mode 100644 packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html diff --git a/.buildkite/pipelines/es_snapshots/verify.yml b/.buildkite/pipelines/es_snapshots/verify.yml index 9af2e938db49..61212e1fcf0a 100755 --- a/.buildkite/pipelines/es_snapshots/verify.yml +++ b/.buildkite/pipelines/es_snapshots/verify.yml @@ -91,13 +91,5 @@ steps: - wait: ~ continue_on_failure: true - - plugins: - - junit-annotate#v1.9.0: - artifacts: target/junit/**/*.xml - job-uuid-file-pattern: '-bk__(.*).xml' - - - wait: ~ - continue_on_failure: true - - command: .buildkite/scripts/lifecycle/post_build.sh label: Post-Build diff --git a/.buildkite/pipelines/hourly.yml b/.buildkite/pipelines/hourly.yml index 89541023be8e..279c8cf96bfe 100644 --- a/.buildkite/pipelines/hourly.yml +++ b/.buildkite/pipelines/hourly.yml @@ -159,13 +159,5 @@ steps: - wait: ~ continue_on_failure: true - - plugins: - - junit-annotate#v1.9.0: - artifacts: target/junit/**/*.xml - job-uuid-file-pattern: '-bk__(.*).xml' - - - wait: ~ - continue_on_failure: true - - command: .buildkite/scripts/lifecycle/post_build.sh label: Post-Build diff --git a/.buildkite/scripts/lifecycle/annotate_test_failures.js b/.buildkite/scripts/lifecycle/annotate_test_failures.js new file mode 100644 index 000000000000..caf1e08c2bb4 --- /dev/null +++ b/.buildkite/scripts/lifecycle/annotate_test_failures.js @@ -0,0 +1,14 @@ +const { TestFailures } = require('kibana-buildkite-library'); + +(async () => { + try { + await TestFailures.annotateTestFailures(); + } catch (ex) { + console.error('Annotate test failures error', ex.message); + if (ex.response) { + console.error('HTTP Error Response Status', ex.response.status); + console.error('HTTP Error Response Body', ex.response.data); + } + process.exit(1); + } +})(); diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index 23f44a586e97..0f98035f9f82 100755 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -23,4 +23,9 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then buildkite-agent artifact upload '.es/**/*.hprof' node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml' + + if [[ -d 'target/test_failures' ]]; then + buildkite-agent artifact upload 'target/test_failures/**/*' + node .buildkite/scripts/lifecycle/annotate_test_failures.js + fi fi diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index 36e81df6d8c3..efaf01f7137d 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -50,6 +50,7 @@ RUNTIME_DEPS = [ "@npm//exit-hook", "@npm//form-data", "@npm//globby", + "@npm//he", "@npm//history", "@npm//jest", "@npm//jest-cli", @@ -85,6 +86,7 @@ TYPES_DEPS = [ "@npm//xmlbuilder", "@npm//@types/chance", "@npm//@types/enzyme", + "@npm//@types/he", "@npm//@types/history", "@npm//@types/jest", "@npm//@types/joi", diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts b/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts new file mode 100644 index 000000000000..aca2e6838fae --- /dev/null +++ b/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createHash } from 'crypto'; +import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs'; +import { join, basename, resolve } from 'path'; + +import { ToolingLog } from '@kbn/dev-utils'; +import { REPO_ROOT } from '@kbn/utils'; +import { escape } from 'he'; + +import { TestFailure } from './get_failures'; + +const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => { + const files = readdirSync(dirPath); + + for (const file of files) { + if (statSync(join(dirPath, file)).isDirectory()) { + if (file.match(/node_modules/)) { + continue; + } + + allScreenshots = findScreenshots(join(dirPath, file), allScreenshots); + } else { + const fullPath = join(dirPath, file); + if (fullPath.match(/screenshots\/failure\/.+\.png$/)) { + allScreenshots.push(fullPath); + } + } + } + + return allScreenshots; +}; + +export function reportFailuresToFile(log: ToolingLog, failures: TestFailure[]) { + if (!failures?.length) { + return; + } + + let screenshots: string[]; + try { + screenshots = [ + ...findScreenshots(join(REPO_ROOT, 'test', 'functional')), + ...findScreenshots(join(REPO_ROOT, 'x-pack', 'test', 'functional')), + ]; + } catch (e) { + log.error(e as Error); + screenshots = []; + } + + const screenshotsByName: Record = {}; + for (const screenshot of screenshots) { + const [name] = basename(screenshot).split('.'); + screenshotsByName[name] = screenshot; + } + + // Jest could, in theory, fail 1000s of tests and write 1000s of failures + // So let's just write files for the first 20 + for (const failure of failures.slice(0, 20)) { + const hash = createHash('md5').update(failure.name).digest('hex'); + const filenameBase = `${ + process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : '' + }${hash}`; + const dir = join('target', 'test_failures'); + + const failureLog = [ + ['Test:', '-----', failure.classname, failure.name, ''], + ['Failure:', '--------', failure.failure], + failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [], + ] + .flat() + .join('\n'); + + const failureJSON = JSON.stringify( + { + ...failure, + hash, + buildId: process.env.BUJILDKITE_BUILD_ID || '', + jobId: process.env.BUILDKITE_JOB_ID || '', + url: process.env.BUILDKITE_BUILD_URL || '', + jobName: process.env.BUILDKITE_LABEL + ? `${process.env.BUILDKITE_LABEL}${ + process.env.BUILDKITE_PARALLEL_JOB ? ` #${process.env.BUILDKITE_PARALLEL_JOB}` : '' + }` + : '', + }, + null, + 2 + ); + + let screenshot = ''; + const screenshotName = `${failure.name.replace(/([^ a-zA-Z0-9-]+)/g, '_')}`; + if (screenshotsByName[screenshotName]) { + try { + screenshot = readFileSync(screenshotsByName[screenshotName]).toString('base64'); + } catch (e) { + log.error(e as Error); + } + } + + const screenshotHtml = screenshot + ? `` + : ''; + + const failureHTML = readFileSync( + resolve( + REPO_ROOT, + 'packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html' + ) + ) + .toString() + .replace('$TITLE', escape(failure.name)) + .replace( + '$MAIN', + ` + ${failure.classname + .split('.') + .map((part) => `
${escape(part.replace('ยท', '.'))}
`) + .join('')} +
+

${escape(failure.name)}

+ 
${escape(failure.failure)}
+ ${screenshotHtml} + 
${escape(failure['system-out'] || '')}
+ ` + ); + + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, `${filenameBase}.log`), failureLog, 'utf8'); + writeFileSync(join(dir, `${filenameBase}.html`), failureHTML, 'utf8'); + writeFileSync(join(dir, `${filenameBase}.json`), failureJSON, 'utf8'); + } +} diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html b/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html new file mode 100644 index 000000000000..485a2c2d4eb3 --- /dev/null +++ b/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html @@ -0,0 +1,41 @@ + + + + + + + + $TITLE + + +
+
$MAIN
+
+ + + diff --git a/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts b/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts index 0129614fe658..6c88b7408b62 100644 --- a/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts +++ b/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts @@ -21,6 +21,7 @@ import { readTestReport } from './test_report'; import { addMessagesToReport } from './add_messages_to_report'; import { getReportMessageIter } from './report_metadata'; import { reportFailuresToEs } from './report_failures_to_es'; +import { reportFailuresToFile } from './report_failures_to_file'; const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')]; @@ -98,6 +99,8 @@ export function runFailedTestsReporterCli() { const messages = Array.from(getReportMessageIter(report)); const failures = await getFailures(report); + reportFailuresToFile(log, failures); + if (indexInEs) { await reportFailuresToEs(log, failures); }