diff --git a/packages/kbn-test/package.json b/packages/kbn-test/package.json index 8c5358c82208..0cc54fa2a64c 100644 --- a/packages/kbn-test/package.json +++ b/packages/kbn-test/package.json @@ -15,7 +15,8 @@ "@kbn/dev-utils": "1.0.0", "@types/parse-link-header": "^1.0.0", "@types/strip-ansi": "^5.2.1", - "@types/xml2js": "^0.4.5" + "@types/xml2js": "^0.4.5", + "diff": "^4.0.1" }, "dependencies": { "chalk": "^2.4.2", diff --git a/packages/kbn-test/src/failed_tests_reporter/README.md b/packages/kbn-test/src/failed_tests_reporter/README.md index 1af309ba525b..20592ecd733b 100644 --- a/packages/kbn-test/src/failed_tests_reporter/README.md +++ b/packages/kbn-test/src/failed_tests_reporter/README.md @@ -12,10 +12,10 @@ copy(`wget "${Array.from($$('a[href$=".xml"]')).filter(a => a.innerText === 'Dow This copies a script to download the reports, which you should execute in the `test/junit` directory. -Next, run the CLI in `--dry-run` mode so that it doesn't actually communicate with Github. +Next, run the CLI in `--no-github-update` mode so that it doesn't actually communicate with Github and `--no-report-update` to prevent the script from mutating the reports on disk and instead log the updated report. ```sh -node scripts/report_failed_tests.js --verbose --dry-run --build-url foo +node scripts/report_failed_tests.js --verbose --no-github-update --no-report-update ``` -If you specify the `GITHUB_TOKEN` environment variable then `--dry-run` will execute read operations but still won't execute write operations. \ No newline at end of file +Unless you specify the `GITHUB_TOKEN` environment variable requests to read existing issues will use anonymous access which is limited to 60 requests per hour. \ No newline at end of file diff --git a/packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts b/packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts new file mode 100644 index 000000000000..02b6b5f06421 --- /dev/null +++ b/packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const Fs = jest.requireActual('fs'); + +export const FTR_REPORT = Fs.readFileSync(require.resolve('./ftr_report.xml'), 'utf8'); +export const JEST_REPORT = Fs.readFileSync(require.resolve('./jest_report.xml'), 'utf8'); +export const KARMA_REPORT = Fs.readFileSync(require.resolve('./karma_report.xml'), 'utf8'); +export const MOCHA_REPORT = Fs.readFileSync(require.resolve('./mocha_report.xml'), 'utf8'); diff --git a/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts b/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts new file mode 100644 index 000000000000..9e800e88bc9b --- /dev/null +++ b/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts @@ -0,0 +1,350 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; + +import { ToolingLog } from '@kbn/dev-utils'; +// @ts-ignore +import { createPatch } from 'diff'; + +// turns out Jest can't encode xml diffs in their JUnit reports... +expect.addSnapshotSerializer({ + test: v => typeof v === 'string' && (v.includes('<') || v.includes('>')), + print: v => + v + .replace(//g, '›') + .replace(/^\s+$/gm, ''), +}); + +jest.mock('fs', () => { + const realFs = jest.requireActual('fs'); + return { + readFile: realFs.read, + writeFile: (...args: any[]) => { + setTimeout(args[args.length - 1], 0); + }, + }; +}); + +import { FTR_REPORT, JEST_REPORT, MOCHA_REPORT, KARMA_REPORT } from './__fixtures__'; +import { parseTestReport } from './test_report'; +import { addMessagesToReport } from './add_messages_to_report'; + +beforeEach(() => { + jest.resetAllMocks(); +}); + +const log = new ToolingLog(); + +it('rewrites ftr reports with minimal changes', async () => { + const xml = await addMessagesToReport({ + report: await parseTestReport(FTR_REPORT), + messages: [ + { + name: 'maps app maps loaded from sample data ecommerce "before all" hook', + classname: + 'Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js', + message: 'foo bar', + }, + ], + log, + reportPath: Path.resolve(__dirname, './__fixtures__/ftr_report.xml'), + }); + + expect(createPatch('ftr.xml', FTR_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(` + Index: ftr.xml + =================================================================== + --- ftr.xml [object Object] + +++ ftr.xml + @@ -2,52 +2,56 @@ + ‹testsuites› + ‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"› + ‹testcase name="maps app maps loaded from sample data ecommerce "before all" hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"› + ‹system-out› + - ‹![CDATA[[00:00:00] │ + + [00:00:00] │ + [00:07:04] └-: maps app + ... + [00:15:02] │ + -]]› + + + ‹/system-out› + ‹failure› + - ‹![CDATA[Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~="layerTocActionsPanelToggleButtonRoad_Map_-_Bright"]) + + Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~="layerTocActionsPanelToggleButtonRoad_Map_-_Bright"]) + Wait timed out after 10055ms + at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17 + at process._tickCallback (internal/process/next_tick.js:68:7) + at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9) + - at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)]]› + - ‹/failure› + + at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13) + + + + + +Failed Tests Reporter: + + - foo bar + +‹/failure› + ‹/testcase› + ‹testcase name="maps app "after all" hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps" time="0.179"› + ‹system-out› + - ‹![CDATA[[00:00:00] │ + + [00:00:00] │ + [00:07:04] └-: maps app + ... + -]]› + + + ‹/system-out› + ‹failure› + - ‹![CDATA[{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used. + + { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used. + at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38) + at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12) + - at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }]]› + + at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' } + ‹/failure› + ‹/testcase› + ‹testcase name="InfraOps app feature controls infrastructure security global infrastructure all privileges shows infrastructure navlink" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/infra/feature_controls/infrastructure_security·ts"› + ‹system-out› + - ‹![CDATA[[00:00:00] │ + + [00:00:00] │ + [00:05:13] └-: InfraOps app + ... + -]]› + + + ‹/system-out› + ‹skipped/› + ‹/testcase› + ‹testcase name="machine learning anomaly detection saved search with lucene query job creation opens the advanced section" classname="Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts" time="6.040"› + - ‹system-out›‹![CDATA[[00:21:57] └-: machine learning...]]›‹/system-out› + - ‹failure›‹![CDATA[{ NoSuchSessionError: Tried to run command without establishing a connection + + ‹system-out›[00:21:57] └-: machine learning...‹/system-out› + + ‹failure›{ NoSuchSessionError: Tried to run command without establishing a connection + at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15) + at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13) + at Executor.execute (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:489:26) + - at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }]]›‹/failure› + + at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }‹/failure› + ‹/testcase› + ‹/testsuite› + -‹/testsuites› + +‹/testsuites› + \\ No newline at end of file + + `); +}); + +it('rewrites jest reports with minimal changes', async () => { + const xml = await addMessagesToReport({ + report: await parseTestReport(JEST_REPORT), + messages: [ + { + classname: 'X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp', + name: 'launcher can reconnect if process died', + message: 'foo bar', + }, + ], + log, + reportPath: Path.resolve(__dirname, './__fixtures__/jest_report.xml'), + }); + + expect(createPatch('jest.xml', JEST_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(` + Index: jest.xml + =================================================================== + --- jest.xml [object Object] + +++ jest.xml + @@ -3,13 +3,17 @@ + ‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"› + ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/› + ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/› + ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"› + - ‹failure› + - ‹![CDATA[TypeError: Cannot read property '0' of undefined + - at Object.‹anonymous›.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)]]› + - ‹/failure› + + ‹failure›‹![CDATA[ + + TypeError: Cannot read property '0' of undefined + + at Object.‹anonymous›.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10) + + + + + +Failed Tests Reporter: + + - foo bar + +]]›‹/failure› + ‹/testcase› + ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="passive launcher can start and end a process" time="0.435"/› + ‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="passive launcher should restart a process if a process died before connected" time="1.502"/› + ‹/testsuite› + -‹/testsuites› + +‹/testsuites› + \\ No newline at end of file + + `); +}); + +it('rewrites mocha reports with minimal changes', async () => { + const xml = await addMessagesToReport({ + report: await parseTestReport(MOCHA_REPORT), + messages: [ + { + name: 'code in multiple nodes "before all" hook', + classname: 'X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts', + message: 'foo bar', + }, + ], + log, + reportPath: Path.resolve(__dirname, './__fixtures__/mocha_report.xml'), + }); + + expect(createPatch('mocha.xml', MOCHA_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(` + Index: mocha.xml + =================================================================== + --- mocha.xml [object Object] + +++ mocha.xml + @@ -2,12 +2,12 @@ + ‹testsuites› + ‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"› + ‹testcase name="code in multiple nodes "before all" hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"› + ‹system-out› + - ‹![CDATA[]]› + + + ‹/system-out› + - ‹failure› + - ‹![CDATA[Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable + + ‹failure›‹![CDATA[ + + Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable + ‹html› + ‹head›‹title›503 Service Temporarily Unavailable‹/title›‹/head› + ‹body bgcolor="white"› + ‹center›‹h1›503 Service Temporarily Unavailable‹/h1›‹/center› + @@ -15,24 +15,28 @@ + ‹/body› + ‹/html› + + at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13) + - at process._tickCallback (internal/process/next_tick.js:68:7)]]› + - ‹/failure› + + at process._tickCallback (internal/process/next_tick.js:68:7) + + + + + +Failed Tests Reporter: + + - foo bar + +]]›‹/failure› + ‹/testcase› + ‹testcase name="code in multiple nodes "after all" hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.003"› + ‹system-out› + - ‹![CDATA[]]› + + + ‹/system-out› + ‹failure› + - ‹![CDATA[TypeError: Cannot read property 'shutdown' of undefined + + TypeError: Cannot read property 'shutdown' of undefined + at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23) + - at process.topLevelDomainCallback (domain.js:120:23)]]› + + at process.topLevelDomainCallback (domain.js:120:23) + ‹/failure› + ‹/testcase› + ‹testcase name="repository service test can not clone a repo by ssh without a key" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/repository_service·ts" time="0.005"› + ‹system-out› + - ‹![CDATA[]]› + + + ‹/system-out› + ‹/testcase› + ‹/testsuite› + -‹/testsuites› + +‹/testsuites› + \\ No newline at end of file + + `); +}); + +it('rewrites karma reports with minimal changes', async () => { + const xml = await addMessagesToReport({ + report: await parseTestReport(KARMA_REPORT), + messages: [ + { + name: + 'CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK', + classname: 'Browser Unit Tests.CoordinateMapsVisualizationTest', + message: 'foo bar', + }, + ], + log, + reportPath: Path.resolve(__dirname, './__fixtures__/karma_report.xml'), + }); + + expect(createPatch('karma.xml', KARMA_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(` + Index: karma.xml + =================================================================== + --- karma.xml [object Object] + +++ karma.xml + @@ -1,5 +1,5 @@ + -‹?xml version="1.0"?› + +‹?xml version="1.0" encoding="utf-8"?› + ‹testsuite name="Chrome 75.0.3770 (Mac OS X 10.14.5)" package="" timestamp="2019-07-02T19:53:21" id="0" hostname="spalger.lan" tests="648" errors="0" failures="4" time="1.759"› + ‹properties› + ‹property name="browser.fullName" value="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"/› + ‹/properties› + @@ -7,27 +7,31 @@ + ‹testcase name="Vis-Editor-Agg-Params plugin directive should hide custom label parameter" time="0" classname="Browser Unit Tests.Vis-Editor-Agg-Params plugin directive"› + ‹skipped/› + ‹/testcase› + ‹testcase name="CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK" time="0.265" classname="Browser Unit Tests.CoordinateMapsVisualizationTest"› + - ‹failure type=""›Error: expected 7069 to be below 64 + - at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11) + - at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8) + - at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15) + - at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60) + + ‹failure type=""›‹![CDATA[Error: expected 7069 to be below 64 + + at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11) + + at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8) + + at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15) + + at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60) + at tryCatch (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:62:40) + at Generator.invoke [as _invoke] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:288:22) + - at Generator.prototype.<computed> [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21) + - at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103) + - at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194) + -‹/failure› + + at Generator.prototype.‹computed› [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21) + + at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103) + + at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194) + + + + + +Failed Tests Reporter: + + - foo bar + +]]›‹/failure› + ‹/testcase› + ‹testcase name="CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should toggle to Heatmap OK" time="0.055" classname="Browser Unit Tests.CoordinateMapsVisualizationTest"/› + ‹testcase name="VegaParser._parseSchema should warn on vega-lite version too new to be supported" time="0.001" classname="Browser Unit Tests.VegaParser·_parseSchema"/› + ‹system-out› + - ‹![CDATA[Chrome 75.0.3770 (Mac OS X 10.14.5) LOG: 'ready to load tests for shard 1 of 4' + + Chrome 75.0.3770 (Mac OS X 10.14.5) LOG: 'ready to load tests for shard 1 of 4' + ,Chrome 75.0.3770 (Mac OS X 10.14.5) WARN: 'Unmatched GET to http://localhost:9876/api/interpreter/fns' + ... + + -]]› + + + ‹/system-out› + ‹system-err/› + -‹/testsuite› + +‹/testsuite› + \\ No newline at end of file + + `); +}); diff --git a/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts b/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts new file mode 100644 index 000000000000..f82e1ef1fc19 --- /dev/null +++ b/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; +import { promisify } from 'util'; + +import { ToolingLog } from '@kbn/dev-utils'; +import xml2js from 'xml2js'; + +import { TestReport, makeFailedTestCaseIter } from './test_report'; + +const writeAsync = promisify(Fs.writeFile); + +export interface Message { + classname: string; + name: string; + message: string; +} + +/** + * Mutate the report to include mentions of Github issues related to test failures, + * then write the updated report to disk + */ +export async function addMessagesToReport(options: { + log: ToolingLog; + report: TestReport; + messages: Message[]; + reportPath: string; + dryRun?: boolean; +}) { + const { log, report, messages, reportPath, dryRun } = options; + + for (const testCase of makeFailedTestCaseIter(report)) { + const { classname, name } = testCase.$; + const messageList = messages + .filter(u => u.classname === classname && u.name === name) + .reduce((acc, u) => `${acc}\n - ${u.message}`, ''); + + if (!messageList) { + continue; + } + + log.info(`${classname} - ${name}:${messageList}`); + const append = `\n\nFailed Tests Reporter:${messageList}\n`; + + if ( + testCase.failure[0] && + typeof testCase.failure[0] === 'object' && + typeof testCase.failure[0]._ === 'string' + ) { + testCase.failure[0]._ += append; + } else { + testCase.failure[0] = String(testCase.failure[0]) + append; + } + } + + const builder = new xml2js.Builder({ + cdata: true, + xmldec: { version: '1.0', encoding: 'utf-8' }, + }); + + const xml = builder + .buildObject(report) + .split('\n') + .map(line => (line.trim() === '' ? '' : line)) + .join('\n'); + + if (dryRun) { + log.info(`updated ${reportPath}\n${xml}`); + } else { + await writeAsync(reportPath, xml, 'utf8'); + } + return xml; +} diff --git a/packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts b/packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts index 1e0514a9b1cb..fe6e0bbc796e 100644 --- a/packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts +++ b/packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts @@ -17,14 +17,12 @@ * under the License. */ -import { ToolingLog } from '@kbn/dev-utils'; - import { getFailures } from './get_failures'; - -const log = new ToolingLog(); +import { parseTestReport } from './test_report'; +import { FTR_REPORT, JEST_REPORT, KARMA_REPORT, MOCHA_REPORT } from './__fixtures__'; it('discovers failures in ftr report', async () => { - const failures = await getFailures(log, require.resolve('./__fixtures__/ftr_report.xml')); + const failures = getFailures(await parseTestReport(FTR_REPORT)); expect(failures).toMatchInlineSnapshot(` Array [ Object { @@ -37,15 +35,39 @@ it('discovers failures in ftr report', async () => { at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9) at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13) ", + "likelyIrrelevant": false, "name": "maps app maps loaded from sample data ecommerce \\"before all\\" hook", "time": "154.378", }, + Object { + "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps", + "failure": " + { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used. + at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38) + at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12) + at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' } + ", + "likelyIrrelevant": true, + "name": "maps app \\"after all\\" hook", + "time": "0.179", + }, + Object { + "classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts", + "failure": "{ NoSuchSessionError: Tried to run command without establishing a connection + at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15) + at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13) + at Executor.execute (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:489:26) + at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }", + "likelyIrrelevant": true, + "name": "machine learning anomaly detection saved search with lucene query job creation opens the advanced section", + "time": "6.040", + }, ] `); }); it('discovers failures in jest report', async () => { - const failures = await getFailures(log, require.resolve('./__fixtures__/jest_report.xml')); + const failures = getFailures(await parseTestReport(JEST_REPORT)); expect(failures).toMatchInlineSnapshot(` Array [ Object { @@ -54,6 +76,7 @@ it('discovers failures in jest report', async () => { TypeError: Cannot read property '0' of undefined at Object..test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10) ", + "likelyIrrelevant": false, "name": "launcher can reconnect if process died", "time": "7.060", }, @@ -62,7 +85,7 @@ it('discovers failures in jest report', async () => { }); it('discovers failures in karma report', async () => { - const failures = await getFailures(log, require.resolve('./__fixtures__/karma_report.xml')); + const failures = getFailures(await parseTestReport(KARMA_REPORT)); expect(failures).toMatchInlineSnapshot(` Array [ Object { @@ -78,6 +101,7 @@ it('discovers failures in karma report', async () => { at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103) at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194) ", + "likelyIrrelevant": false, "name": "CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK", "time": "0.265", }, @@ -86,6 +110,39 @@ it('discovers failures in karma report', async () => { }); it('discovers failures in mocha report', async () => { - const failures = await getFailures(log, require.resolve('./__fixtures__/mocha_report.xml')); - expect(failures).toMatchInlineSnapshot(`Array []`); + const failures = getFailures(await parseTestReport(MOCHA_REPORT)); + expect(failures).toMatchInlineSnapshot(` + Array [ + Object { + "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts", + "failure": " + Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable + + 503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.13.7
+ + + + at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13) + at process._tickCallback (internal/process/next_tick.js:68:7) + ", + "likelyIrrelevant": true, + "name": "code in multiple nodes \\"before all\\" hook", + "time": "0.121", + }, + Object { + "classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts", + "failure": " + TypeError: Cannot read property 'shutdown' of undefined + at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23) + at process.topLevelDomainCallback (domain.js:120:23) + ", + "likelyIrrelevant": true, + "name": "code in multiple nodes \\"after all\\" hook", + "time": "0.003", + }, + ] + `); }); diff --git a/packages/kbn-test/src/failed_tests_reporter/get_failures.ts b/packages/kbn-test/src/failed_tests_reporter/get_failures.ts index 85eff8eb07f5..be058791f737 100644 --- a/packages/kbn-test/src/failed_tests_reporter/get_failures.ts +++ b/packages/kbn-test/src/failed_tests_reporter/get_failures.ts @@ -17,69 +17,16 @@ * under the License. */ -import { promisify } from 'util'; -import Fs from 'fs'; - -import xml2js from 'xml2js'; import stripAnsi from 'strip-ansi'; -import { ToolingLog } from '@kbn/dev-utils'; -type TestReport = - | { - testsuites: { - testsuite: TestSuite[]; - }; - } - | { - testsuite: TestSuite; - }; +import { FailedTestCase, TestReport, makeFailedTestCaseIter } from './test_report'; -interface TestSuite { - $: { - /* ISO8601 timetamp when test suite ran */ - timestamp: string; - /* number of second this tests suite took */ - time: string; - /* number of tests as a string */ - tests: string; - /* number of failed tests as a string */ - failures: string; - /* number of skipped tests as a string */ - skipped: string; - }; - testcase: TestCase[]; -} - -interface TestCase { - $: { - /* unique test name */ - name: string; - /* somewhat human readable combination of test name and file */ - classname: string; - /* number of seconds this test took */ - time: string; - }; - /* contents of system-out elements */ - 'system-out'?: string[]; - /* contents of failure elements */ - failure?: Array; - /* contents of skipped elements */ - skipped?: string[]; -} - -export type TestFailure = TestCase['$'] & { +export type TestFailure = FailedTestCase['$'] & { failure: string; + likelyIrrelevant: boolean; }; -const readAsync = promisify(Fs.readFile); - -const indent = (text: string) => - ` ${text - .split('\n') - .map(l => ` ${l}`) - .join('\n')}`; - -const getFailureText = (failure: NonNullable) => { +const getFailureText = (failure: FailedTestCase['failure']) => { const [failureNode] = failure; if (failureNode && typeof failureNode === 'object' && typeof failureNode._ === 'string') { @@ -89,7 +36,7 @@ const getFailureText = (failure: NonNullable) => { return stripAnsi(String(failureNode)); }; -const isLikelyIrrelevant = ({ name, failure }: TestFailure) => { +const isLikelyIrrelevant = (name: string, failure: string) => { if ( failure.includes('NoSuchSessionError: This driver instance does not have a valid session ID') || failure.includes('NoSuchSessionError: Tried to run command without establishing a connection') @@ -118,47 +65,25 @@ const isLikelyIrrelevant = ({ name, failure }: TestFailure) => { if (failure.includes('Unable to fetch Kibana status API response from Kibana')) { return true; } + + return false; }; -export async function getFailures(log: ToolingLog, testReportPath: string) { - const xml = await readAsync(testReportPath, 'utf8'); - - // Parses junit XML files - const report: TestReport = await xml2js.parseStringPromise(xml); - - // Grab the failures. Reporters may report multiple testsuites in a single file. - const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite]; - +export function getFailures(report: TestReport) { const failures: TestFailure[] = []; - for (const testSuite of testSuites) { - for (const testCase of testSuite.testcase) { - const { failure } = testCase; - if (!failure) { - continue; - } + for (const testCase of makeFailedTestCaseIter(report)) { + const failure = getFailureText(testCase.failure); + const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure); + failures.push({ // unwrap xml weirdness - const failureCase: TestFailure = { - ...testCase.$, - // Strip ANSI color characters - failure: getFailureText(failure), - }; - - if (isLikelyIrrelevant(failureCase)) { - log.warning( - `Ignoring likely irrelevant failure: ${failureCase.classname} - ${ - failureCase.name - }\n${indent(failureCase.failure)}` - ); - continue; - } - - failures.push(failureCase); - } + ...testCase.$, + // Strip ANSI color characters + failure, + likelyIrrelevant, + }); } - log.info(`Found ${failures.length} test failures`); - return failures; } diff --git a/packages/kbn-test/src/failed_tests_reporter/github_api.ts b/packages/kbn-test/src/failed_tests_reporter/github_api.ts index 46278044c840..d8a952bee42e 100644 --- a/packages/kbn-test/src/failed_tests_reporter/github_api.ts +++ b/packages/kbn-test/src/failed_tests_reporter/github_api.ts @@ -19,7 +19,7 @@ import Url from 'url'; -import Axios, { AxiosRequestConfig } from 'axios'; +import Axios, { AxiosRequestConfig, AxiosInstance } from 'axios'; import parseLinkHeader from 'parse-link-header'; import { ToolingLog, isAxiosResponseError, isAxiosRequestError } from '@kbn/dev-utils'; @@ -40,25 +40,34 @@ type RequestOptions = AxiosRequestConfig & { }; export class GithubApi { - private readonly x = Axios.create({ - headers: { - Authorization: `token ${this.token}`, - 'User-Agent': 'elastic/kibana#failed_test_reporter', - }, - }); + private readonly log: ToolingLog; + private readonly token: string | undefined; + private readonly dryRun: boolean; + private readonly x: AxiosInstance; /** * Create a GithubApi helper object, if token is undefined requests won't be * sent, but will instead be logged. */ - constructor( - private readonly log: ToolingLog, - private readonly token: string | undefined, - private readonly dryRun: boolean - ) { - if (!token && !dryRun) { + constructor(options: { + log: GithubApi['log']; + token: GithubApi['token']; + dryRun: GithubApi['dryRun']; + }) { + this.log = options.log; + this.token = options.token; + this.dryRun = options.dryRun; + + if (!this.token && !this.dryRun) { throw new TypeError('token parameter is required'); } + + this.x = Axios.create({ + headers: { + ...(this.token ? { Authorization: `token ${this.token}` } : {}), + 'User-Agent': 'elastic/kibana#failed_test_reporter', + }, + }); } private failedTestIssuesPageCache: { diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts b/packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts index 0e9f8db587cb..ef6ab3c51ab1 100644 --- a/packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts +++ b/packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts @@ -18,19 +18,14 @@ */ import dedent from 'dedent'; -import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/dev-utils'; -import { createFailureIssue, updatedFailureIssue } from './report_failure'; +import { createFailureIssue, updateFailureIssue } from './report_failure'; jest.mock('./github_api'); const { GithubApi } = jest.requireMock('./github_api'); describe('createFailureIssue()', () => { it('creates new github issue with failure text, link to issue, and valid metadata', async () => { - const log = new ToolingLog(); - const writer = new ToolingLogCollectingWriter(); - log.setWriters([writer]); - const api = new GithubApi(); await createFailureIssue( @@ -40,8 +35,8 @@ describe('createFailureIssue()', () => { failure: 'this is the failure text', name: 'test name', time: '2018-01-01T01:00:00Z', + likelyIrrelevant: false, }, - log, api ); @@ -72,23 +67,14 @@ describe('createFailureIssue()', () => { ], } `); - expect(writer.messages).toMatchInlineSnapshot(` - Array [ - " info Created issue undefined", - ] - `); }); }); -describe('updatedFailureIssue()', () => { +describe('updateFailureIssue()', () => { it('increments failure count and adds new comment to issue', async () => { - const log = new ToolingLog(); - const writer = new ToolingLogCollectingWriter(); - log.setWriters([writer]); - const api = new GithubApi(); - await updatedFailureIssue( + await updateFailureIssue( 'https://build-url', { html_url: 'https://github.com/issues/1234', @@ -101,7 +87,6 @@ describe('updatedFailureIssue()', () => { " `, }, - log, api ); @@ -139,10 +124,5 @@ describe('updatedFailureIssue()', () => { ], } `); - expect(writer.messages).toMatchInlineSnapshot(` - Array [ - " info Updated issue https://github.com/issues/1234, failCount: 11", - ] - `); }); }); diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failure.ts b/packages/kbn-test/src/failed_tests_reporter/report_failure.ts index afb46c429c7f..97e9d517576f 100644 --- a/packages/kbn-test/src/failed_tests_reporter/report_failure.ts +++ b/packages/kbn-test/src/failed_tests_reporter/report_failure.ts @@ -17,18 +17,11 @@ * under the License. */ -import { ToolingLog } from '@kbn/dev-utils'; - import { TestFailure } from './get_failures'; import { GithubIssue, GithubApi } from './github_api'; import { getIssueMetadata, updateIssueMetadata } from './issue_metadata'; -export async function createFailureIssue( - buildUrl: string, - failure: TestFailure, - log: ToolingLog, - api: GithubApi -) { +export async function createFailureIssue(buildUrl: string, failure: TestFailure, api: GithubApi) { const title = `Failing test: ${failure.classname} - ${failure.name}`; const body = updateIssueMetadata( @@ -48,16 +41,10 @@ export async function createFailureIssue( } ); - const newIssueUrl = await api.createIssue(title, body, ['failed-test']); - log.info(`Created issue ${newIssueUrl}`); + return await api.createIssue(title, body, ['failed-test']); } -export async function updatedFailureIssue( - buildUrl: string, - issue: GithubIssue, - log: ToolingLog, - api: GithubApi -) { +export async function updateFailureIssue(buildUrl: string, issue: GithubIssue, api: GithubApi) { // Increment failCount const newCount = getIssueMetadata(issue.body, 'test.failCount', 0) + 1; const newBody = updateIssueMetadata(issue.body, { @@ -67,5 +54,5 @@ export async function updatedFailureIssue( await api.editIssueBodyAndEnsureOpen(issue.number, newBody); await api.addIssueComment(issue.number, `New failure: [Jenkins Build](${buildUrl})`); - log.info(`Updated issue ${issue.html_url}, failCount: ${newCount}`); + return newCount; } diff --git a/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts b/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts index 0eea1d32e5c2..b3c2a8dc338d 100644 --- a/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts +++ b/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts @@ -22,19 +22,22 @@ import globby from 'globby'; import { getFailures } from './get_failures'; import { GithubApi } from './github_api'; -import { updatedFailureIssue, createFailureIssue } from './report_failure'; +import { updateFailureIssue, createFailureIssue } from './report_failure'; import { getIssueMetadata } from './issue_metadata'; +import { readTestReport } from './test_report'; +import { addMessagesToReport, Message } from './add_messages_to_report'; export function runFailedTestsReporterCli() { run( async ({ log, flags }) => { - const buildUrl = flags['build-url']; - if (typeof buildUrl !== 'string' || !buildUrl) { - throw createFlagError('Missing --build-url or process.env.BUILD_URL'); + let updateGithub = flags['github-update']; + if (updateGithub && !process.env.GITHUB_TOKEN) { + throw createFailError( + 'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag' + ); } - const dryRun = !!flags['dry-run']; - if (!dryRun) { + if (updateGithub) { // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//); const branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH; @@ -48,26 +51,43 @@ export function runFailedTestsReporterCli() { const isMasterOrVersion = branch.match(/^(origin\/){0,1}master$/) || branch.match(/^(origin\/){0,1}\d+\.(x|\d+)$/); if (!isMasterOrVersion || isPr) { - throw createFailError('Failure issues only created on master/version branch jobs', { - exitCode: 0, - }); - } - - if (!process.env.GITHUB_TOKEN) { - throw createFailError( - 'GITHUB_TOKEN environment variable must be set, otherwise use --dry-run flag' - ); + log.info('Failure issues only created on master/version branch jobs'); + updateGithub = false; } } - const githubApi = new GithubApi(log, process.env.GITHUB_TOKEN, dryRun); + const githubApi = new GithubApi({ + log, + token: process.env.GITHUB_TOKEN, + dryRun: !updateGithub, + }); + + const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl'); + if (typeof buildUrl !== 'string' || !buildUrl) { + throw createFlagError('Missing --build-url or process.env.BUILD_URL'); + } + const reportPaths = await globby(['target/junit/**/*.xml'], { cwd: REPO_ROOT, absolute: true, }); for (const reportPath of reportPaths) { - for (const failure of await getFailures(log, reportPath)) { + const report = await readTestReport(reportPath); + const messages: Message[] = []; + + for (const failure of await getFailures(report)) { + if (failure.likelyIrrelevant) { + messages.push({ + classname: failure.classname, + name: failure.name, + message: + 'Failure is likely irrelevant' + + (updateGithub ? ', so an issue was not created or updated' : ''), + }); + continue; + } + const existingIssue = await githubApi.findFailedTestIssue( i => getIssueMetadata(i.body, 'test.class') === failure.classname && @@ -75,23 +95,57 @@ export function runFailedTestsReporterCli() { ); if (existingIssue) { - await updatedFailureIssue(buildUrl, existingIssue, log, githubApi); - } else { - await createFailureIssue(buildUrl, failure, log, githubApi); + const newFailureCount = await updateFailureIssue(buildUrl, existingIssue, githubApi); + const url = existingIssue.html_url; + const message = + `Test has failed ${newFailureCount - 1} times on tracked branches: ${url}` + + (updateGithub + ? `. Updated existing issue: ${url} (fail count: ${newFailureCount})` + : ''); + + messages.push({ + classname: failure.classname, + name: failure.name, + message, + }); + continue; } + + const newIssueUrl = await createFailureIssue(buildUrl, failure, githubApi); + const message = + `Test has not failed recently on tracked branches` + + (updateGithub ? `Created new issue: ${newIssueUrl}` : ''); + + messages.push({ + classname: failure.classname, + name: failure.name, + message, + }); } + + // mutates report to include messages and writes updated report to disk + await addMessagesToReport({ + report, + messages, + log, + reportPath, + dryRun: !flags['report-update'], + }); } }, { description: `a cli that opens issues or updates existing issues based on junit reports`, flags: { - boolean: ['dry-run'], + boolean: ['github-update', 'report-update'], string: ['build-url'], default: { + 'github-update': true, + 'report-update': true, 'build-url': process.env.BUILD_URL, }, help: ` - --dry-run Execute the CLI without contacting Github + --no-github-update Execute the CLI without writing to Github + --no-report-update Execute the CLI without writing to the JUnit reports --build-url URL of the failed build, defaults to process.env.BUILD_URL `, }, diff --git a/packages/kbn-test/src/failed_tests_reporter/test_report.ts b/packages/kbn-test/src/failed_tests_reporter/test_report.ts new file mode 100644 index 000000000000..644a4cc9fd5a --- /dev/null +++ b/packages/kbn-test/src/failed_tests_reporter/test_report.ts @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; +import { promisify } from 'util'; + +import xml2js from 'xml2js'; + +const readAsync = promisify(Fs.readFile); + +export type TestReport = + | { + testsuites: { + testsuite: TestSuite[]; + }; + } + | { + testsuite: TestSuite; + }; + +export interface TestSuite { + $: { + /* ISO8601 timetamp when test suite ran */ + timestamp: string; + /* number of second this tests suite took */ + time: string; + /* number of tests as a string */ + tests: string; + /* number of failed tests as a string */ + failures: string; + /* number of skipped tests as a string */ + skipped: string; + }; + testcase: TestCase[]; +} + +export interface TestCase { + $: { + /* unique test name */ + name: string; + /* somewhat human readable combination of test name and file */ + classname: string; + /* number of seconds this test took */ + time: string; + }; + /* contents of system-out elements */ + 'system-out'?: string[]; + /* contents of failure elements */ + failure?: Array; + /* contents of skipped elements */ + skipped?: string[]; +} + +export interface FailedTestCase extends TestCase { + failure: Array; +} + +/** + * Parse JUnit XML Files + */ +export async function parseTestReport(xml: string): Promise { + return await xml2js.parseStringPromise(xml); +} + +export async function readTestReport(testReportPath: string) { + return await parseTestReport(await readAsync(testReportPath, 'utf8')); +} + +export function* makeFailedTestCaseIter(report: TestReport) { + // Grab the failures. Reporters may report multiple testsuites in a single file. + const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite]; + + for (const testSuite of testSuites) { + for (const testCase of testSuite.testcase) { + const { failure } = testCase; + + if (!failure) { + continue; + } + + yield testCase as FailedTestCase; + } + } +} diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 0058f21f2356..5b3cd071316e 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -30,6 +30,10 @@ def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) { uploadAllGcsArtifacts(name) } + catchError { + runErrorReporter() + } + catchError { runbld.junit() } @@ -37,10 +41,6 @@ def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) { catchError { publishJunit() } - - catchError { - runErrorReporter() - } } } } @@ -103,10 +103,10 @@ def legacyJobRunner(name) { uploadAllGcsArtifacts(name) } catchError { - publishJunit() + runErrorReporter() } catchError { - runErrorReporter() + publishJunit() } } }