maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[7.x] [failed_tests_cli] update reports with links to github i… (#52303)

Browse source 
* [failed_tests_cli] update reports with links to github issues (#52048)

* [failed_tests_cli] update reports with links to github issues

* reorder test report hooks so that published Junit includes modified reports

* force failures and enable dry-run mode for debugging

* auto-switch to --dry-run when running in non-tracked branches/prs

* add --skip-junit-update flag to skip mutating the reports

* remove comma after URL to support auto-linking in Jenkins

* Revert "force failures and enable dry-run mode for debugging"

This reverts commit ac0c287a3f.

* fix method call

* extend TestResult to include relevence flag rather than wrapping

* fix createFailureIssue() tests

* make report messages more consistent, append when not dry-run

* rename module

* update snapshots to not contain valid xml

* don't send authorization header if no token defined

* merge with master modified fixtures

* [ci/reportFailures] --dry-run is overloaded, split it up (#52314)

* [ci/reportFailures] --dry-run is overloaded, split it up

* force some failures to verify the fix

* Revert "force some failures to verify the fix"

This reverts commit cf2a58e139.

* update readme to mention new flags

* remove unnecessary commas

(cherry picked from commit 8e8571bae0)
This commit is contained in:
Spencer 2019-12-05 14:43:33 -07:00 committed by GitHub
parent 330dde2ac1
commit e36afbc8d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 765 additions and 187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/kbn-test/package.json
View file

@ -15,7 +15,8 @@
"@kbn/dev-utils": "1.0.0",
"@types/parse-link-header": "^1.0.0",
"@types/strip-ansi": "^5.2.1",
"@types/xml2js": "^0.4.5"
"@types/xml2js": "^0.4.5",
"diff": "^4.0.1"
},
"dependencies": {
"chalk": "^2.4.2",

6
packages/kbn-test/src/failed_tests_reporter/README.md
View file

@ -12,10 +12,10 @@ copy(`wget "${Array.from($$('a[href$=".xml"]')).filter(a => a.innerText === 'Dow
This copies a script to download the reports, which you should execute in the `test/junit` directory.
Next, run the CLI in `--dry-run` mode so that it doesn't actually communicate with Github.
Next, run the CLI in `--no-github-update` mode so that it doesn't actually communicate with Github and `--no-report-update` to prevent the script from mutating the reports on disk and instead log the updated report.
```sh
node scripts/report_failed_tests.js --verbose --dry-run --build-url foo
node scripts/report_failed_tests.js --verbose --no-github-update --no-report-update
```
If you specify the `GITHUB_TOKEN` environment variable then `--dry-run` will execute read operations but still won't execute write operations.
Unless you specify the `GITHUB_TOKEN` environment variable requests to read existing issues will use anonymous access which is limited to 60 requests per hour.

25
packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts Normal file
View file

@ -0,0 +1,25 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
const Fs = jest.requireActual('fs');
export const FTR_REPORT = Fs.readFileSync(require.resolve('./ftr_report.xml'), 'utf8');
export const JEST_REPORT = Fs.readFileSync(require.resolve('./jest_report.xml'), 'utf8');
export const KARMA_REPORT = Fs.readFileSync(require.resolve('./karma_report.xml'), 'utf8');
export const MOCHA_REPORT = Fs.readFileSync(require.resolve('./mocha_report.xml'), 'utf8');

350
packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts Normal file
View file

@ -0,0 +1,350 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Path from 'path';
import { ToolingLog } from '@kbn/dev-utils';
// @ts-ignore
import { createPatch } from 'diff';
// turns out Jest can't encode xml diffs in their JUnit reports...
expect.addSnapshotSerializer({
test: v => typeof v === 'string' && (v.includes('<') || v.includes('>')),
print: v =>
v
.replace(/</g, '‹')
.replace(/>/g, '›')
.replace(/^\s+$/gm, ''),
});
jest.mock('fs', () => {
const realFs = jest.requireActual('fs');
return {
readFile: realFs.read,
writeFile: (...args: any[]) => {
setTimeout(args[args.length - 1], 0);
},
};
});
import { FTR_REPORT, JEST_REPORT, MOCHA_REPORT, KARMA_REPORT } from './__fixtures__';
import { parseTestReport } from './test_report';
import { addMessagesToReport } from './add_messages_to_report';
beforeEach(() => {
jest.resetAllMocks();
});
const log = new ToolingLog();
it('rewrites ftr reports with minimal changes', async () => {
const xml = await addMessagesToReport({
report: await parseTestReport(FTR_REPORT),
messages: [
{
name: 'maps app maps loaded from sample data ecommerce "before all" hook',
classname:
'Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js',
message: 'foo bar',
},
],
log,
reportPath: Path.resolve(__dirname, './__fixtures__/ftr_report.xml'),
});
expect(createPatch('ftr.xml', FTR_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
Index: ftr.xml
===================================================================
--- ftr.xml [object Object]
+++ ftr.xml
@@ -2,52 +2,56 @@
‹testsuites›
‹testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71"›
‹testcase name="maps app maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378"›
‹system-out›
- ‹![CDATA[[00:00:00] │
+ [00:00:00] │
[00:07:04] └-: maps app
...
[00:15:02] │
-]]›
+
‹/system-out›
‹failure›
- ‹![CDATA[Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~="layerTocActionsPanelToggleButtonRoad_Map_-_Bright"])
+ Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~="layerTocActionsPanelToggleButtonRoad_Map_-_Bright"])
Wait timed out after 10055ms
at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17
at process._tickCallback (internal/process/next_tick.js:68:7)
at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
- at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)]]›
- ‹/failure›
+ at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
+
+
+Failed Tests Reporter:
+ - foo bar
+‹/failure›
‹/testcase›
‹testcase name="maps app &quot;after all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps" time="0.179"›
‹system-out›
- ‹![CDATA[[00:00:00] │
+ [00:00:00] │
[00:07:04] └-: maps app
...
-]]›
+
‹/system-out›
‹failure›
- ‹![CDATA[{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
+ { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12)
- at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }]]›
+ at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }
‹/failure›
‹/testcase›
‹testcase name="InfraOps app feature controls infrastructure security global infrastructure all privileges shows infrastructure navlink" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/infra/feature_controls/infrastructure_security·ts"›
‹system-out›
- ‹![CDATA[[00:00:00] │
+ [00:00:00] │
[00:05:13] └-: InfraOps app
...
-]]›
+
‹/system-out›
‹skipped/›
‹/testcase›
‹testcase name="machine learning anomaly detection saved search with lucene query job creation opens the advanced section" classname="Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts" time="6.040"›
- ‹system-out›‹![CDATA[[00:21:57] └-: machine learning...]]›‹/system-out›
- ‹failure›‹![CDATA[{ NoSuchSessionError: Tried to run command without establishing a connection
+ ‹system-out›[00:21:57] └-: machine learning...‹/system-out›
+ ‹failure›{ NoSuchSessionError: Tried to run command without establishing a connection
at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15)
at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13)
at Executor.execute (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:489:26)
- at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }]]›‹/failure›
+ at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }‹/failure›
‹/testcase›
‹/testsuite›
-‹/testsuites›
+‹/testsuites›
\\ No newline at end of file
`);
});
it('rewrites jest reports with minimal changes', async () => {
const xml = await addMessagesToReport({
report: await parseTestReport(JEST_REPORT),
messages: [
{
classname: 'X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp',
name: 'launcher can reconnect if process died',
message: 'foo bar',
},
],
log,
reportPath: Path.resolve(__dirname, './__fixtures__/jest_report.xml'),
});
expect(createPatch('jest.xml', JEST_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
Index: jest.xml
===================================================================
--- jest.xml [object Object]
+++ jest.xml
@@ -3,13 +3,17 @@
‹testsuite name="x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts"›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060"›
- ‹failure›
- ‹![CDATA[TypeError: Cannot read property '0' of undefined
- at Object.‹anonymous›.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)]]›
- ‹/failure›
+ ‹failure›‹![CDATA[
+ TypeError: Cannot read property '0' of undefined
+ at Object.‹anonymous›.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
+
+
+Failed Tests Reporter:
+ - foo bar
+]]›‹/failure›
‹/testcase›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="passive launcher can start and end a process" time="0.435"/›
‹testcase classname="X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp" name="passive launcher should restart a process if a process died before connected" time="1.502"/›
‹/testsuite›
-‹/testsuites›
+‹/testsuites›
\\ No newline at end of file
`);
});
it('rewrites mocha reports with minimal changes', async () => {
const xml = await addMessagesToReport({
report: await parseTestReport(MOCHA_REPORT),
messages: [
{
name: 'code in multiple nodes "before all" hook',
classname: 'X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts',
message: 'foo bar',
},
],
log,
reportPath: Path.resolve(__dirname, './__fixtures__/mocha_report.xml'),
});
expect(createPatch('mocha.xml', MOCHA_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
Index: mocha.xml
===================================================================
--- mocha.xml [object Object]
+++ mocha.xml
@@ -2,12 +2,12 @@
‹testsuites›
‹testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3"›
‹testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.121"›
‹system-out›
- ‹![CDATA[]]›
+
‹/system-out›
- ‹failure›
- ‹![CDATA[Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
+ ‹failure›‹![CDATA[
+ Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
‹html›
‹head›‹title›503 Service Temporarily Unavailable‹/title›‹/head›
‹body bgcolor="white"›
‹center›‹h1›503 Service Temporarily Unavailable‹/h1›‹/center›
@@ -15,24 +15,28 @@
‹/body›
‹/html›
at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13)
- at process._tickCallback (internal/process/next_tick.js:68:7)]]›
- ‹/failure›
+ at process._tickCallback (internal/process/next_tick.js:68:7)
+
+
+Failed Tests Reporter:
+ - foo bar
+]]›‹/failure›
‹/testcase›
‹testcase name="code in multiple nodes &quot;after all&quot; hook" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts" time="0.003"›
‹system-out›
- ‹![CDATA[]]›
+
‹/system-out›
‹failure›
- ‹![CDATA[TypeError: Cannot read property 'shutdown' of undefined
+ TypeError: Cannot read property 'shutdown' of undefined
at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
- at process.topLevelDomainCallback (domain.js:120:23)]]›
+ at process.topLevelDomainCallback (domain.js:120:23)
‹/failure›
‹/testcase›
‹testcase name="repository service test can not clone a repo by ssh without a key" classname="X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/repository_service·ts" time="0.005"›
‹system-out›
- ‹![CDATA[]]›
+
‹/system-out›
‹/testcase›
‹/testsuite›
-‹/testsuites›
+‹/testsuites›
\\ No newline at end of file
`);
});
it('rewrites karma reports with minimal changes', async () => {
const xml = await addMessagesToReport({
report: await parseTestReport(KARMA_REPORT),
messages: [
{
name:
'CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK',
classname: 'Browser Unit Tests.CoordinateMapsVisualizationTest',
message: 'foo bar',
},
],
log,
reportPath: Path.resolve(__dirname, './__fixtures__/karma_report.xml'),
});
expect(createPatch('karma.xml', KARMA_REPORT, xml, { context: 0 })).toMatchInlineSnapshot(`
Index: karma.xml
===================================================================
--- karma.xml [object Object]
+++ karma.xml
@@ -1,5 +1,5 @@
-‹?xml version="1.0"?›
+‹?xml version="1.0" encoding="utf-8"?›
‹testsuite name="Chrome 75.0.3770 (Mac OS X 10.14.5)" package="" timestamp="2019-07-02T19:53:21" id="0" hostname="spalger.lan" tests="648" errors="0" failures="4" time="1.759"›
‹properties›
‹property name="browser.fullName" value="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"/›
‹/properties›
@@ -7,27 +7,31 @@
‹testcase name="Vis-Editor-Agg-Params plugin directive should hide custom label parameter" time="0" classname="Browser Unit Tests.Vis-Editor-Agg-Params plugin directive"›
‹skipped/›
‹/testcase›
‹testcase name="CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK" time="0.265" classname="Browser Unit Tests.CoordinateMapsVisualizationTest"›
- ‹failure type=""›Error: expected 7069 to be below 64
- at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&amp;shard_num=1:13671:11)
- at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&amp;shard_num=1:13891:8)
- at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&amp;shard_num=1:14078:15)
- at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&amp;shard_num=1:158985:60)
+ ‹failure type=""›‹![CDATA[Error: expected 7069 to be below 64
+ at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11)
+ at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8)
+ at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15)
+ at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60)
at tryCatch (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:62:40)
at Generator.invoke [as _invoke] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:288:22)
- at Generator.prototype.&lt;computed&gt; [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
- at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&amp;shard_num=1:158772:103)
- at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&amp;shard_num=1:158774:194)
-‹/failure›
+ at Generator.prototype.‹computed› [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
+ at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
+ at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
+
+
+Failed Tests Reporter:
+ - foo bar
+]]›‹/failure›
‹/testcase›
‹testcase name="CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should toggle to Heatmap OK" time="0.055" classname="Browser Unit Tests.CoordinateMapsVisualizationTest"/›
‹testcase name="VegaParser._parseSchema should warn on vega-lite version too new to be supported" time="0.001" classname="Browser Unit Tests.VegaParser·_parseSchema"/›
‹system-out›
- ‹![CDATA[Chrome 75.0.3770 (Mac OS X 10.14.5) LOG: 'ready to load tests for shard 1 of 4'
+ Chrome 75.0.3770 (Mac OS X 10.14.5) LOG: 'ready to load tests for shard 1 of 4'
,Chrome 75.0.3770 (Mac OS X 10.14.5) WARN: 'Unmatched GET to http://localhost:9876/api/interpreter/fns'
...
-]]›
+
‹/system-out›
‹system-err/›
-‹/testsuite›
+‹/testsuite›
\\ No newline at end of file
`);
});

90
packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts Normal file
View file

@ -0,0 +1,90 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Fs from 'fs';
import { promisify } from 'util';
import { ToolingLog } from '@kbn/dev-utils';
import xml2js from 'xml2js';
import { TestReport, makeFailedTestCaseIter } from './test_report';
const writeAsync = promisify(Fs.writeFile);
export interface Message {
classname: string;
name: string;
message: string;
}
/**
* Mutate the report to include mentions of Github issues related to test failures,
* then write the updated report to disk
*/
export async function addMessagesToReport(options: {
log: ToolingLog;
report: TestReport;
messages: Message[];
reportPath: string;
dryRun?: boolean;
}) {
const { log, report, messages, reportPath, dryRun } = options;
for (const testCase of makeFailedTestCaseIter(report)) {
const { classname, name } = testCase.$;
const messageList = messages
.filter(u => u.classname === classname && u.name === name)
.reduce((acc, u) => `${acc}\n - ${u.message}`, '');
if (!messageList) {
continue;
}
log.info(`${classname} - ${name}:${messageList}`);
const append = `\n\nFailed Tests Reporter:${messageList}\n`;
if (
testCase.failure[0] &&
typeof testCase.failure[0] === 'object' &&
typeof testCase.failure[0]._ === 'string'
) {
testCase.failure[0]._ += append;
} else {
testCase.failure[0] = String(testCase.failure[0]) + append;
}
}
const builder = new xml2js.Builder({
cdata: true,
xmldec: { version: '1.0', encoding: 'utf-8' },
});
const xml = builder
.buildObject(report)
.split('\n')
.map(line => (line.trim() === '' ? '' : line))
.join('\n');
if (dryRun) {
log.info(`updated ${reportPath}\n${xml}`);
} else {
await writeAsync(reportPath, xml, 'utf8');
}
return xml;
}

75
packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts
View file

@ -17,14 +17,12 @@
* under the License.
*/
import { ToolingLog } from '@kbn/dev-utils';
import { getFailures } from './get_failures';
const log = new ToolingLog();
import { parseTestReport } from './test_report';
import { FTR_REPORT, JEST_REPORT, KARMA_REPORT, MOCHA_REPORT } from './__fixtures__';
it('discovers failures in ftr report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/ftr_report.xml'));
const failures = getFailures(await parseTestReport(FTR_REPORT));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
@ -37,15 +35,39 @@ it('discovers failures in ftr report', async () => {
at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
",
"likelyIrrelevant": false,
"name": "maps app maps loaded from sample data ecommerce \\"before all\\" hook",
"time": "154.378",
},
Object {
"classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps",
"failure": "
{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12)
at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }
",
"likelyIrrelevant": true,
"name": "maps app \\"after all\\" hook",
"time": "0.179",
},
Object {
"classname": "Firefox XPack UI Functional Tests.x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job·ts",
"failure": "{ NoSuchSessionError: Tried to run command without establishing a connection
at Object.throwDecodedError (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/error.js:550:15)
at parseHttpResponse (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:563:13)
at Executor.execute (/dev/shm/workspace/kibana/node_modules/selenium-webdriver/lib/http.js:489:26)
at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }",
"likelyIrrelevant": true,
"name": "machine learning anomaly detection saved search with lucene query job creation opens the advanced section",
"time": "6.040",
},
]
`);
});
it('discovers failures in jest report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/jest_report.xml'));
const failures = getFailures(await parseTestReport(JEST_REPORT));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
@ -54,6 +76,7 @@ it('discovers failures in jest report', async () => {
TypeError: Cannot read property '0' of undefined
at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
",
"likelyIrrelevant": false,
"name": "launcher can reconnect if process died",
"time": "7.060",
},
@ -62,7 +85,7 @@ it('discovers failures in jest report', async () => {
});
it('discovers failures in karma report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/karma_report.xml'));
const failures = getFailures(await parseTestReport(KARMA_REPORT));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
@ -78,6 +101,7 @@ it('discovers failures in karma report', async () => {
at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
",
"likelyIrrelevant": false,
"name": "CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK",
"time": "0.265",
},
@ -86,6 +110,39 @@ it('discovers failures in karma report', async () => {
});
it('discovers failures in mocha report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/mocha_report.xml'));
expect(failures).toMatchInlineSnapshot(`Array []`);
const failures = getFailures(await parseTestReport(MOCHA_REPORT));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
"classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
"failure": "
Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
<html>
<head><title>503 Service Temporarily Unavailable</title></head>
<body bgcolor=\\"white\\">
<center><h1>503 Service Temporarily Unavailable</h1></center>
<hr><center>nginx/1.13.7</center>
</body>
</html>
at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13)
at process._tickCallback (internal/process/next_tick.js:68:7)
",
"likelyIrrelevant": true,
"name": "code in multiple nodes \\"before all\\" hook",
"time": "0.121",
},
Object {
"classname": "X-Pack Mocha Tests.x-pack/legacy/plugins/code/server/__tests__/multi_node·ts",
"failure": "
TypeError: Cannot read property 'shutdown' of undefined
at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
at process.topLevelDomainCallback (domain.js:120:23)
",
"likelyIrrelevant": true,
"name": "code in multiple nodes \\"after all\\" hook",
"time": "0.003",
},
]
`);
});

109
packages/kbn-test/src/failed_tests_reporter/get_failures.ts
View file

@ -17,69 +17,16 @@
* under the License.
*/
import { promisify } from 'util';
import Fs from 'fs';
import xml2js from 'xml2js';
import stripAnsi from 'strip-ansi';
import { ToolingLog } from '@kbn/dev-utils';
type TestReport =
| {
testsuites: {
testsuite: TestSuite[];
};
}
| {
testsuite: TestSuite;
};
import { FailedTestCase, TestReport, makeFailedTestCaseIter } from './test_report';
interface TestSuite {
$: {
/* ISO8601 timetamp when test suite ran */
timestamp: string;
/* number of second this tests suite took */
time: string;
/* number of tests as a string */
tests: string;
/* number of failed tests as a string */
failures: string;
/* number of skipped tests as a string */
skipped: string;
};
testcase: TestCase[];
}
interface TestCase {
$: {
/* unique test name */
name: string;
/* somewhat human readable combination of test name and file */
classname: string;
/* number of seconds this test took */
time: string;
};
/* contents of system-out elements */
'system-out'?: string[];
/* contents of failure elements */
failure?: Array<string | { _: string }>;
/* contents of skipped elements */
skipped?: string[];
}
export type TestFailure = TestCase['$'] & {
export type TestFailure = FailedTestCase['$'] & {
failure: string;
likelyIrrelevant: boolean;
};
const readAsync = promisify(Fs.readFile);
const indent = (text: string) =>
` ${text
.split('\n')
.map(l => ` ${l}`)
.join('\n')}`;
const getFailureText = (failure: NonNullable<TestCase['failure']>) => {
const getFailureText = (failure: FailedTestCase['failure']) => {
const [failureNode] = failure;
if (failureNode && typeof failureNode === 'object' && typeof failureNode._ === 'string') {
@ -89,7 +36,7 @@ const getFailureText = (failure: NonNullable<TestCase['failure']>) => {
return stripAnsi(String(failureNode));
};
const isLikelyIrrelevant = ({ name, failure }: TestFailure) => {
const isLikelyIrrelevant = (name: string, failure: string) => {
if (
failure.includes('NoSuchSessionError: This driver instance does not have a valid session ID') ||
failure.includes('NoSuchSessionError: Tried to run command without establishing a connection')
@ -118,47 +65,25 @@ const isLikelyIrrelevant = ({ name, failure }: TestFailure) => {
if (failure.includes('Unable to fetch Kibana status API response from Kibana')) {
return true;
}
return false;
};
export async function getFailures(log: ToolingLog, testReportPath: string) {
const xml = await readAsync(testReportPath, 'utf8');
// Parses junit XML files
const report: TestReport = await xml2js.parseStringPromise(xml);
// Grab the failures. Reporters may report multiple testsuites in a single file.
const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite];
export function getFailures(report: TestReport) {
const failures: TestFailure[] = [];
for (const testSuite of testSuites) {
for (const testCase of testSuite.testcase) {
const { failure } = testCase;
if (!failure) {
continue;
}
for (const testCase of makeFailedTestCaseIter(report)) {
const failure = getFailureText(testCase.failure);
const likelyIrrelevant = isLikelyIrrelevant(testCase.$.name, failure);
failures.push({
// unwrap xml weirdness
const failureCase: TestFailure = {
...testCase.$,
// Strip ANSI color characters
failure: getFailureText(failure),
};
if (isLikelyIrrelevant(failureCase)) {
log.warning(
`Ignoring likely irrelevant failure: ${failureCase.classname} - ${
failureCase.name
}\n${indent(failureCase.failure)}`
);
continue;
}
failures.push(failureCase);
}
...testCase.$,
// Strip ANSI color characters
failure,
likelyIrrelevant,
});
}
log.info(`Found ${failures.length} test failures`);
return failures;
}

35
packages/kbn-test/src/failed_tests_reporter/github_api.ts
View file

@ -19,7 +19,7 @@
import Url from 'url';
import Axios, { AxiosRequestConfig } from 'axios';
import Axios, { AxiosRequestConfig, AxiosInstance } from 'axios';
import parseLinkHeader from 'parse-link-header';
import { ToolingLog, isAxiosResponseError, isAxiosRequestError } from '@kbn/dev-utils';
@ -40,25 +40,34 @@ type RequestOptions = AxiosRequestConfig & {
};
export class GithubApi {
private readonly x = Axios.create({
headers: {
Authorization: `token ${this.token}`,
'User-Agent': 'elastic/kibana#failed_test_reporter',
},
});
private readonly log: ToolingLog;
private readonly token: string | undefined;
private readonly dryRun: boolean;
private readonly x: AxiosInstance;
/**
* Create a GithubApi helper object, if token is undefined requests won't be
* sent, but will instead be logged.
*/
constructor(
private readonly log: ToolingLog,
private readonly token: string | undefined,
private readonly dryRun: boolean
) {
if (!token && !dryRun) {
constructor(options: {
log: GithubApi['log'];
token: GithubApi['token'];
dryRun: GithubApi['dryRun'];
}) {
this.log = options.log;
this.token = options.token;
this.dryRun = options.dryRun;
if (!this.token && !this.dryRun) {
throw new TypeError('token parameter is required');
}
this.x = Axios.create({
headers: {
...(this.token ? { Authorization: `token ${this.token}` } : {}),
'User-Agent': 'elastic/kibana#failed_test_reporter',
},
});
}
private failedTestIssuesPageCache: {

28
packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts
View file

@ -18,19 +18,14 @@
*/
import dedent from 'dedent';
import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/dev-utils';
import { createFailureIssue, updatedFailureIssue } from './report_failure';
import { createFailureIssue, updateFailureIssue } from './report_failure';
jest.mock('./github_api');
const { GithubApi } = jest.requireMock('./github_api');
describe('createFailureIssue()', () => {
it('creates new github issue with failure text, link to issue, and valid metadata', async () => {
const log = new ToolingLog();
const writer = new ToolingLogCollectingWriter();
log.setWriters([writer]);
const api = new GithubApi();
await createFailureIssue(
@ -40,8 +35,8 @@ describe('createFailureIssue()', () => {
failure: 'this is the failure text',
name: 'test name',
time: '2018-01-01T01:00:00Z',
likelyIrrelevant: false,
},
log,
api
);
@ -72,23 +67,14 @@ describe('createFailureIssue()', () => {
],
}
`);
expect(writer.messages).toMatchInlineSnapshot(`
Array [
" info Created issue undefined",
]
`);
});
});
describe('updatedFailureIssue()', () => {
describe('updateFailureIssue()', () => {
it('increments failure count and adds new comment to issue', async () => {
const log = new ToolingLog();
const writer = new ToolingLogCollectingWriter();
log.setWriters([writer]);
const api = new GithubApi();
await updatedFailureIssue(
await updateFailureIssue(
'https://build-url',
{
html_url: 'https://github.com/issues/1234',
@ -101,7 +87,6 @@ describe('updatedFailureIssue()', () => {
<!-- kibanaCiData = {"failed-test":{"test.failCount":10}} -->"
`,
},
log,
api
);
@ -139,10 +124,5 @@ describe('updatedFailureIssue()', () => {
],
}
`);
expect(writer.messages).toMatchInlineSnapshot(`
Array [
" info Updated issue https://github.com/issues/1234, failCount: 11",
]
`);
});
});

21
packages/kbn-test/src/failed_tests_reporter/report_failure.ts
View file

@ -17,18 +17,11 @@
* under the License.
*/
import { ToolingLog } from '@kbn/dev-utils';
import { TestFailure } from './get_failures';
import { GithubIssue, GithubApi } from './github_api';
import { getIssueMetadata, updateIssueMetadata } from './issue_metadata';
export async function createFailureIssue(
buildUrl: string,
failure: TestFailure,
log: ToolingLog,
api: GithubApi
) {
export async function createFailureIssue(buildUrl: string, failure: TestFailure, api: GithubApi) {
const title = `Failing test: ${failure.classname} - ${failure.name}`;
const body = updateIssueMetadata(
@ -48,16 +41,10 @@ export async function createFailureIssue(
}
);
const newIssueUrl = await api.createIssue(title, body, ['failed-test']);
log.info(`Created issue ${newIssueUrl}`);
return await api.createIssue(title, body, ['failed-test']);
}
export async function updatedFailureIssue(
buildUrl: string,
issue: GithubIssue,
log: ToolingLog,
api: GithubApi
) {
export async function updateFailureIssue(buildUrl: string, issue: GithubIssue, api: GithubApi) {
// Increment failCount
const newCount = getIssueMetadata(issue.body, 'test.failCount', 0) + 1;
const newBody = updateIssueMetadata(issue.body, {
@ -67,5 +54,5 @@ export async function updatedFailureIssue(
await api.editIssueBodyAndEnsureOpen(issue.number, newBody);
await api.addIssueComment(issue.number, `New failure: [Jenkins Build](${buildUrl})`);
log.info(`Updated issue ${issue.html_url}, failCount: ${newCount}`);
return newCount;
}

98
packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts
View file

@ -22,19 +22,22 @@ import globby from 'globby';
import { getFailures } from './get_failures';
import { GithubApi } from './github_api';
import { updatedFailureIssue, createFailureIssue } from './report_failure';
import { updateFailureIssue, createFailureIssue } from './report_failure';
import { getIssueMetadata } from './issue_metadata';
import { readTestReport } from './test_report';
import { addMessagesToReport, Message } from './add_messages_to_report';
export function runFailedTestsReporterCli() {
run(
async ({ log, flags }) => {
const buildUrl = flags['build-url'];
if (typeof buildUrl !== 'string' || !buildUrl) {
throw createFlagError('Missing --build-url or process.env.BUILD_URL');
let updateGithub = flags['github-update'];
if (updateGithub && !process.env.GITHUB_TOKEN) {
throw createFailError(
'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag'
);
}
const dryRun = !!flags['dry-run'];
if (!dryRun) {
if (updateGithub) {
// JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
const branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH;
@ -48,26 +51,43 @@ export function runFailedTestsReporterCli() {
const isMasterOrVersion =
branch.match(/^(origin\/){0,1}master$/) || branch.match(/^(origin\/){0,1}\d+\.(x|\d+)$/);
if (!isMasterOrVersion || isPr) {
throw createFailError('Failure issues only created on master/version branch jobs', {
exitCode: 0,
});
}
if (!process.env.GITHUB_TOKEN) {
throw createFailError(
'GITHUB_TOKEN environment variable must be set, otherwise use --dry-run flag'
);
log.info('Failure issues only created on master/version branch jobs');
updateGithub = false;
}
}
const githubApi = new GithubApi(log, process.env.GITHUB_TOKEN, dryRun);
const githubApi = new GithubApi({
log,
token: process.env.GITHUB_TOKEN,
dryRun: !updateGithub,
});
const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
if (typeof buildUrl !== 'string' || !buildUrl) {
throw createFlagError('Missing --build-url or process.env.BUILD_URL');
}
const reportPaths = await globby(['target/junit/**/*.xml'], {
cwd: REPO_ROOT,
absolute: true,
});
for (const reportPath of reportPaths) {
for (const failure of await getFailures(log, reportPath)) {
const report = await readTestReport(reportPath);
const messages: Message[] = [];
for (const failure of await getFailures(report)) {
if (failure.likelyIrrelevant) {
messages.push({
classname: failure.classname,
name: failure.name,
message:
'Failure is likely irrelevant' +
(updateGithub ? ', so an issue was not created or updated' : ''),
});
continue;
}
const existingIssue = await githubApi.findFailedTestIssue(
i =>
getIssueMetadata(i.body, 'test.class') === failure.classname &&
@ -75,23 +95,57 @@ export function runFailedTestsReporterCli() {
);
if (existingIssue) {
await updatedFailureIssue(buildUrl, existingIssue, log, githubApi);
} else {
await createFailureIssue(buildUrl, failure, log, githubApi);
const newFailureCount = await updateFailureIssue(buildUrl, existingIssue, githubApi);
const url = existingIssue.html_url;
const message =
`Test has failed ${newFailureCount - 1} times on tracked branches: ${url}` +
(updateGithub
? `. Updated existing issue: ${url} (fail count: ${newFailureCount})`
: '');
messages.push({
classname: failure.classname,
name: failure.name,
message,
});
continue;
}
const newIssueUrl = await createFailureIssue(buildUrl, failure, githubApi);
const message =
`Test has not failed recently on tracked branches` +
(updateGithub ? `Created new issue: ${newIssueUrl}` : '');
messages.push({
classname: failure.classname,
name: failure.name,
message,
});
}
// mutates report to include messages and writes updated report to disk
await addMessagesToReport({
report,
messages,
log,
reportPath,
dryRun: !flags['report-update'],
});
}
},
{
description: `a cli that opens issues or updates existing issues based on junit reports`,
flags: {
boolean: ['dry-run'],
boolean: ['github-update', 'report-update'],
string: ['build-url'],
default: {
'github-update': true,
'report-update': true,
'build-url': process.env.BUILD_URL,
},
help: `
--dry-run Execute the CLI without contacting Github
--no-github-update Execute the CLI without writing to Github
--no-report-update Execute the CLI without writing to the JUnit reports
--build-url URL of the failed build, defaults to process.env.BUILD_URL
`,
},

100
packages/kbn-test/src/failed_tests_reporter/test_report.ts Normal file
View file

@ -0,0 +1,100 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Fs from 'fs';
import { promisify } from 'util';
import xml2js from 'xml2js';
const readAsync = promisify(Fs.readFile);
export type TestReport =
| {
testsuites: {
testsuite: TestSuite[];
};
}
| {
testsuite: TestSuite;
};
export interface TestSuite {
$: {
/* ISO8601 timetamp when test suite ran */
timestamp: string;
/* number of second this tests suite took */
time: string;
/* number of tests as a string */
tests: string;
/* number of failed tests as a string */
failures: string;
/* number of skipped tests as a string */
skipped: string;
};
testcase: TestCase[];
}
export interface TestCase {
$: {
/* unique test name */
name: string;
/* somewhat human readable combination of test name and file */
classname: string;
/* number of seconds this test took */
time: string;
};
/* contents of system-out elements */
'system-out'?: string[];
/* contents of failure elements */
failure?: Array<string | { _: string }>;
/* contents of skipped elements */
skipped?: string[];
}
export interface FailedTestCase extends TestCase {
failure: Array<string | { _: string }>;
}
/**
* Parse JUnit XML Files
*/
export async function parseTestReport(xml: string): Promise<TestReport> {
return await xml2js.parseStringPromise(xml);
}
export async function readTestReport(testReportPath: string) {
return await parseTestReport(await readAsync(testReportPath, 'utf8'));
}
export function* makeFailedTestCaseIter(report: TestReport) {
// Grab the failures. Reporters may report multiple testsuites in a single file.
const testSuites = 'testsuites' in report ? report.testsuites.testsuite : [report.testsuite];
for (const testSuite of testSuites) {
for (const testCase of testSuite.testcase) {
const { failure } = testCase;
if (!failure) {
continue;
}
yield testCase as FailedTestCase;
}
}
}

12
vars/kibanaPipeline.groovy
View file

@ -30,6 +30,10 @@ def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
uploadAllGcsArtifacts(name)
}
catchError {
runErrorReporter()
}
catchError {
runbld.junit()
}
@ -37,10 +41,6 @@ def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
catchError {
publishJunit()
}
catchError {
runErrorReporter()
}
}
}
}
@ -103,10 +103,10 @@ def legacyJobRunner(name) {
uploadAllGcsArtifacts(name)
}
catchError {
publishJunit()
runErrorReporter()
}
catchError {
runErrorReporter()
publishJunit()
}
}
}