[CI] Produce junit test reports (#15281)

* [mocha] use custom reporter for legible results in jenkins

* [jest] use custom result processor for legible results in jenkins

* [karma] enable junit output on CI

* [mocha/junitReporter] accept rootDirectory as configuration

* [jest/reporter] use reporters option added in jest 20

* [toolingLog] remove black/white specific colors

* [dev/mocha/junit] no reason for junit to be a "reporter"

* typos

* [dev/mocha/junit] use else if

* [karma/junit] use string#replace for explicitness

* [junit] use test file path as "classname"

* [ftr/mocha] no longer a "console" specific reporter

package.json

@@ -268,6 +268,7 @@
     "karma-coverage": "1.1.1",
     "karma-firefox-launcher": "1.0.1",
     "karma-ie-launcher": "1.0.0",
+    "karma-junit-reporter": "1.2.0",
     "karma-mocha": "1.3.0",
     "karma-safari-launcher": "1.0.0",
     "keymirror": "0.1.1",
@@ -294,6 +295,7 @@
     "supertest-as-promised": "2.0.2",
     "tree-kill": "1.1.0",
     "webpack-dev-server": "2.9.1",
+    "xmlbuilder": "9.0.4",
     "yeoman-generator": "1.1.1",
     "yo": "2.0.0"
   },

scripts/jest.js

@@ -11,4 +11,4 @@
 // See all cli options in https://facebook.github.io/jest/docs/cli.html
 
 require('../src/babel-register');
-require('../src/jest/cli');
+require('../src/dev/jest/cli');

src/dev/index.js

@@ -1 +1,5 @@
 export { createToolingLog } from './tooling_log';
+export {
+  createAutoJunitReporter,
+  setupJunitReportGeneration,
+} from './mocha';

src/dev/jest/babel_transform.js

@@ -2,6 +2,6 @@ const babelJest = require('babel-jest');
 
 module.exports = babelJest.createTransformer({
   presets: [
-    require.resolve('../babel-preset/node')
+    require.resolve('../../babel-preset/node')
   ]
 });

src/dev/jest/config.json

@@ -1,5 +1,5 @@
 {
-  "rootDir": "../../",
+  "rootDir": "../../..",
   "roots": [
     "<rootDir>/src/ui/public",
     "<rootDir>/src/core_plugins",
@@ -18,12 +18,12 @@
     "^ui_framework/services": "<rootDir>/ui_framework/services",
     "^ui_framework/src/test": "<rootDir>/ui_framework/src/test",
     "^ui/(.*)": "<rootDir>/src/ui/public/$1",
-    "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/src/jest/mocks/file_mock.js",
-    "\\.(css|less|scss)$": "<rootDir>/src/jest/mocks/style_mock.js"
+    "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/src/dev/jest/mocks/file_mock.js",
+    "\\.(css|less|scss)$": "<rootDir>/src/dev/jest/mocks/style_mock.js"
   },
   "setupFiles": [
-    "<rootDir>/src/jest/setup/babel_polyfill.js",
-    "<rootDir>/src/jest/setup/request_animation_frame_polyfill.js"
+    "<rootDir>/src/dev/jest/setup/babel_polyfill.js",
+    "<rootDir>/src/dev/jest/setup/request_animation_frame_polyfill.js"
   ],
   "coverageDirectory": "<rootDir>/target/jest-coverage",
   "coverageReporters": [
@@ -42,12 +42,16 @@
     "<rootDir>/ui_framework/generator-kui/"
   ],
   "transform": {
-    "^.+\\.js$": "<rootDir>/src/jest/babelTransform.js"
+    "^.+\\.js$": "<rootDir>/src/dev/jest/babel_transform.js"
   },
   "transformIgnorePatterns": [
     "[/\\\\]node_modules[/\\\\].+\\.js$"
   ],
   "snapshotSerializers": [
     "<rootDir>/node_modules/enzyme-to-json/serializer"
+  ],
+  "reporters": [
+    "default",
+    "<rootDir>/src/dev/jest/junit_reporter.js"
   ]
 }

src/dev/jest/junit_reporter.js

@@ -0,0 +1,99 @@
+import { resolve, dirname, relative } from 'path';
+import { writeFileSync } from 'fs';
+
+import mkdirp from 'mkdirp';
+import xmlBuilder from 'xmlbuilder';
+
+const ROOT_DIR = dirname(require.resolve('../../../package.json'));
+
+/**
+ * Jest reporter that produces JUnit report when running on CI
+ * @class JestJunitReporter
+ */
+export default class JestJunitReporter {
+  constructor(globalConfig, options = {}) {
+    const {
+      reportName = 'Jest Tests',
+      rootDirectory = ROOT_DIR
+    } = options;
+
+    this._reportName = reportName;
+    this._rootDirectory = rootDirectory;
+  }
+
+  /**
+   * Called by jest when all tests complete
+   * @param {Object} contexts
+   * @param {JestResults} results see https://facebook.github.io/jest/docs/en/configuration.html#testresultsprocessor-string
+   * @return {undefined}
+   */
+  onRunComplete(contexts, results) {
+    if (!process.env.CI) {
+      return;
+    }
+
+    const reportName = this._reportName;
+    const rootDirectory = this._rootDirectory;
+    const root = xmlBuilder.create(
+      'testsuites',
+      { encoding: 'utf-8' },
+      {},
+      { skipNullAttributes: true }
+    );
+
+    const msToIso = ms => ms ? new Date(ms).toISOString().slice(0, -5) : undefined;
+    const msToSec = ms => ms ? (ms / 1000).toFixed(3) : undefined;
+
+    root.att({
+      name: 'jest',
+      timestamp: msToIso(results.startTime),
+      time: msToSec(Date.now() - results.startTime),
+      tests: results.numTotalTests,
+      failures: results.numFailedTests,
+      skipped: results.numPendingTests,
+    });
+
+    // top level test results are the files/suites
+    results.testResults.forEach(suite => {
+      const suiteEl = root.ele('testsuite', {
+        name: relative(rootDirectory, suite.testFilePath),
+        timestamp: msToIso(suite.perfStats.start),
+        time: msToSec(suite.perfStats.end - suite.perfStats.start),
+        tests: suite.testResults.length,
+        failures: suite.numFailedTests,
+        skipped: suite.numPendingTests,
+        file: suite.testFilePath
+      });
+
+      // nested in there are the tests in that file
+      const relativePath = dirname(relative(rootDirectory, suite.testFilePath));
+      const classname = `${reportName}.${relativePath.replace(/\./g, '·')}`;
+      suite.testResults.forEach(test => {
+        const testEl = suiteEl.ele('testcase', {
+          classname,
+          name: [...test.ancestorTitles, test.title].join(' '),
+          time: msToSec(test.duration)
+        });
+
+        test.failureMessages.forEach((message) => {
+          testEl.ele('failure').dat(message);
+        });
+
+        if (test.status === 'pending') {
+          testEl.ele('skipped');
+        }
+      });
+    });
+
+    const reportPath = resolve(rootDirectory, `target/junit/${reportName}.xml`);
+    const reportXML = root.end({
+      pretty: true,
+      indent: '  ',
+      newline: '\n',
+      spacebeforeslash: ''
+    });
+
+    mkdirp.sync(dirname(reportPath));
+    writeFileSync(reportPath, reportXML, 'utf8');
+  }
+}

src/dev/jest/setup/babel_polyfill.js

@@ -1,4 +1,4 @@
 // Note: In theory importing the polyfill should not be needed, as Babel should
 // include the necessary polyfills when using `babel-preset-env`, but for some
 // reason it did not work. See https://github.com/elastic/kibana/issues/14506
-import '../../babel-register/polyfill';
+import '../../../babel-register/polyfill';

src/dev/mocha/auto_junit_reporter.js

@@ -0,0 +1,18 @@
+import mocha from 'mocha';
+import { setupJunitReportGeneration } from './junit_report_generation';
+
+const MochaSpecReporter = mocha.reporters.spec;
+
+export function createAutoJunitReporter(junitReportOptions) {
+  return class createAutoJunitReporter {
+    constructor(runner, options) {
+      // setup a spec reporter for console output
+      new MochaSpecReporter(runner, options);
+
+      // in CI we also setup the Junit reporter
+      if (process.env.CI) {
+        setupJunitReportGeneration(runner, junitReportOptions);
+      }
+    }
+  };
+}

src/dev/mocha/index.js

@@ -0,0 +1,2 @@
+export { createAutoJunitReporter } from './auto_junit_reporter';
+export { setupJunitReportGeneration } from './junit_report_generation';

src/dev/mocha/junit_report_generation.js

@@ -0,0 +1,136 @@
+import { resolve, dirname, relative } from 'path';
+import { writeFileSync } from 'fs';
+import { inspect } from 'util';
+
+import mkdirp from 'mkdirp';
+import xmlBuilder from 'xmlbuilder';
+
+export function setupJunitReportGeneration(runner, options = {}) {
+  const {
+    reportName = 'Unnamed Mocha Tests',
+    rootDirectory = dirname(require.resolve('../../../package.json')),
+  } = options;
+
+  const rootSuite = runner.suite;
+  const isTestFailed = test => test.state === 'failed';
+  const isTestPending = test => !!test.pending;
+  const returnTrue = () => true;
+
+  const getDuration = (node) => (
+    node.startTime && node.endTime
+      ? ((node.endTime - node.startTime) / 1000).toFixed(3)
+      : null
+  );
+
+  const getTimestamp = (node) => (
+    node.startTime
+      ? new Date(node.startTime).toISOString().slice(0, -5)
+      : null
+  );
+
+  const countTests = (suite, filter = returnTrue) => (
+    suite.suites.reduce((sum, suite) => (
+      sum + countTests(suite, filter)
+    ), suite.tests.filter(filter).length)
+  );
+
+  const getFullTitle = node => {
+    const parentTitle = node.parent && getFullTitle(node.parent);
+    return parentTitle ? `${parentTitle} ${node.title}` : node.title;
+  };
+
+  const getPath = node => {
+    if (node.file) {
+      return relative(rootDirectory, node.file);
+    }
+
+    if (node.parent) {
+      return getPath(node.parent);
+    }
+
+    return 'unknown';
+  };
+
+  runner.on('start', () => {
+    rootSuite.startTime = Date.now();
+  });
+
+  runner.on('suite', (suite) => {
+    suite.startTime = Date.now();
+  });
+
+  runner.on('test', (test) => {
+    test.startTime = Date.now();
+  });
+
+  runner.on('test end', (test) => {
+    test.endTime = Date.now();
+  });
+
+  runner.on('suite end', (suite) => {
+    suite.endTime = Date.now();
+  });
+
+  runner.on('end', () => {
+    rootSuite.endTime = Date.now();
+    const builder = xmlBuilder.create(
+      'testsuites',
+      { encoding: 'utf-8' },
+      {},
+      { skipNullAttributes: true }
+    );
+
+    function addSuite(parent, suite) {
+      const attributes = {
+        name: suite.title,
+        timestamp: getTimestamp(suite),
+        time: getDuration(suite),
+        tests: countTests(suite),
+        failures: countTests(suite, isTestFailed),
+        skipped: countTests(suite, isTestPending),
+        file: suite.file
+      };
+
+      const el = suite === rootSuite
+        ? parent.att(attributes)
+        : parent.ele('testsuite', attributes);
+
+      suite.suites.forEach(childSuite => {
+        addSuite(el, childSuite);
+      });
+
+      suite.tests.forEach(test => {
+        addTest(el, test);
+      });
+    }
+
+    function addTest(parent, test) {
+      const el = parent.ele('testcase', {
+        name: getFullTitle(test),
+        classname: `${reportName}.${getPath(test).replace(/\./g, '·')}`,
+        time: getDuration(test),
+      });
+
+      if (isTestFailed(test)) {
+        el
+          .ele('failure')
+          .dat(inspect(test.err));
+      } else if (isTestPending(test)) {
+        el.ele('skipped');
+      }
+    }
+
+    addSuite(builder, rootSuite);
+
+    const reportPath = resolve(rootDirectory, `target/junit/${reportName}.xml`);
+    const reportXML = builder.end({
+      pretty: true,
+      indent: '  ',
+      newline: '\n',
+      spacebeforeslash: ''
+    });
+
+    mkdirp.sync(dirname(reportPath));
+    writeFileSync(reportPath, reportXML, 'utf8');
+  });
+}

src/dev/precommit_hook/casing_check_config.js

@@ -100,7 +100,6 @@ export const TEMPORARILY_IGNORED_PATHS = [
   'src/core_plugins/timelion/server/series_functions/__tests__/fixtures/tlConfig.js',
   'src/fixtures/config_upgrade_from_4.0.0_to_4.0.1-snapshot.json',
   'src/fixtures/vislib/mock_data/terms/_seriesMultiple.js',
-  'src/jest/babelTransform.js',
   'src/ui/i18n/__tests__/fixtures/translations/test_plugin_1/es-ES.json',
   'src/ui/public/angular-bootstrap/accordion/accordion-group.html',
   'src/ui/public/angular-bootstrap/bindHtml/bindHtml.js',

src/dev/tooling_log/tooling_log.js

@@ -1,7 +1,7 @@
 import { format } from 'util';
 import { PassThrough } from 'stream';
 
-import { magenta, yellow, red, blue, green, brightBlack } from 'ansicolors';
+import { magenta, yellow, red, blue, green, dim } from 'chalk';
 
 import { parseLogLevel } from './log_levels';
 
@@ -25,7 +25,7 @@ export function createToolingLog(initialLogLevelName = 'silent') {
 
     debug(...args) {
       if (!logLevel.flags.debug) return;
-      this.write(' %s ', brightBlack('debg'), format(...args));
+      this.write(' %s ', dim('debg'), format(...args));
     }
 
     info(...args) {

src/functional_test_runner/lib/config/schema.js

@@ -2,8 +2,6 @@ import { resolve, dirname } from 'path';
 
 import Joi from 'joi';
 
-import { ConsoleReporterProvider } from '../reporters';
-
 // valid pattern for ID
 // enforced camel-case identifiers for consistency
 const ID_PATTERN = /^[a-zA-Z0-9_]+$/;
@@ -62,7 +60,12 @@ export const schema = Joi.object().keys({
     slow: Joi.number().default(30000),
     timeout: Joi.number().default(INSPECTING ? Infinity : 120000),
     ui: Joi.string().default('bdd'),
-    reporterProvider: Joi.func().default(ConsoleReporterProvider),
+  }).default(),
+
+  junit: Joi.object().keys({
+    enabled: Joi.boolean().default(!!process.env.CI),
+    reportName: Joi.string(),
+    rootDirectory: Joi.string(),
   }).default(),
 
   users: Joi.object().pattern(

src/functional_test_runner/lib/mocha/reporter/colors.js

@@ -1,7 +1,7 @@
-import { brightBlack, green, yellow, red, brightWhite, brightCyan } from 'ansicolors';
+import { bold, dim, green, yellow, red, cyan } from 'chalk';
 
-export const suite = brightWhite;
-export const pending = brightCyan;
+export const suite = bold;
+export const pending = cyan;
 export const pass = green;
 export const fail = red;
 
@@ -14,6 +14,6 @@ export function speed(name, txt) {
     case 'slow':
       return red(txt);
     default:
-      return brightBlack(txt);
+      return dim(txt);
   }
 }

src/functional_test_runner/lib/mocha/reporter/index.js

@@ -0,0 +1 @@
+export { MochaReporterProvider } from './reporter';

src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -2,17 +2,19 @@ import { format } from 'util';
 
 import Mocha from 'mocha';
 
+import { setupJunitReportGeneration } from '../../../../dev';
 import * as colors from './colors';
 import * as symbols from './symbols';
 import { ms } from './ms';
 import { writeEpilogue } from './write_epilogue';
 
-export function ConsoleReporterProvider({ getService }) {
+export function MochaReporterProvider({ getService }) {
   const log = getService('log');
+  const config = getService('config');
 
   return class MochaReporter extends Mocha.reporters.Base {
-    constructor(runner) {
-      super(runner);
+    constructor(runner, options) {
+      super(runner, options);
       runner.on('start', this.onStart);
       runner.on('hook', this.onHookStart);
       runner.on('hook end', this.onHookEnd);
@@ -24,6 +26,13 @@ export function ConsoleReporterProvider({ getService }) {
       runner.on('test end', this.onTestEnd);
       runner.on('suite end', this.onSuiteEnd);
       runner.on('end', this.onEnd);
+
+      if (config.get('junit.enabled') && config.get('junit.reportName')) {
+        setupJunitReportGeneration(runner, {
+          reportName: config.get('junit.reportName'),
+          rootDirectory: config.get('junit.rootDirectory')
+        });
+      }
     }
 
     onStart = () => {

src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -1,6 +1,7 @@
 import Mocha from 'mocha';
 
 import { loadTestFiles } from './load_test_files';
+import { MochaReporterProvider } from './reporter';
 
 /**
  *  Instansiate mocha and load testfiles into it
@@ -16,8 +17,8 @@ export async function setupMocha(lifecycle, log, config, providers) {
   const mocha = new Mocha({
     ...config.get('mochaOpts'),
     reporter: await providers.loadExternalService(
-      'configured mocha reporter',
-      config.get('mochaOpts.reporterProvider')
+      'mocha reporter',
+      MochaReporterProvider
     )
   });
 

src/functional_test_runner/lib/reporters/console_reporter/index.js

@@ -1 +0,0 @@
-export { ConsoleReporterProvider } from './console_reporter';

src/functional_test_runner/lib/reporters/index.js

@@ -1 +0,0 @@
-export { ConsoleReporterProvider } from './console_reporter';

tasks/config/karma.js

@@ -1,6 +1,8 @@
 import { times } from 'lodash';
+import { resolve, dirname } from 'path';
 
 const TOTAL_CI_SHARDS = 4;
+const ROOT = dirname(require.resolve('../../package.json'));
 
 module.exports = function (grunt) {
   const config = {
@@ -18,7 +20,20 @@ module.exports = function (grunt) {
       browsers: ['<%= karmaBrowser %>'],
 
       // available reporters: https://npmjs.org/browse/keyword/karma-reporter
-      reporters: process.env.CI ? ['dots'] : ['progress'],
+      reporters: process.env.CI ? ['dots', 'junit'] : ['progress'],
+
+      junitReporter: {
+        outputFile: resolve(ROOT, 'target/junit/karma.xml'),
+        useBrowserName: false,
+        nameFormatter: (browser, result) => [
+          ...result.suite,
+          result.description
+        ].join(' '),
+        classNameFormatter: (browser, result) => {
+          const rootSuite = result.suite[0] || result.description;
+          return `Browser Unit Tests.${rootSuite.replace(/\./g, '·')}`;
+        }
+      },
 
       // list of files / patterns to load in the browser
       files: [

tasks/config/simplemocha.js

@@ -1,14 +1,17 @@
-module.exports = {
+import { createAutoJunitReporter } from '../../src/dev';
+
+export default {
   options: {
     timeout: 10000,
     slow: 5000,
     ignoreLeaks: false,
-    reporter: 'spec',
-    globals: ['nil']
+    reporter: createAutoJunitReporter({
+      reportName: 'Server Mocha Tests'
+    }),
+    globals: ['nil'],
   },
   all: {
     src: [
-      'test/mocha_setup.js',
       'test/**/__tests__/**/*.js',
       'src/**/__tests__/**/*.js',
       'tasks/**/__tests__/**/*.js',

test/api_integration/config.js

@@ -18,5 +18,8 @@ export default async function ({ readConfigFile }) {
       chance: ChanceProvider,
     },
     servers: commonConfig.get('servers'),
+    junit: {
+      reportName: 'API Integration Tests'
+    }
   };
 }

test/functional/config.js

@@ -107,5 +107,8 @@ export default async function ({ readConfigFile }) {
         hash: '/dev_tools/console',
       },
     },
+    junit: {
+      reportName: 'UI Functional Tests'
+    }
   };
 }

test/mocha.opts

@@ -1 +1 @@
---require test/mocha_setup.js
+--require src/babel-register

test/mocha_setup.js

@@ -1 +0,0 @@
-require('../src/babel-register');