Add jenkins:report task for test failures (#22682)

* Add jenkins:report task

* PR comments

package.json

@@ -210,6 +210,7 @@
     "@kbn/eslint-plugin-license-header": "link:packages/kbn-eslint-plugin-license-header",
     "@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
     "@kbn/test": "link:packages/kbn-test",
+    "@octokit/rest": "^15.10.0",
     "@types/angular": "^1.6.45",
     "@types/babel-core": "^6.25.5",
     "@types/bluebird": "^3.1.1",

src/dev/failed_tests/report.js

@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import xml2js from 'xml2js';
+import vfs from 'vinyl-fs';
+import es from 'event-stream';
+import { getGithubClient, markdownMetadata, paginate } from '../github_utils';
+import { find } from 'lodash';
+import stripAnsi from 'strip-ansi';
+
+const GITHUB_FLAKY_TEST_LABEL = 'failed-test';
+const GITHUB_OWNER = 'elastic';
+const GITHUB_REPO = 'kibana';
+const BUILD_URL = process.env.BUILD_URL;
+
+/**
+ * Parses junit XML files into JSON
+ */
+const mapXml = es.map((file, cb) => {
+  xml2js.parseString(file.contents.toString(), (err, result) => {
+    cb(null, result);
+  });
+});
+
+/**
+ * Filters all testsuites to find failed testcases
+ */
+const filterFailures = es.map((testSuite, cb) => {
+  const testFiles = testSuite.testsuites.testsuite;
+
+  const failures = testFiles.reduce((failures, testFile) => {
+    for (const testCase of testFile.testcase) {
+      if (testCase.failure) {
+        // unwrap xml weirdness
+        failures.push({
+          ...testCase.$,
+          // Strip ANSI color characters
+          failure: stripAnsi(testCase.failure[0])
+        });
+      }
+    }
+
+    return failures;
+  }, []);
+
+  console.log(`Found ${failures.length} test failures`);
+
+  cb(null, failures);
+});
+
+/**
+ * Creates and updates github issues for the given testcase failures.
+ */
+const updateGithubIssues = (githubClient, issues) => {
+  return es.map(async (failureCases, cb) => {
+
+    const issueOps = failureCases.map(async (failureCase) => {
+      const existingIssue = find(issues, (issue) => {
+        return markdownMetadata.get(issue.body, 'test.class') === failureCase.classname &&
+          markdownMetadata.get(issue.body, 'test.name') === failureCase.name;
+      });
+
+      if (existingIssue) {
+        // Increment failCount
+        const newCount = (markdownMetadata.get(existingIssue.body, 'test.failCount') || 0) + 1;
+        const newBody = markdownMetadata.set(existingIssue.body, 'test.failCount', newCount);
+
+        await githubClient.issues.edit({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          number: existingIssue.number,
+          state: 'open',  // Reopen issue if it was closed.
+          body: newBody
+        });
+
+        // Append a new comment
+        await githubClient.issues.createComment({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          number: existingIssue.number,
+          body: `New failure: [Jenkins Build](${BUILD_URL})`
+        });
+
+        console.log(`Updated issue ${existingIssue.html_url}, failCount: ${newCount}`);
+      } else {
+        let body = 'A test failed on a tracked branch\n' +
+          '```\n' + failureCase.failure + '\n```\n' +
+          `First failure: [Jenkins Build](${BUILD_URL})`;
+        body = markdownMetadata.set(body, {
+          'test.class': failureCase.classname,
+          'test.name': failureCase.name,
+          'test.failCount': 1
+        });
+
+        const newIssue = await githubClient.issues.create({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          title: `Failing test: ${failureCase.classname} - ${failureCase.name}`,
+          body: body,
+          labels: [GITHUB_FLAKY_TEST_LABEL]
+        });
+
+        console.log(`Created issue ${newIssue.data.html_url}`);
+      }
+    });
+
+    Promise
+      .all(issueOps)
+      .then(() => cb(null, failureCases))
+      .catch(e => cb(e));
+  });
+};
+
+/**
+ * Scans all junit XML files in ./target/junit/ and reports any found test failures to Github Issues.
+ */
+export async function reportFailedTests(done) {
+  const githubClient = getGithubClient();
+  const issues = await paginate(githubClient, githubClient.issues.getForRepo({
+    owner: GITHUB_OWNER,
+    repo: GITHUB_REPO,
+    labels: GITHUB_FLAKY_TEST_LABEL,
+    state: 'all',
+    per_page: 100
+  }));
+
+  vfs
+    .src(['./target/junit/**/*.xml'])
+    .pipe(mapXml)
+    .pipe(filterFailures)
+    .pipe(updateGithubIssues(githubClient, issues))
+    .on('done', done);
+}

src/dev/github_utils/index.js

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Octokit from '@octokit/rest';
+import { markdownMetadata } from './metadata';
+
+export { markdownMetadata };
+
+export function getGithubClient() {
+  const client = new Octokit();
+  client.authenticate({
+    type: 'token',
+    token: process.env.GITHUB_TOKEN
+  });
+
+  return client;
+}
+
+export async function paginate(client, promise) {
+  let response = await promise;
+  let { data } = response;
+  while (client.hasNextPage(response)) {
+    response = await client.getNextPage(response);
+    data = data.concat(response.data);
+  }
+  return data;
+}

src/dev/github_utils/metadata.js

@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const REGEX = /\n\n<!-- kibanaCiData = (.*) -->/;
+
+/**
+ * Allows retrieving and setting key/value pairs on a Github Issue. Keys and values must be JSON-serializable.
+ * Derived from https://github.com/probot/metadata/blob/6ae1523d5035ba727d09c0e7f77a6a154d9a4777/index.js
+ *
+ * `body` is a string that contains markdown and any existing metadata (eg. an issue or comment body)
+ * `prefix` is a string that can be used to namespace the metadata, defaults to `ci`.
+ *
+ *
+ * @notice
+ * This product bundles code based on probot-metadata@1.0.0 which is
+ * available under a "MIT" license.
+ *
+ * ISC License
+ *
+ * Copyright (c) 2017 Brandon Keepers
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+export const markdownMetadata = {
+  get(body, key = null, prefix = 'failed-test') {
+    const match = body.match(REGEX);
+
+    if (match) {
+      const data = JSON.parse(match[1])[prefix];
+      return key ? data && data[key] : data;
+    } else {
+      return null;
+    }
+  },
+
+  /**
+   * Set data on the body. Can either be set individually with `key` and `value` OR
+   */
+  set(body, key, value, prefix = 'failed-test') {
+    let newData = {};
+    // If second arg is an object, use all supplied values.
+    if (typeof key === 'object') {
+      newData = key;
+      prefix = value || prefix;  // need to move third arg to prefix.
+    } else {
+      newData[key] = value;
+    }
+
+    let data = {};
+
+    body = body.replace(REGEX, (_, json) => {
+      data = JSON.parse(json);
+      return '';
+    });
+
+    if (!data[prefix]) data[prefix] = {};
+
+    Object.assign(data[prefix], newData);
+
+    return `${body}\n\n<!-- kibanaCiData = ${JSON.stringify(data)} -->`;
+  }
+};

tasks/jenkins.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { reportFailedTests } from '../src/dev/failed_tests/report';
+
 module.exports = function (grunt) {
   grunt.registerTask('jenkins:docs', [
     'docker:docs'
@@ -44,4 +46,12 @@ module.exports = function (grunt) {
     'run:functionalTestsRelease',
     'run:pluginFunctionalTestsRelease',
   ]);
+
+  grunt.registerTask(
+    'jenkins:report',
+    'Reports failed tests found in junit xml files to Github issues',
+    function () {
+      reportFailedTests(this.async());
+    }
+  );
 };

test/scripts/jenkins_report_failed_tests.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+xvfb-run "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:report;

yarn.lock

@@ -139,6 +139,13 @@
   version "0.2.3"
   resolved "https://registry.yarnpkg.com/@elastic/ui-ace/-/ui-ace-0.2.3.tgz#5281aed47a79b7216c55542b0675e435692f20cd"
 
+"@gimenete/type-writer@^0.1.3":
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/@gimenete/type-writer/-/type-writer-0.1.3.tgz#2d4f26118b18d71f5b34ca24fdd6d1fd455c05b6"
+  dependencies:
+    camelcase "^5.0.0"
+    prettier "^1.13.7"
+
 "@kbn/babel-preset@link:packages/kbn-babel-preset":
   version "0.0.0"
   uid ""
@@ -194,6 +201,20 @@
     call-me-maybe "^1.0.1"
     glob-to-regexp "^0.3.0"
 
+"@octokit/rest@^15.10.0":
+  version "15.10.0"
+  resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-15.10.0.tgz#9baf7430e55edf1a1024c35ae72ed2f5fc6e90e9"
+  dependencies:
+    "@gimenete/type-writer" "^0.1.3"
+    before-after-hook "^1.1.0"
+    btoa-lite "^1.0.0"
+    debug "^3.1.0"
+    http-proxy-agent "^2.1.0"
+    https-proxy-agent "^2.2.0"
+    lodash "^4.17.4"
+    node-fetch "^2.1.1"
+    url-template "^2.0.8"
+
 "@samverschueren/stream-to-observable@^0.3.0":
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.0.tgz#ecdf48d532c58ea477acfcab80348424f8d0662f"
@@ -1968,6 +1989,10 @@ bcrypt-pbkdf@^1.0.0:
   dependencies:
     tweetnacl "^0.14.3"
 
+before-after-hook@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-1.1.0.tgz#83165e15a59460d13702cb8febd6a1807896db5a"
+
 better-assert@~1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522"
@@ -2285,6 +2310,10 @@ bser@^2.0.0:
   dependencies:
     node-int64 "^0.4.0"
 
+btoa-lite@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337"
+
 buffer-crc32@~0.2.3:
   version "0.2.13"
   resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
@@ -2505,6 +2534,10 @@ camelcase@^4.0.0, camelcase@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
 
+camelcase@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42"
+
 caniuse-api@^1.5.2:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-1.6.1.tgz#b534e7c734c4f81ec5fbe8aca2ad24354b962c6c"
@@ -6478,7 +6511,7 @@ https-browserify@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"
 
-https-proxy-agent@2.2.1, https-proxy-agent@^2.2.1:
+https-proxy-agent@2.2.1, https-proxy-agent@^2.2.0, https-proxy-agent@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0"
   dependencies:
@@ -9360,6 +9393,10 @@ node-fetch@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.1.tgz#369ca70b82f50c86496104a6c776d274f4e4a2d4"
 
+node-fetch@^2.1.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5"
+
 node-gyp@^3.3.1:
   version "3.6.2"
   resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.6.2.tgz#9bfbe54562286284838e750eac05295853fa1c60"
@@ -10577,6 +10614,10 @@ preserve@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b"
 
+prettier@^1.13.7:
+  version "1.14.2"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.14.2.tgz#0ac1c6e1a90baa22a62925f41963c841983282f9"
+
 prettier@^1.14.0:
   version "1.14.0"
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.14.0.tgz#847c235522035fd988100f1f43cf20a7d24f9372"
@@ -13935,6 +13976,10 @@ url-regex@^3.0.0:
   dependencies:
     ip-regex "^1.0.1"
 
+url-template@^2.0.8:
+  version "2.0.8"
+  resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21"
+
 url-to-options@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/url-to-options/-/url-to-options-1.0.1.tgz#1505a03a289a48cbd7a434efbaeec5055f5633a9"