Using re2 for Timelion regular expressions (#67416) (#68792)

* Revert "Revert "Using re2 for Timelion regular expressions (#55208)""

This reverts commit c90293d03f.

* Updating re2 to 1.14.0. Still need to update build patching

* Extract the gzip to the destination, supporting multiple extract methods

* Adding 'node' to jest's moduleFileExtensions

'node' is in the defaults, not sure why we aren't using the defaults...
https://jestjs.io/docs/en/configuration#modulefileextensions-arraystring

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.gitignore

@@ -4,6 +4,7 @@
 /.es
 .DS_Store
 .node_binaries
+.native_modules
 node_modules
 !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules
 !/src/dev/notice/__fixtures__/node_modules

package.json

@@ -233,6 +233,7 @@
     "pug": "^2.0.4",
     "query-string": "5.1.1",
     "raw-loader": "3.1.0",
+    "re2": "1.14.0",
     "react": "^16.12.0",
     "react-color": "^2.13.8",
     "react-dom": "^16.12.0",

src/dev/build/build_distributables.js

@@ -47,6 +47,7 @@ import {
   InstallDependenciesTask,
   BuildKibanaPlatformPluginsTask,
   OptimizeBuildTask,
+  PatchNativeModulesTask,
   RemovePackageJsonDepsTask,
   RemoveWorkspacesTask,
   TranspileBabelTask,
@@ -136,6 +137,7 @@ export async function buildDistributables(options) {
    * directories and perform platform-specific steps
    */
   await run(CreateArchivesSourcesTask);
+  await run(PatchNativeModulesTask);
   await run(CleanExtraBinScriptsTask);
   await run(CleanExtraBrowsersTask);
   await run(CleanNodeBuildsTask);

src/dev/build/lib/__tests__/fs.js

@@ -23,11 +23,12 @@ import { chmodSync, statSync } from 'fs';
 import del from 'del';
 import expect from '@kbn/expect';
 
-import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar } from '../fs';
+import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar, gunzip } from '../fs';
 
 const TMP = resolve(__dirname, '__tmp__');
 const FIXTURES = resolve(__dirname, 'fixtures');
 const FOO_TAR_PATH = resolve(FIXTURES, 'foo_dir.tar.gz');
+const FOO_GZIP_PATH = resolve(FIXTURES, 'foo.txt.gz');
 const BAR_TXT_PATH = resolve(FIXTURES, 'foo_dir/bar.txt');
 const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable');
 
@@ -323,4 +324,39 @@ describe('dev/build/lib/fs', () => {
       expect(await read(resolve(destination, 'foo/foo.txt'))).to.be('foo\n');
     });
   });
+
+  describe('gunzip()', () => {
+    it('rejects if source path is not absolute', async () => {
+      try {
+        await gunzip('foo/bar', '**/*', __dirname);
+        throw new Error('Expected gunzip() to reject');
+      } catch (error) {
+        assertNonAbsoluteError(error);
+      }
+    });
+
+    it('rejects if destination path is not absolute', async () => {
+      try {
+        await gunzip(__dirname, '**/*', 'foo/bar');
+        throw new Error('Expected gunzip() to reject');
+      } catch (error) {
+        assertNonAbsoluteError(error);
+      }
+    });
+
+    it('rejects if neither path is not absolute', async () => {
+      try {
+        await gunzip('foo/bar', '**/*', 'foo/bar');
+        throw new Error('Expected gunzip() to reject');
+      } catch (error) {
+        assertNonAbsoluteError(error);
+      }
+    });
+
+    it('extracts gzip from source into destination, creating destination if necessary', async () => {
+      const destination = resolve(TMP, 'z/y/x/v/u/t/foo.txt');
+      await gunzip(FOO_GZIP_PATH, destination);
+      expect(await read(resolve(destination))).to.be('foo\n');
+    });
+  });
 });

src/dev/build/lib/download.js

@@ -24,7 +24,7 @@ import chalk from 'chalk';
 import { createHash } from 'crypto';
 import Axios from 'axios';
 
-import { mkdirp } from '../../lib';
+import { mkdirp } from './fs';
 
 function tryUnlink(path) {
   try {

src/dev/build/lib/fs.js

@@ -195,6 +195,19 @@ export async function untar(source, destination, extractOptions = {}) {
   ]);
 }
 
+export async function gunzip(source, destination) {
+  assertAbsolute(source);
+  assertAbsolute(destination);
+
+  await mkdirAsync(dirname(destination), { recursive: true });
+
+  await createPromiseFromStreams([
+    fs.createReadStream(source),
+    createGunzip(),
+    fs.createWriteStream(destination),
+  ]);
+}
+
 export async function compress(type, options = {}, source, destination) {
   const output = fs.createWriteStream(destination);
   const archive = archiver(type, options.archiverOptions);

src/dev/build/lib/index.js

@@ -28,10 +28,12 @@ export {
   copyAll,
   getFileHash,
   untar,
+  gunzip,
   deleteAll,
   deleteEmptyFolders,
   compress,
   isFileAccessible,
 } from './fs';
+export { download } from './download';
 export { scanDelete } from './scan_delete';
 export { scanCopy } from './scan_copy';

src/dev/build/tasks/index.js

@@ -33,6 +33,7 @@ export * from './nodejs_modules';
 export * from './notice_file_task';
 export * from './optimize_task';
 export * from './os_packages';
+export * from './patch_native_modules_task';
 export * from './transpile_babel_task';
 export * from './transpile_scss_task';
 export * from './verify_env_task';

src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js

@@ -22,7 +22,7 @@ import expect from '@kbn/expect';
 
 import * as NodeShasumsNS from '../node_shasums';
 import * as NodeDownloadInfoNS from '../node_download_info';
-import * as DownloadNS from '../download';
+import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
 import { DownloadNodeBuildsTask } from '../download_node_builds_task';
 
 describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {

src/dev/build/tasks/nodejs/download_node_builds_task.js

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { download } from './download';
+import { download } from '../../lib';
 import { getNodeShasums } from './node_shasums';
 import { getNodeDownloadInfo } from './node_download_info';
 

src/dev/build/tasks/patch_native_modules_task.js

@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import fs from 'fs';
+import path from 'path';
+import util from 'util';
+import { deleteAll, download, gunzip, untar } from '../lib';
+
+const DOWNLOAD_DIRECTORY = '.native_modules';
+
+const packages = [
+  {
+    name: 're2',
+    version: '1.14.0',
+    destinationPath: 'node_modules/re2/build/Release/re2.node',
+    extractMethod: 'gunzip',
+    archives: {
+      darwin: {
+        url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/darwin-x64-64.gz',
+        sha256: '54c8386cb7cd53895cf379522114bfe82378e300e127e58d392ddd40a77e396f',
+      },
+      linux: {
+        url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/linux-x64-64.gz',
+        sha256: 'f54f059035e71a7ccb3fa201080e260c41d228d13a8247974b4bb157691b6757',
+      },
+      windows: {
+        url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/win32-x64-64.gz',
+        sha256: 'de708446a8b802f4634c2cfef097c2625a2811fdcd8133dfd7b7c485f966caa9',
+      },
+    },
+  },
+];
+
+async function getInstalledVersion(config, packageName) {
+  const packageJSONPath = config.resolveFromRepo(
+    path.join('node_modules', packageName, 'package.json')
+  );
+  const buffer = await util.promisify(fs.readFile)(packageJSONPath);
+  const packageJSON = JSON.parse(buffer);
+  return packageJSON.version;
+}
+
+async function patchModule(config, log, build, platform, pkg) {
+  const installedVersion = await getInstalledVersion(config, pkg.name);
+  if (installedVersion !== pkg.version) {
+    throw new Error(
+      `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
+    );
+  }
+  const platformName = platform.getName();
+  const archive = pkg.archives[platformName];
+  const archiveName = path.basename(archive.url);
+  const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, pkg.name, archiveName);
+  const extractPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
+  log.debug(`Patching ${pkg.name} binaries from ${archive.url} to ${extractPath}`);
+
+  await deleteAll([extractPath], log);
+  await download({
+    log,
+    url: archive.url,
+    destination: downloadPath,
+    sha256: archive.sha256,
+    retries: 3,
+  });
+  switch (pkg.extractMethod) {
+    case 'gunzip':
+      await gunzip(downloadPath, extractPath);
+      break;
+    case 'untar':
+      await untar(downloadPath, extractPath);
+      break;
+    default:
+      throw new Error(`Extract method of ${pkg.extractMethod} is not supported`);
+  }
+}
+
+export const PatchNativeModulesTask = {
+  description: 'Patching platform-specific native modules',
+  async run(config, log, build) {
+    for (const pkg of packages) {
+      await Promise.all(
+        config.getTargetPlatforms().map(async (platform) => {
+          await patchModule(config, log, build, platform, pkg);
+        })
+      );
+    }
+  },
+};

src/dev/jest/config.js

@@ -77,7 +77,7 @@ export default {
   ],
   coverageDirectory: '<rootDir>/target/kibana-coverage/jest',
   coverageReporters: ['html', 'text'],
-  moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
+  moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'],
   modulePathIgnorePatterns: ['__fixtures__/', 'target/'],
   testMatch: ['**/*.test.{js,ts,tsx}'],
   testPathIgnorePatterns: [

src/plugins/vis_type_timelion/server/series_functions/label.js

@@ -51,7 +51,10 @@ export default new Chainable('label', {
     const config = args.byName;
     return alter(args, function (eachSeries) {
       if (config.regex) {
-        eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label);
+        // not using a standard `import` so that if there's an issue with the re2 native module
+        // that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels
+        const RE2 = require('re2');
+        eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label);
       } else {
         eachSeries.label = config.label;
       }

x-pack/dev-tools/jest/create_jest_config.js

@@ -9,7 +9,7 @@ export function createJestConfig({ kibanaDirectory, rootDir, xPackKibanaDirector
   return {
     rootDir,
     roots: ['<rootDir>/plugins', '<rootDir>/legacy/plugins', '<rootDir>/legacy/server'],
-    moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
+    moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'],
     moduleNameMapper: {
       '@elastic/eui$': `${kibanaDirectory}/node_modules/@elastic/eui/test-env`,
       '@elastic/eui/lib/(.*)?': `${kibanaDirectory}/node_modules/@elastic/eui/test-env/$1`,

x-pack/test_utils/jest/config.js

@@ -29,7 +29,7 @@ export default {
   ],
   coverageDirectory: '<rootDir>/../target/kibana-coverage/jest',
   coverageReporters: ['html'],
-  moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
+  moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'],
   modulePathIgnorePatterns: ['__fixtures__/', 'target/'],
   testMatch: ['**/*.test.{js,ts,tsx}'],
   testPathIgnorePatterns: [

yarn.lock

@@ -12975,6 +12975,11 @@ env-paths@^1.0.0:
   resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-1.0.0.tgz#4168133b42bb05c38a35b1ae4397c8298ab369e0"
   integrity sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA=
 
+env-paths@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43"
+  integrity sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA==
+
 env-variable@0.0.x:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88"
@@ -21923,6 +21928,11 @@ nan@^2.12.1, nan@^2.13.2:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
   integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==
 
+nan@^2.14.1:
+  version "2.14.1"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01"
+  integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==
+
 nano-css@^5.2.1:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/nano-css/-/nano-css-5.2.1.tgz#73b8470fa40b028a134d3393ae36bbb34b9fa332"
@@ -22189,6 +22199,23 @@ node-gyp@^3.8.0:
     tar "^2.0.0"
     which "1"
 
+node-gyp@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-6.1.0.tgz#64e31c61a4695ad304c1d5b82cf6b7c79cc79f3f"
+  integrity sha512-h4A2zDlOujeeaaTx06r4Vy+8MZ1679lU+wbCKDS4ZtvY2A37DESo37oejIw0mtmR3+rvNwts5B6Kpt1KrNYdNw==
+  dependencies:
+    env-paths "^2.2.0"
+    glob "^7.1.4"
+    graceful-fs "^4.2.2"
+    mkdirp "^0.5.1"
+    nopt "^4.0.1"
+    npmlog "^4.1.2"
+    request "^2.88.0"
+    rimraf "^2.6.3"
+    semver "^5.7.1"
+    tar "^4.4.12"
+    which "^1.3.1"
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
@@ -22386,6 +22413,14 @@ nopt@^2.2.0:
   dependencies:
     abbrev "1"
 
+nopt@^4.0.1:
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
+  integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==
+  dependencies:
+    abbrev "1"
+    osenv "^0.1.4"
+
 nopt@~1.0.10:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee"
@@ -23080,7 +23115,7 @@ os-tmpdir@^1.0.0, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2:
   resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
   integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=
 
-osenv@0, osenv@^0.1.0:
+osenv@0, osenv@^0.1.0, osenv@^0.1.4:
   version "0.1.5"
   resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410"
   integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==
@@ -24964,6 +24999,14 @@ re-resizable@^6.1.1:
   dependencies:
     fast-memoize "^2.5.1"
 
+re2@1.14.0:
+  version "1.14.0"
+  resolved "https://registry.yarnpkg.com/re2/-/re2-1.14.0.tgz#727076590acfe868cf04e115a3a3f6c373ddd63b"
+  integrity sha512-TYogJmzni8zNVaw4gNOVORRTUaggLZwnMhJoTD0POKeACEoCxTWa9BAYehRnh3S1JUXIMEfcEUa7piiGEn71Zg==
+  dependencies:
+    nan "^2.14.1"
+    node-gyp "^6.1.0"
+
 react-ace@^5.5.0:
   version "5.10.0"
   resolved "https://registry.yarnpkg.com/react-ace/-/react-ace-5.10.0.tgz#e328b37ac52759f700be5afdb86ada2f5ec84c5e"
@@ -29405,7 +29448,7 @@ tar-stream@^2.1.0:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
-tar@4.4.13:
+tar@4.4.13, tar@^4.4.12:
   version "4.4.13"
   resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525"
   integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==