* Revert "Revert "Using re2 for Timelion regular expressions (#55208)""
This reverts commit c90293d03f
.
* Updating re2 to 1.14.0. Still need to update build patching
* Extract the gzip to the destination, supporting multiple extract methods
* Adding 'node' to jest's moduleFileExtensions
'node' is in the defaults, not sure why we aren't using the defaults...
https://jestjs.io/docs/en/configuration#modulefileextensions-arraystring
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
parent
0d0ea7030c
commit
388af030dc
18 changed files with 215 additions and 10 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -4,6 +4,7 @@
|
|||
/.es
|
||||
.DS_Store
|
||||
.node_binaries
|
||||
.native_modules
|
||||
node_modules
|
||||
!/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules
|
||||
!/src/dev/notice/__fixtures__/node_modules
|
||||
|
|
|
@ -233,6 +233,7 @@
|
|||
"pug": "^2.0.4",
|
||||
"query-string": "5.1.1",
|
||||
"raw-loader": "3.1.0",
|
||||
"re2": "1.14.0",
|
||||
"react": "^16.12.0",
|
||||
"react-color": "^2.13.8",
|
||||
"react-dom": "^16.12.0",
|
||||
|
|
|
@ -47,6 +47,7 @@ import {
|
|||
InstallDependenciesTask,
|
||||
BuildKibanaPlatformPluginsTask,
|
||||
OptimizeBuildTask,
|
||||
PatchNativeModulesTask,
|
||||
RemovePackageJsonDepsTask,
|
||||
RemoveWorkspacesTask,
|
||||
TranspileBabelTask,
|
||||
|
@ -136,6 +137,7 @@ export async function buildDistributables(options) {
|
|||
* directories and perform platform-specific steps
|
||||
*/
|
||||
await run(CreateArchivesSourcesTask);
|
||||
await run(PatchNativeModulesTask);
|
||||
await run(CleanExtraBinScriptsTask);
|
||||
await run(CleanExtraBrowsersTask);
|
||||
await run(CleanNodeBuildsTask);
|
||||
|
|
BIN
src/dev/build/lib/__tests__/fixtures/foo.txt.gz
Normal file
BIN
src/dev/build/lib/__tests__/fixtures/foo.txt.gz
Normal file
Binary file not shown.
|
@ -23,11 +23,12 @@ import { chmodSync, statSync } from 'fs';
|
|||
import del from 'del';
|
||||
import expect from '@kbn/expect';
|
||||
|
||||
import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar } from '../fs';
|
||||
import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar, gunzip } from '../fs';
|
||||
|
||||
const TMP = resolve(__dirname, '__tmp__');
|
||||
const FIXTURES = resolve(__dirname, 'fixtures');
|
||||
const FOO_TAR_PATH = resolve(FIXTURES, 'foo_dir.tar.gz');
|
||||
const FOO_GZIP_PATH = resolve(FIXTURES, 'foo.txt.gz');
|
||||
const BAR_TXT_PATH = resolve(FIXTURES, 'foo_dir/bar.txt');
|
||||
const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable');
|
||||
|
||||
|
@ -323,4 +324,39 @@ describe('dev/build/lib/fs', () => {
|
|||
expect(await read(resolve(destination, 'foo/foo.txt'))).to.be('foo\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gunzip()', () => {
|
||||
it('rejects if source path is not absolute', async () => {
|
||||
try {
|
||||
await gunzip('foo/bar', '**/*', __dirname);
|
||||
throw new Error('Expected gunzip() to reject');
|
||||
} catch (error) {
|
||||
assertNonAbsoluteError(error);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects if destination path is not absolute', async () => {
|
||||
try {
|
||||
await gunzip(__dirname, '**/*', 'foo/bar');
|
||||
throw new Error('Expected gunzip() to reject');
|
||||
} catch (error) {
|
||||
assertNonAbsoluteError(error);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects if neither path is not absolute', async () => {
|
||||
try {
|
||||
await gunzip('foo/bar', '**/*', 'foo/bar');
|
||||
throw new Error('Expected gunzip() to reject');
|
||||
} catch (error) {
|
||||
assertNonAbsoluteError(error);
|
||||
}
|
||||
});
|
||||
|
||||
it('extracts gzip from source into destination, creating destination if necessary', async () => {
|
||||
const destination = resolve(TMP, 'z/y/x/v/u/t/foo.txt');
|
||||
await gunzip(FOO_GZIP_PATH, destination);
|
||||
expect(await read(resolve(destination))).to.be('foo\n');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -24,7 +24,7 @@ import chalk from 'chalk';
|
|||
import { createHash } from 'crypto';
|
||||
import Axios from 'axios';
|
||||
|
||||
import { mkdirp } from '../../lib';
|
||||
import { mkdirp } from './fs';
|
||||
|
||||
function tryUnlink(path) {
|
||||
try {
|
|
@ -195,6 +195,19 @@ export async function untar(source, destination, extractOptions = {}) {
|
|||
]);
|
||||
}
|
||||
|
||||
export async function gunzip(source, destination) {
|
||||
assertAbsolute(source);
|
||||
assertAbsolute(destination);
|
||||
|
||||
await mkdirAsync(dirname(destination), { recursive: true });
|
||||
|
||||
await createPromiseFromStreams([
|
||||
fs.createReadStream(source),
|
||||
createGunzip(),
|
||||
fs.createWriteStream(destination),
|
||||
]);
|
||||
}
|
||||
|
||||
export async function compress(type, options = {}, source, destination) {
|
||||
const output = fs.createWriteStream(destination);
|
||||
const archive = archiver(type, options.archiverOptions);
|
||||
|
|
|
@ -28,10 +28,12 @@ export {
|
|||
copyAll,
|
||||
getFileHash,
|
||||
untar,
|
||||
gunzip,
|
||||
deleteAll,
|
||||
deleteEmptyFolders,
|
||||
compress,
|
||||
isFileAccessible,
|
||||
} from './fs';
|
||||
export { download } from './download';
|
||||
export { scanDelete } from './scan_delete';
|
||||
export { scanCopy } from './scan_copy';
|
||||
|
|
|
@ -33,6 +33,7 @@ export * from './nodejs_modules';
|
|||
export * from './notice_file_task';
|
||||
export * from './optimize_task';
|
||||
export * from './os_packages';
|
||||
export * from './patch_native_modules_task';
|
||||
export * from './transpile_babel_task';
|
||||
export * from './transpile_scss_task';
|
||||
export * from './verify_env_task';
|
||||
|
|
|
@ -22,7 +22,7 @@ import expect from '@kbn/expect';
|
|||
|
||||
import * as NodeShasumsNS from '../node_shasums';
|
||||
import * as NodeDownloadInfoNS from '../node_download_info';
|
||||
import * as DownloadNS from '../download';
|
||||
import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly
|
||||
import { DownloadNodeBuildsTask } from '../download_node_builds_task';
|
||||
|
||||
describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => {
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import { download } from './download';
|
||||
import { download } from '../../lib';
|
||||
import { getNodeShasums } from './node_shasums';
|
||||
import { getNodeDownloadInfo } from './node_download_info';
|
||||
|
||||
|
|
103
src/dev/build/tasks/patch_native_modules_task.js
Normal file
103
src/dev/build/tasks/patch_native_modules_task.js
Normal file
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import util from 'util';
|
||||
import { deleteAll, download, gunzip, untar } from '../lib';
|
||||
|
||||
const DOWNLOAD_DIRECTORY = '.native_modules';
|
||||
|
||||
const packages = [
|
||||
{
|
||||
name: 're2',
|
||||
version: '1.14.0',
|
||||
destinationPath: 'node_modules/re2/build/Release/re2.node',
|
||||
extractMethod: 'gunzip',
|
||||
archives: {
|
||||
darwin: {
|
||||
url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/darwin-x64-64.gz',
|
||||
sha256: '54c8386cb7cd53895cf379522114bfe82378e300e127e58d392ddd40a77e396f',
|
||||
},
|
||||
linux: {
|
||||
url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/linux-x64-64.gz',
|
||||
sha256: 'f54f059035e71a7ccb3fa201080e260c41d228d13a8247974b4bb157691b6757',
|
||||
},
|
||||
windows: {
|
||||
url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/win32-x64-64.gz',
|
||||
sha256: 'de708446a8b802f4634c2cfef097c2625a2811fdcd8133dfd7b7c485f966caa9',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
async function getInstalledVersion(config, packageName) {
|
||||
const packageJSONPath = config.resolveFromRepo(
|
||||
path.join('node_modules', packageName, 'package.json')
|
||||
);
|
||||
const buffer = await util.promisify(fs.readFile)(packageJSONPath);
|
||||
const packageJSON = JSON.parse(buffer);
|
||||
return packageJSON.version;
|
||||
}
|
||||
|
||||
async function patchModule(config, log, build, platform, pkg) {
|
||||
const installedVersion = await getInstalledVersion(config, pkg.name);
|
||||
if (installedVersion !== pkg.version) {
|
||||
throw new Error(
|
||||
`Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}`
|
||||
);
|
||||
}
|
||||
const platformName = platform.getName();
|
||||
const archive = pkg.archives[platformName];
|
||||
const archiveName = path.basename(archive.url);
|
||||
const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, pkg.name, archiveName);
|
||||
const extractPath = build.resolvePathForPlatform(platform, pkg.destinationPath);
|
||||
log.debug(`Patching ${pkg.name} binaries from ${archive.url} to ${extractPath}`);
|
||||
|
||||
await deleteAll([extractPath], log);
|
||||
await download({
|
||||
log,
|
||||
url: archive.url,
|
||||
destination: downloadPath,
|
||||
sha256: archive.sha256,
|
||||
retries: 3,
|
||||
});
|
||||
switch (pkg.extractMethod) {
|
||||
case 'gunzip':
|
||||
await gunzip(downloadPath, extractPath);
|
||||
break;
|
||||
case 'untar':
|
||||
await untar(downloadPath, extractPath);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Extract method of ${pkg.extractMethod} is not supported`);
|
||||
}
|
||||
}
|
||||
|
||||
export const PatchNativeModulesTask = {
|
||||
description: 'Patching platform-specific native modules',
|
||||
async run(config, log, build) {
|
||||
for (const pkg of packages) {
|
||||
await Promise.all(
|
||||
config.getTargetPlatforms().map(async (platform) => {
|
||||
await patchModule(config, log, build, platform, pkg);
|
||||
})
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
|
@ -77,7 +77,7 @@ export default {
|
|||
],
|
||||
coverageDirectory: '<rootDir>/target/kibana-coverage/jest',
|
||||
coverageReporters: ['html', 'text'],
|
||||
moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
|
||||
moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'],
|
||||
modulePathIgnorePatterns: ['__fixtures__/', 'target/'],
|
||||
testMatch: ['**/*.test.{js,ts,tsx}'],
|
||||
testPathIgnorePatterns: [
|
||||
|
|
|
@ -51,7 +51,10 @@ export default new Chainable('label', {
|
|||
const config = args.byName;
|
||||
return alter(args, function (eachSeries) {
|
||||
if (config.regex) {
|
||||
eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label);
|
||||
// not using a standard `import` so that if there's an issue with the re2 native module
|
||||
// that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels
|
||||
const RE2 = require('re2');
|
||||
eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label);
|
||||
} else {
|
||||
eachSeries.label = config.label;
|
||||
}
|
||||
|
|
|
@ -9,7 +9,7 @@ export function createJestConfig({ kibanaDirectory, rootDir, xPackKibanaDirector
|
|||
return {
|
||||
rootDir,
|
||||
roots: ['<rootDir>/plugins', '<rootDir>/legacy/plugins', '<rootDir>/legacy/server'],
|
||||
moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
|
||||
moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'],
|
||||
moduleNameMapper: {
|
||||
'@elastic/eui$': `${kibanaDirectory}/node_modules/@elastic/eui/test-env`,
|
||||
'@elastic/eui/lib/(.*)?': `${kibanaDirectory}/node_modules/@elastic/eui/test-env/$1`,
|
||||
|
|
|
@ -29,7 +29,7 @@ export default {
|
|||
],
|
||||
coverageDirectory: '<rootDir>/../target/kibana-coverage/jest',
|
||||
coverageReporters: ['html'],
|
||||
moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
|
||||
moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'],
|
||||
modulePathIgnorePatterns: ['__fixtures__/', 'target/'],
|
||||
testMatch: ['**/*.test.{js,ts,tsx}'],
|
||||
testPathIgnorePatterns: [
|
||||
|
|
47
yarn.lock
47
yarn.lock
|
@ -12975,6 +12975,11 @@ env-paths@^1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-1.0.0.tgz#4168133b42bb05c38a35b1ae4397c8298ab369e0"
|
||||
integrity sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA=
|
||||
|
||||
env-paths@^2.2.0:
|
||||
version "2.2.0"
|
||||
resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43"
|
||||
integrity sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA==
|
||||
|
||||
env-variable@0.0.x:
|
||||
version "0.0.5"
|
||||
resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88"
|
||||
|
@ -21923,6 +21928,11 @@ nan@^2.12.1, nan@^2.13.2:
|
|||
resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
|
||||
integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==
|
||||
|
||||
nan@^2.14.1:
|
||||
version "2.14.1"
|
||||
resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01"
|
||||
integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==
|
||||
|
||||
nano-css@^5.2.1:
|
||||
version "5.2.1"
|
||||
resolved "https://registry.yarnpkg.com/nano-css/-/nano-css-5.2.1.tgz#73b8470fa40b028a134d3393ae36bbb34b9fa332"
|
||||
|
@ -22189,6 +22199,23 @@ node-gyp@^3.8.0:
|
|||
tar "^2.0.0"
|
||||
which "1"
|
||||
|
||||
node-gyp@^6.1.0:
|
||||
version "6.1.0"
|
||||
resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-6.1.0.tgz#64e31c61a4695ad304c1d5b82cf6b7c79cc79f3f"
|
||||
integrity sha512-h4A2zDlOujeeaaTx06r4Vy+8MZ1679lU+wbCKDS4ZtvY2A37DESo37oejIw0mtmR3+rvNwts5B6Kpt1KrNYdNw==
|
||||
dependencies:
|
||||
env-paths "^2.2.0"
|
||||
glob "^7.1.4"
|
||||
graceful-fs "^4.2.2"
|
||||
mkdirp "^0.5.1"
|
||||
nopt "^4.0.1"
|
||||
npmlog "^4.1.2"
|
||||
request "^2.88.0"
|
||||
rimraf "^2.6.3"
|
||||
semver "^5.7.1"
|
||||
tar "^4.4.12"
|
||||
which "^1.3.1"
|
||||
|
||||
node-int64@^0.4.0:
|
||||
version "0.4.0"
|
||||
resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
|
||||
|
@ -22386,6 +22413,14 @@ nopt@^2.2.0:
|
|||
dependencies:
|
||||
abbrev "1"
|
||||
|
||||
nopt@^4.0.1:
|
||||
version "4.0.3"
|
||||
resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
|
||||
integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==
|
||||
dependencies:
|
||||
abbrev "1"
|
||||
osenv "^0.1.4"
|
||||
|
||||
nopt@~1.0.10:
|
||||
version "1.0.10"
|
||||
resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee"
|
||||
|
@ -23080,7 +23115,7 @@ os-tmpdir@^1.0.0, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2:
|
|||
resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
|
||||
integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=
|
||||
|
||||
osenv@0, osenv@^0.1.0:
|
||||
osenv@0, osenv@^0.1.0, osenv@^0.1.4:
|
||||
version "0.1.5"
|
||||
resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410"
|
||||
integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==
|
||||
|
@ -24964,6 +24999,14 @@ re-resizable@^6.1.1:
|
|||
dependencies:
|
||||
fast-memoize "^2.5.1"
|
||||
|
||||
re2@1.14.0:
|
||||
version "1.14.0"
|
||||
resolved "https://registry.yarnpkg.com/re2/-/re2-1.14.0.tgz#727076590acfe868cf04e115a3a3f6c373ddd63b"
|
||||
integrity sha512-TYogJmzni8zNVaw4gNOVORRTUaggLZwnMhJoTD0POKeACEoCxTWa9BAYehRnh3S1JUXIMEfcEUa7piiGEn71Zg==
|
||||
dependencies:
|
||||
nan "^2.14.1"
|
||||
node-gyp "^6.1.0"
|
||||
|
||||
react-ace@^5.5.0:
|
||||
version "5.10.0"
|
||||
resolved "https://registry.yarnpkg.com/react-ace/-/react-ace-5.10.0.tgz#e328b37ac52759f700be5afdb86ada2f5ec84c5e"
|
||||
|
@ -29405,7 +29448,7 @@ tar-stream@^2.1.0:
|
|||
inherits "^2.0.3"
|
||||
readable-stream "^3.1.1"
|
||||
|
||||
tar@4.4.13:
|
||||
tar@4.4.13, tar@^4.4.12:
|
||||
version "4.4.13"
|
||||
resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525"
|
||||
integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==
|
||||
|
|
Loading…
Reference in a new issue