[7.x] [optimizer] validate the syntax of bundled node_modules… (#61904)

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
# Conflicts:
#	renovate.json5
This commit is contained in:
Spencer 2020-03-30 18:30:57 -07:00 committed by GitHub
parent 4b567b0296
commit 6065f58fa9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 442 additions and 15 deletions

View file

@ -14,9 +14,12 @@
"@kbn/babel-preset": "1.0.0",
"@kbn/dev-utils": "1.0.0",
"@kbn/ui-shared-deps": "1.0.0",
"@types/estree": "^0.0.44",
"@types/loader-utils": "^1.1.3",
"@types/watchpack": "^1.1.5",
"@types/webpack": "^4.41.3",
"acorn": "^7.1.1",
"acorn-walk": "^7.1.1",
"autoprefixer": "^9.7.4",
"babel-loader": "^8.0.6",
"clean-webpack-plugin": "^3.0.0",

View file

@ -4,6 +4,7 @@ exports[`parseDirPath() parses / 1`] = `
Object {
"dirs": Array [],
"filename": undefined,
"query": undefined,
"root": "/",
}
`;
@ -14,6 +15,7 @@ Object {
"foo",
],
"filename": undefined,
"query": undefined,
"root": "/",
}
`;
@ -26,6 +28,7 @@ Object {
"baz",
],
"filename": undefined,
"query": undefined,
"root": "/",
}
`;
@ -38,6 +41,7 @@ Object {
"baz",
],
"filename": undefined,
"query": undefined,
"root": "/",
}
`;
@ -46,6 +50,7 @@ exports[`parseDirPath() parses c:\\ 1`] = `
Object {
"dirs": Array [],
"filename": undefined,
"query": undefined,
"root": "c:",
}
`;
@ -56,6 +61,7 @@ Object {
"foo",
],
"filename": undefined,
"query": undefined,
"root": "c:",
}
`;
@ -68,6 +74,7 @@ Object {
"baz",
],
"filename": undefined,
"query": undefined,
"root": "c:",
}
`;
@ -80,6 +87,7 @@ Object {
"baz",
],
"filename": undefined,
"query": undefined,
"root": "c:",
}
`;
@ -88,6 +96,7 @@ exports[`parseFilePath() parses /foo 1`] = `
Object {
"dirs": Array [],
"filename": "foo",
"query": undefined,
"root": "/",
}
`;
@ -99,6 +108,7 @@ Object {
"bar",
],
"filename": "baz",
"query": undefined,
"root": "/",
}
`;
@ -110,6 +120,36 @@ Object {
"bar",
],
"filename": "baz.json",
"query": undefined,
"root": "/",
}
`;
exports[`parseFilePath() parses /foo/bar/baz.json?light 1`] = `
Object {
"dirs": Array [
"foo",
"bar",
],
"filename": "baz.json",
"query": Object {
"light": "",
},
"root": "/",
}
`;
exports[`parseFilePath() parses /foo/bar/baz.json?light=true&dark=false 1`] = `
Object {
"dirs": Array [
"foo",
"bar",
],
"filename": "baz.json",
"query": Object {
"dark": "false",
"light": "true",
},
"root": "/",
}
`;
@ -121,6 +161,7 @@ Object {
"bar",
],
"filename": "baz.json",
"query": undefined,
"root": "c:",
}
`;
@ -129,6 +170,7 @@ exports[`parseFilePath() parses c:\\foo 1`] = `
Object {
"dirs": Array [],
"filename": "foo",
"query": undefined,
"root": "c:",
}
`;
@ -140,6 +182,7 @@ Object {
"bar",
],
"filename": "baz",
"query": undefined,
"root": "c:",
}
`;
@ -151,6 +194,36 @@ Object {
"bar",
],
"filename": "baz.json",
"query": undefined,
"root": "c:",
}
`;
exports[`parseFilePath() parses c:\\foo\\bar\\baz.json?dark 1`] = `
Object {
"dirs": Array [
"foo",
"bar",
],
"filename": "baz.json",
"query": Object {
"dark": "",
},
"root": "c:",
}
`;
exports[`parseFilePath() parses c:\\foo\\bar\\baz.json?dark=true&light=false 1`] = `
Object {
"dirs": Array [
"foo",
"bar",
],
"filename": "baz.json",
"query": Object {
"dark": "true",
"light": "false",
},
"root": "c:",
}
`;

View file

@ -0,0 +1,194 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import estree from 'estree';
export interface DisallowedSyntaxCheck {
name: string;
nodeType: estree.Node['type'] | Array<estree.Node['type']>;
test?: (n: any) => boolean | void;
}
export const checks: DisallowedSyntaxCheck[] = [
/**
* es2015
*/
// https://github.com/estree/estree/blob/master/es2015.md#functions
{
name: '[es2015] generator function',
nodeType: ['FunctionDeclaration', 'FunctionExpression'],
test: (n: estree.FunctionDeclaration | estree.FunctionExpression) => !!n.generator,
},
// https://github.com/estree/estree/blob/master/es2015.md#forofstatement
{
name: '[es2015] for-of statement',
nodeType: 'ForOfStatement',
},
// https://github.com/estree/estree/blob/master/es2015.md#variabledeclaration
{
name: '[es2015] let/const variable declaration',
nodeType: 'VariableDeclaration',
test: (n: estree.VariableDeclaration) => n.kind === 'let' || n.kind === 'const',
},
// https://github.com/estree/estree/blob/master/es2015.md#expressions
{
name: '[es2015] `super`',
nodeType: 'Super',
},
// https://github.com/estree/estree/blob/master/es2015.md#expressions
{
name: '[es2015] ...spread',
nodeType: 'SpreadElement',
},
// https://github.com/estree/estree/blob/master/es2015.md#arrowfunctionexpression
{
name: '[es2015] arrow function expression',
nodeType: 'ArrowFunctionExpression',
},
// https://github.com/estree/estree/blob/master/es2015.md#yieldexpression
{
name: '[es2015] `yield` expression',
nodeType: 'YieldExpression',
},
// https://github.com/estree/estree/blob/master/es2015.md#templateliteral
{
name: '[es2015] template literal',
nodeType: 'TemplateLiteral',
},
// https://github.com/estree/estree/blob/master/es2015.md#patterns
{
name: '[es2015] destructuring',
nodeType: ['ObjectPattern', 'ArrayPattern', 'AssignmentPattern'],
},
// https://github.com/estree/estree/blob/master/es2015.md#classes
{
name: '[es2015] class',
nodeType: [
'ClassDeclaration',
'ClassExpression',
'ClassBody',
'MethodDefinition',
'MetaProperty',
],
},
/**
* es2016
*/
{
name: '[es2016] exponent operator',
nodeType: 'BinaryExpression',
test: (n: estree.BinaryExpression) => n.operator === '**',
},
{
name: '[es2016] exponent assignment',
nodeType: 'AssignmentExpression',
test: (n: estree.AssignmentExpression) => n.operator === '**=',
},
/**
* es2017
*/
// https://github.com/estree/estree/blob/master/es2017.md#function
{
name: '[es2017] async function',
nodeType: ['FunctionDeclaration', 'FunctionExpression'],
test: (n: estree.FunctionDeclaration | estree.FunctionExpression) => n.async,
},
// https://github.com/estree/estree/blob/master/es2017.md#awaitexpression
{
name: '[es2017] await expression',
nodeType: 'AwaitExpression',
},
/**
* es2018
*/
// https://github.com/estree/estree/blob/master/es2018.md#statements
{
name: '[es2018] for-await-of statements',
nodeType: 'ForOfStatement',
test: (n: estree.ForOfStatement) => n.await,
},
// https://github.com/estree/estree/blob/master/es2018.md#expressions
{
name: '[es2018] object spread properties',
nodeType: 'ObjectExpression',
test: (n: estree.ObjectExpression) => n.properties.some(p => p.type === 'SpreadElement'),
},
// https://github.com/estree/estree/blob/master/es2018.md#template-literals
{
name: '[es2018] tagged template literal with invalid escape',
nodeType: 'TemplateElement',
test: (n: estree.TemplateElement) => n.value.cooked === null,
},
// https://github.com/estree/estree/blob/master/es2018.md#patterns
{
name: '[es2018] rest properties',
nodeType: 'ObjectPattern',
test: (n: estree.ObjectPattern) => n.properties.some(p => p.type === 'RestElement'),
},
/**
* es2019
*/
// https://github.com/estree/estree/blob/master/es2019.md#catchclause
{
name: '[es2019] catch clause without a binding',
nodeType: 'CatchClause',
test: (n: estree.CatchClause) => !n.param,
},
/**
* es2020
*/
// https://github.com/estree/estree/blob/master/es2020.md#bigintliteral
{
name: '[es2020] bigint literal',
nodeType: 'Literal',
test: (n: estree.Literal) => typeof n.value === 'bigint',
},
/**
* webpack transforms import/export in order to support tree shaking and async imports
*
* // https://github.com/estree/estree/blob/master/es2020.md#importexpression
* {
* name: '[es2020] import expression',
* nodeType: 'ImportExpression',
* },
* // https://github.com/estree/estree/blob/master/es2020.md#exportalldeclaration
* {
* name: '[es2020] export all declaration',
* nodeType: 'ExportAllDeclaration',
* },
*
*/
];
export const checksByNodeType = new Map<estree.Node['type'], DisallowedSyntaxCheck[]>();
for (const check of checks) {
const nodeTypes = Array.isArray(check.nodeType) ? check.nodeType : [check.nodeType];
for (const nodeType of nodeTypes) {
if (!checksByNodeType.has(nodeType)) {
checksByNodeType.set(nodeType, []);
}
checksByNodeType.get(nodeType)!.push(check);
}
}

View file

@ -0,0 +1,73 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import webpack from 'webpack';
import acorn from 'acorn';
import * as AcornWalk from 'acorn-walk';
import { checksByNodeType, DisallowedSyntaxCheck } from './disallowed_syntax';
import { parseFilePath } from '../parse_path';
export class DisallowedSyntaxPlugin {
apply(compiler: webpack.Compiler) {
compiler.hooks.normalModuleFactory.tap(DisallowedSyntaxPlugin.name, factory => {
factory.hooks.parser.for('javascript/auto').tap(DisallowedSyntaxPlugin.name, parser => {
parser.hooks.program.tap(DisallowedSyntaxPlugin.name, (program: acorn.Node) => {
const module = parser.state?.current;
if (!module || !module.resource) {
return;
}
const resource: string = module.resource;
const { dirs } = parseFilePath(resource);
if (!dirs.includes('node_modules')) {
return;
}
const failedChecks = new Set<DisallowedSyntaxCheck>();
AcornWalk.full(program, node => {
const checks = checksByNodeType.get(node.type as any);
if (!checks) {
return;
}
for (const check of checks) {
if (!check.test || check.test(node)) {
failedChecks.add(check);
}
}
});
if (!failedChecks.size) {
return;
}
// throw an error to trigger a parse failure, causing this module to be reported as invalid
throw new Error(
`disallowed syntax found in file ${resource}:\n - ${Array.from(failedChecks)
.map(c => c.name)
.join('\n - ')}`
);
});
});
});
}
}

View file

@ -0,0 +1,20 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export * from './disallowed_syntax_plugin';

View file

@ -26,3 +26,5 @@ export * from './ts_helpers';
export * from './rxjs_helpers';
export * from './array_helpers';
export * from './event_stream_helpers';
export * from './disallowed_syntax_plugin';
export * from './parse_path';

View file

@ -21,7 +21,15 @@ import { parseFilePath, parseDirPath } from './parse_path';
const DIRS = ['/', '/foo/bar/baz/', 'c:\\', 'c:\\foo\\bar\\baz\\'];
const AMBIGUOUS = ['/foo', '/foo/bar/baz', 'c:\\foo', 'c:\\foo\\bar\\baz'];
const FILES = ['/foo/bar/baz.json', 'c:/foo/bar/baz.json', 'c:\\foo\\bar\\baz.json'];
const FILES = [
'/foo/bar/baz.json',
'c:/foo/bar/baz.json',
'c:\\foo\\bar\\baz.json',
'/foo/bar/baz.json?light',
'/foo/bar/baz.json?light=true&dark=false',
'c:\\foo\\bar\\baz.json?dark',
'c:\\foo\\bar\\baz.json?dark=true&light=false',
];
describe('parseFilePath()', () => {
it.each([...FILES, ...AMBIGUOUS])('parses %s', path => {

View file

@ -18,6 +18,7 @@
*/
import normalizePath from 'normalize-path';
import Qs from 'querystring';
/**
* Parse an absolute path, supporting normalized paths from webpack,
@ -33,11 +34,19 @@ export function parseDirPath(path: string) {
}
export function parseFilePath(path: string) {
const normalized = normalizePath(path);
let normalized = normalizePath(path);
let query;
const queryIndex = normalized.indexOf('?');
if (queryIndex !== -1) {
query = Qs.parse(normalized.slice(queryIndex + 1));
normalized = normalized.slice(0, queryIndex);
}
const [root, ...others] = normalized.split('/');
return {
root: root === '' ? '/' : root,
dirs: others.slice(0, -1),
query,
filename: others[others.length - 1] || undefined,
};
}

View file

@ -20,3 +20,4 @@
export { OptimizerConfig } from './optimizer';
export * from './run_optimizer';
export * from './log_optimizer_state';
export * from './common/disallowed_syntax_plugin';

View file

@ -27,10 +27,17 @@ import webpack, { Stats } from 'webpack';
import * as Rx from 'rxjs';
import { mergeMap, map, mapTo, takeUntil } from 'rxjs/operators';
import { CompilerMsgs, CompilerMsg, maybeMap, Bundle, WorkerConfig, ascending } from '../common';
import {
CompilerMsgs,
CompilerMsg,
maybeMap,
Bundle,
WorkerConfig,
ascending,
parseFilePath,
} from '../common';
import { getWebpackConfig } from './webpack.config';
import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers';
import { parseFilePath } from './parse_path';
import {
isExternalModule,
isNormalModule,

View file

@ -29,8 +29,7 @@ import webpackMerge from 'webpack-merge';
import { CleanWebpackPlugin } from 'clean-webpack-plugin';
import * as SharedDeps from '@kbn/ui-shared-deps';
import { Bundle, WorkerConfig } from '../common';
import { parseDirPath } from './parse_path';
import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common';
const PUBLIC_PATH_PLACEHOLDER = '__REPLACE_WITH_PUBLIC_PATH__';
const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');
@ -75,7 +74,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) {
...SharedDeps.externals,
},
plugins: [new CleanWebpackPlugin()],
plugins: [new CleanWebpackPlugin(), new DisallowedSyntaxPlugin()],
module: {
// no parse rules for a few known large packages which have no require() statements

View file

@ -18,7 +18,6 @@
*/
import webpack from 'webpack';
import { defaults } from 'lodash';
// @ts-ignore
import Stats from 'webpack/lib/Stats';
@ -55,12 +54,14 @@ const STATS_WARNINGS_FILTER = new RegExp(
);
export function failedStatsToErrorMessage(stats: webpack.Stats) {
const details = stats.toString(
defaults(
{ colors: true, warningsFilter: STATS_WARNINGS_FILTER },
Stats.presetToOptions('minimal')
)
);
const details = stats.toString({
...Stats.presetToOptions('minimal'),
colors: true,
warningsFilter: STATS_WARNINGS_FILTER,
errors: true,
errorDetails: true,
moduleTrace: true,
});
return `Optimizations failure.\n${details.split('\n').join('\n ')}`;
}

View file

@ -198,6 +198,7 @@ export default () =>
}),
workers: Joi.number().min(1),
profile: Joi.boolean().default(false),
validateSyntaxOfNodeModules: Joi.boolean().default(true),
}).default(),
status: Joi.object({
allowAnonymous: Joi.boolean().default(false),

View file

@ -73,6 +73,7 @@ export class UiBundlesController {
this._workingDir = config.get('optimize.bundleDir');
this._env = config.get('env.name');
this._validateSyntaxOfNodeModules = config.get('optimize.validateSyntaxOfNodeModules');
this._context = {
env: config.get('env.name'),
sourceMaps: config.get('optimize.sourceMaps'),
@ -135,6 +136,10 @@ export class UiBundlesController {
return this._env === 'development';
}
shouldValidateSyntaxOfNodeModules() {
return !!this._validateSyntaxOfNodeModules;
}
getWebpackPluginProviders() {
return this._webpackPluginProviders || [];
}

View file

@ -28,6 +28,7 @@ import * as UiSharedDeps from '@kbn/ui-shared-deps';
function generateDLL(config) {
const {
dllAlias,
dllValidateSyntax,
dllNoParseRules,
dllContext,
dllEntry,
@ -44,6 +45,22 @@ function generateDLL(config) {
const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset');
const BABEL_EXCLUDE_RE = [/[\/\\](webpackShims|node_modules|bower_components)[\/\\]/];
/**
* Wrap plugin loading in a function so that we can require
* `@kbn/optimizer` only when absolutely necessary since we
* don't ship this package in the distributable but this code
* is still shipped, though it's not used.
*/
const getValidateSyntaxPlugins = () => {
if (!dllValidateSyntax) {
return [];
}
// only require @kbn/optimizer
const { DisallowedSyntaxPlugin } = require('@kbn/optimizer');
return [new DisallowedSyntaxPlugin()];
};
return {
entry: dllEntry,
context: dllContext,
@ -140,6 +157,7 @@ function generateDLL(config) {
new MiniCssExtractPlugin({
filename: dllStyleFilename,
}),
...getValidateSyntaxPlugins(),
],
// Single runtime for the dll bundles which assures that common transient dependencies won't be evaluated twice.
// The module cache will be shared, even when module code may be duplicated across chunks.
@ -163,6 +181,7 @@ function generateDLL(config) {
function extendRawConfig(rawConfig) {
// Build all extended configs from raw config
const dllAlias = rawConfig.uiBundles.getAliases();
const dllValidateSyntax = rawConfig.uiBundles.shouldValidateSyntaxOfNodeModules();
const dllNoParseRules = rawConfig.uiBundles.getWebpackNoParseRules();
const dllDevMode = rawConfig.uiBundles.isDevMode();
const dllContext = rawConfig.context;
@ -195,6 +214,7 @@ function extendRawConfig(rawConfig) {
// Export dll config map
return {
dllAlias,
dllValidateSyntax,
dllNoParseRules,
dllDevMode,
dllContext,

View file

@ -58,6 +58,7 @@ module.exports = function(grunt) {
'--env.name=development',
'--plugins.initialize=false',
'--optimize.bundleFilter=tests',
'--optimize.validateSyntaxOfNodeModules=false',
'--server.port=5610',
'--migrations.skip=true',
];

View file

@ -3796,6 +3796,11 @@
resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f"
integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==
"@types/estree@^0.0.44":
version "0.0.44"
resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.44.tgz#980cc5a29a3ef3bea6ff1f7d021047d7ea575e21"
integrity sha512-iaIVzr+w2ZJ5HkidlZ3EJM8VTZb2MJLCjw3V+505yVts0gRC4UMvjw0d1HPtGqI/HQC/KdsYtayfzl+AXY2R8g==
"@types/events@*":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@types/events/-/events-1.2.0.tgz#81a6731ce4df43619e5c8c945383b3e62a89ea86"
@ -5287,6 +5292,11 @@ acorn-walk@^7.0.0:
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.0.0.tgz#c8ba6f0f1aac4b0a9e32d1f0af12be769528f36b"
integrity sha512-7Bv1We7ZGuU79zZbb6rRqcpxo3OY+zrdtloZWoyD8fmGX+FeXRjE+iuGkZjSXLVovLzrsvMGMy0EkwA0E0umxg==
acorn-walk@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.1.1.tgz#345f0dffad5c735e7373d2fec9a1023e6a44b83e"
integrity sha512-wdlPY2tm/9XBr7QkKlq0WQVgiuGTX6YWPyRyBviSoScBuLfTVQhvwg6wJ369GJ/1nPfTLMfnrFIfjqVg6d+jQQ==
acorn@5.X, acorn@^5.0.0, acorn@^5.0.3, acorn@^5.1.2, acorn@^5.5.0:
version "5.7.4"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e"
@ -5307,7 +5317,7 @@ acorn@^6.0.1, acorn@^6.2.1:
resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474"
integrity sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==
acorn@^7.0.0, acorn@^7.1.0:
acorn@^7.0.0, acorn@^7.1.0, acorn@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.1.1.tgz#e35668de0b402f359de515c5482a1ab9f89a69bf"
integrity sha512-add7dgA5ppRPxCFJoAGfMDi7PIBXq1RtGo7BhbLaxwrXPOmw8gq48Y9ozT01hUKy9byMjlR20EJhu5zlkErEkg==