maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

chore(NA): merge and solve conflicts with master (#54782)

Browse source
This commit is contained in:
Tiago Costa 2020-01-14 19:45:30 +00:00 committed by GitHub
parent e9fc194ac5
commit 63d0b60e4f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 215 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -251,6 +251,7 @@
"rison-node": "1.0.2",
"rxjs": "^6.5.3",
"script-loader": "0.7.2",
"seedrandom": "^3.0.5",
"semver": "^5.5.0",
"style-loader": "0.23.1",
"symbol-observable": "^1.2.0",

2
src/dev/build/README.md
View file

@ -44,7 +44,7 @@ The majority of this logic is extracted from the grunt build that has existed fo
We have introduced in our bundle a webpack dll for the client vendor modules in order to improve
the optimization time both in dev and in production. As for those modules we already have the
code into the vendors.bundle.dll.js we have decided to delete those bundled modules from the
code into the vendors_${chunk_number}.bundle.dll.js we have decided to delete those bundled modules from the
distributable node_modules folder. However, in order to accomplish this, we need to exclude
every node_module used in the server side code. This logic is performed
under `nodejs_modules/clean_client_modules_on_dll_task.js`. In case we need to add any new cli

10
src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js
View file

@ -98,12 +98,16 @@ export const CleanClientModulesOnDLLTask = {
// Consider this as our whiteList for the modules we can't delete
const whiteListedModules = [...serverDependencies, ...kbnWebpackLoaders, ...manualExceptions];
// Resolve the client vendors dll manifest path
const dllManifestPath = `${baseDir}/built_assets/dlls/vendors.manifest.dll.json`;
// Resolve the client vendors dll manifest paths
// excluding the runtime one
const dllManifestPaths = await globby([
`${baseDir}/built_assets/dlls/vendors_*.manifest.dll.json`,
`!${baseDir}/built_assets/dlls/vendors_runtime.manifest.dll.json`,
]);
// Get dll entries filtering out the ones
// from any whitelisted module
const dllEntries = await getDllEntries(dllManifestPath, whiteListedModules, baseDir);
const dllEntries = await getDllEntries(dllManifestPaths, whiteListedModules, baseDir);
for (const relativeEntryPath of dllEntries) {
const entryPath = `${baseDir}/${relativeEntryPath}`;

42
src/dev/build/tasks/nodejs_modules/webpack_dll.js
View file

@ -28,27 +28,37 @@ function checkDllEntryAccess(entry, baseDir = '') {
return isFileAccessible(resolvedPath);
}
export async function getDllEntries(manifestPath, whiteListedModules, baseDir = '') {
const manifest = JSON.parse(await read(manifestPath));
export async function getDllEntries(manifestPaths, whiteListedModules, baseDir = '') {
// Read and parse all manifests
const manifests = await Promise.all(
manifestPaths.map(async manifestPath => JSON.parse(await read(manifestPath)))
);
if (!manifest || !manifest.content) {
// It should fails because if we don't have the manifest file
// or it is malformed something wrong is happening and we
// should stop
throw new Error(`The following dll manifest doesn't exists: ${manifestPath}`);
}
// Process and group modules from all manifests
const manifestsModules = manifests.flatMap((manifest, idx) => {
if (!manifest || !manifest.content) {
// It should fails because if we don't have the manifest file
// or it is malformed something wrong is happening and we
// should stop
throw new Error(`The following dll manifest doesn't exists: ${manifestPaths[idx]}`);
}
const modules = Object.keys(manifest.content);
if (!modules.length) {
// It should fails because if we don't have any
// module inside the client vendors dll something
// wrong is happening and we should stop too
throw new Error(`The following dll manifest is reporting an empty dll: ${manifestPath}`);
}
const modules = Object.keys(manifest.content);
if (!modules.length) {
// It should fails because if we don't have any
// module inside the client vendors dll something
// wrong is happening and we should stop too
throw new Error(
`The following dll manifest is reporting an empty dll: ${manifestPaths[idx]}`
);
}
return modules;
});
// Only includes modules who are not in the white list of modules
// and that are node_modules
return modules.filter(entry => {
return manifestsModules.filter(entry => {
const isWhiteListed = whiteListedModules.some(nonEntry =>
normalizePosixPath(entry).includes(`node_modules/${nonEntry}`)
);

10
src/dev/build/tasks/nodejs_modules/webpack_dll.test.js
View file

@ -52,7 +52,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
isFileAccessible.mockImplementation(() => true);
const mockManifestPath = '/mock/mock_dll_manifest.json';
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
const mockModulesWhitelist = ['dep1'];
const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
@ -66,7 +66,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
isFileAccessible.mockImplementation(() => false);
const mockManifestPath = '/mock/mock_dll_manifest.json';
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
const mockModulesWhitelist = ['dep1'];
const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist);
@ -78,7 +78,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
it('should throw an error for no manifest file', async () => {
read.mockImplementationOnce(async () => noManifestMock);
const mockManifestPath = '/mock/mock_dll_manifest.json';
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
try {
await getDllEntries(mockManifestPath, []);
@ -92,7 +92,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
it('should throw an error for no manifest content field', async () => {
read.mockImplementation(async () => noContentFieldManifestMock);
const mockManifestPath = '/mock/mock_dll_manifest.json';
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
try {
await getDllEntries(mockManifestPath, []);
@ -106,7 +106,7 @@ describe('Webpack DLL Build Tasks Utils', () => {
it('should throw an error for manifest file without any content', async () => {
read.mockImplementation(async () => emptyManifestContentMock);
const mockManifestPath = '/mock/mock_dll_manifest.json';
const mockManifestPath = ['/mock/mock_dll_manifest.json'];
try {
await getDllEntries(mockManifestPath, []);

5
src/legacy/ui/ui_render/bootstrap/template.js.hbs
View file

@ -13,7 +13,10 @@ if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) {
window.onload = function () {
var files = [
'{{dllBundlePath}}/vendors.bundle.dll.js',
'{{dllBundlePath}}/vendors_runtime.bundle.dll.js',
{{#each dllJsChunks}}
'{{this}}',
{{/each}}
'{{regularBundlePath}}/kbn-ui-shared-deps/{{sharedDepsFilename}}',
'{{regularBundlePath}}/commons.bundle.js',
'{{regularBundlePath}}/{{appId}}.bundle.js'

10
src/legacy/ui/ui_render/ui_render_mixin.js
View file

@ -25,6 +25,7 @@ import * as UiSharedDeps from '@kbn/ui-shared-deps';
import { AppBootstrap } from './bootstrap';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { fromRoot } from '../../../core/server/utils';
import { DllCompiler } from '../../../optimize/dynamic_dll_plugin';
/**
* @typedef {import('../../server/kbn_server').default} KbnServer
@ -102,8 +103,14 @@ export function uiRenderMixin(kbnServer, server, config) {
const basePath = config.get('server.basePath');
const regularBundlePath = `${basePath}/bundles`;
const dllBundlePath = `${basePath}/built_assets/dlls`;
const dllStyleChunks = DllCompiler.getRawDllConfig().chunks.map(
chunk => `${dllBundlePath}/vendors${chunk}.style.dll.css`
);
const dllJsChunks = DllCompiler.getRawDllConfig().chunks.map(
chunk => `${dllBundlePath}/vendors${chunk}.bundle.dll.js`
);
const styleSheetPaths = [
`${dllBundlePath}/vendors.style.dll.css`,
...dllStyleChunks,
...(darkMode
? [
`${basePath}/bundles/kbn-ui-shared-deps/${UiSharedDeps.darkCssDistFilename}`,
@ -131,6 +138,7 @@ export function uiRenderMixin(kbnServer, server, config) {
appId: isCore ? 'core' : app.getId(),
regularBundlePath,
dllBundlePath,
dllJsChunks,
styleSheetPaths,
sharedDepsFilename: UiSharedDeps.distFilename,
},

137
src/optimize/dynamic_dll_plugin/dll_compiler.js
View file

@ -23,6 +23,11 @@ import {
notInNodeModules,
inDllPluginPublic,
} from './dll_allowed_modules';
import {
dllEntryFileContentArrayToString,
dllEntryFileContentStringToArray,
dllMergeAllEntryFilesContent,
} from './dll_entry_template';
import { fromRoot } from '../../core/server/utils';
import { PUBLIC_PATH_PLACEHOLDER } from '../public_path_placeholder';
import fs from 'fs';
@ -30,6 +35,8 @@ import webpack from 'webpack';
import { promisify } from 'util';
import path from 'path';
import del from 'del';
import { chunk } from 'lodash';
import seedrandom from 'seedrandom';
const readFileAsync = promisify(fs.readFile);
const mkdirAsync = promisify(fs.mkdir);
@ -37,11 +44,17 @@ const accessAsync = promisify(fs.access);
const writeFileAsync = promisify(fs.writeFile);
export class DllCompiler {
static getRawDllConfig(uiBundles = {}, babelLoaderCacheDir = '', threadLoaderPoolConfig = {}) {
static getRawDllConfig(
uiBundles = {},
babelLoaderCacheDir = '',
threadLoaderPoolConfig = {},
chunks = Array.from(Array(4).keys()).map(chunkN => `_${chunkN}`)
) {
return {
uiBundles,
babelLoaderCacheDir,
threadLoaderPoolConfig,
chunks,
context: fromRoot('.'),
entryName: 'vendors',
dllName: '[name]',
@ -66,13 +79,49 @@ export class DllCompiler {
}
async init() {
await this.ensureEntryFileExists();
await this.ensureManifestFileExists();
await this.ensureEntryFilesExists();
await this.ensureManifestFilesExists();
await this.ensureOutputPathExists();
}
async upsertEntryFile(content) {
await this.upsertFile(this.getEntryPath(), content);
seededShuffle(array) {
// Implementation based on https://github.com/TimothyGu/knuth-shuffle-seeded/blob/gh-pages/index.js#L46
let currentIndex;
let temporaryValue;
let randomIndex;
const rand = seedrandom('predictable', { global: false });
if (array.constructor !== Array) throw new Error('Input is not an array');
currentIndex = array.length;
// While there remain elements to shuffle...
while (0 !== currentIndex) {
// Pick a remaining element...
randomIndex = Math.floor(rand() * currentIndex--);
// And swap it with the current element.
temporaryValue = array[currentIndex];
array[currentIndex] = array[randomIndex];
array[randomIndex] = temporaryValue;
}
return array;
}
async upsertEntryFiles(content) {
const arrayContent = this.seededShuffle(dllEntryFileContentStringToArray(content));
const chunks = chunk(
arrayContent,
Math.ceil(arrayContent.length / this.rawDllConfig.chunks.length)
);
const entryPaths = this.getEntryPaths();
await Promise.all(
entryPaths.map(
async (entryPath, idx) =>
await this.upsertFile(entryPath, dllEntryFileContentArrayToString(chunks[idx]))
)
);
}
async upsertFile(filePath, content = '') {
@ -80,38 +129,57 @@ export class DllCompiler {
await writeFileAsync(filePath, content, 'utf8');
}
getDllPath() {
return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.dllExt}`);
getDllPaths() {
return this.rawDllConfig.chunks.map(chunk =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.dllExt}`)
);
}
getEntryPath() {
return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.entryExt}`);
getEntryPaths() {
return this.rawDllConfig.chunks.map(chunk =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.entryExt}`)
);
}
getManifestPath() {
return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.manifestExt}`);
getManifestPaths() {
return this.rawDllConfig.chunks.map(chunk =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.manifestExt}`)
);
}
getStylePath() {
return this.resolvePath(`${this.rawDllConfig.entryName}${this.rawDllConfig.styleExt}`);
getStylePaths() {
return this.rawDllConfig.chunks.map(chunk =>
this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.styleExt}`)
);
}
async ensureEntryFileExists() {
await this.ensureFileExists(this.getEntryPath());
async ensureEntryFilesExists() {
const entryPaths = this.getEntryPaths();
await Promise.all(entryPaths.map(async entryPath => await this.ensureFileExists(entryPath)));
}
async ensureManifestFileExists() {
await this.ensureFileExists(
this.getManifestPath(),
JSON.stringify({
name: this.rawDllConfig.entryName,
content: {},
})
async ensureManifestFilesExists() {
const manifestPaths = this.getManifestPaths();
await Promise.all(
manifestPaths.map(
async (manifestPath, idx) =>
await this.ensureFileExists(
manifestPath,
JSON.stringify({
name: `${this.rawDllConfig.entryName}${this.rawDllConfig.chunks[idx]}`,
content: {},
})
)
)
);
}
async ensureStyleFileExists() {
await this.ensureFileExists(this.getStylePath());
const stylePaths = this.getStylePaths();
await Promise.all(stylePaths.map(async stylePath => await this.ensureFileExists(stylePath)));
}
async ensureFileExists(filePath, content) {
@ -137,8 +205,10 @@ export class DllCompiler {
await this.ensurePathExists(this.rawDllConfig.outputPath);
}
dllExistsSync() {
return this.existsSync(this.getDllPath());
dllsExistsSync() {
const dllPaths = this.getDllPaths();
return dllPaths.every(dllPath => this.existsSync(dllPath));
}
existsSync(filePath) {
@ -149,8 +219,16 @@ export class DllCompiler {
return path.resolve(this.rawDllConfig.outputPath, ...arguments);
}
async readEntryFile() {
return await this.readFile(this.getEntryPath());
async readEntryFiles() {
const entryPaths = this.getEntryPaths();
const entryFilesContent = await Promise.all(
entryPaths.map(async entryPath => await this.readFile(entryPath))
);
// merge all the module contents from entry files again into
// sorted single one
return dllMergeAllEntryFilesContent(entryFilesContent);
}
async readFile(filePath, content) {
@ -160,7 +238,7 @@ export class DllCompiler {
async run(dllEntries) {
const dllConfig = this.dllConfigGenerator(this.rawDllConfig);
await this.upsertEntryFile(dllEntries);
await this.upsertEntryFiles(dllEntries);
try {
this.logWithMetadata(
@ -234,7 +312,7 @@ export class DllCompiler {
// ignore if this module represents the
// dll entry file
if (module.resource === this.getEntryPath()) {
if (this.getEntryPaths().includes(module.resource)) {
return;
}
@ -259,7 +337,6 @@ export class DllCompiler {
// node_module or no?
if (notInNodeModules(reason.module.resource)) {
notAllowedModules.push(module.resource);
return;
}
});
}

15
src/optimize/dynamic_dll_plugin/dll_config_model.js
View file

@ -140,6 +140,13 @@ function generateDLL(config) {
filename: dllStyleFilename,
}),
],
// Single runtime for the dll bundles which assures that common transient dependencies won't be evaluated twice.
// The module cache will be shared, even when module code may be duplicated across chunks.
optimization: {
runtimeChunk: {
name: 'vendors_runtime',
},
},
performance: {
// NOTE: we are disabling this as those hints
// are more tailored for the final bundles result
@ -158,6 +165,7 @@ function extendRawConfig(rawConfig) {
const dllNoParseRules = rawConfig.uiBundles.getWebpackNoParseRules();
const dllDevMode = rawConfig.uiBundles.isDevMode();
const dllContext = rawConfig.context;
const dllChunks = rawConfig.chunks;
const dllEntry = {};
const dllEntryName = rawConfig.entryName;
const dllBundleName = rawConfig.dllName;
@ -176,7 +184,12 @@ function extendRawConfig(rawConfig) {
const threadLoaderPoolConfig = rawConfig.threadLoaderPoolConfig;
// Create webpack entry object key with the provided dllEntryName
dllEntry[dllEntryName] = [`${dllOutputPath}/${dllEntryName}${dllEntryExt}`];
dllChunks.reduce((dllEntryObj, chunk) => {
dllEntryObj[`${dllEntryName}${chunk}`] = [
`${dllOutputPath}/${dllEntryName}${chunk}${dllEntryExt}`,
];
return dllEntryObj;
}, dllEntry);
// Export dll config map
return {

16
src/optimize/dynamic_dll_plugin/dll_entry_template.js
View file

@ -23,3 +23,19 @@ export function dllEntryTemplate(requirePaths = []) {
.sort()
.join('\n');
}
export function dllEntryFileContentStringToArray(content = '') {
return content.split('\n');
}
export function dllEntryFileContentArrayToString(content = []) {
return content.join('\n');
}
export function dllMergeAllEntryFilesContent(content = []) {
return content
.join('\n')
.split('\n')
.sort()
.join('\n');
}

20
src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js
View file

@ -44,7 +44,7 @@ export class DynamicDllPlugin {
async init() {
await this.dllCompiler.init();
this.entryPaths = await this.dllCompiler.readEntryFile();
this.entryPaths = await this.dllCompiler.readEntryFiles();
}
apply(compiler) {
@ -70,12 +70,14 @@ export class DynamicDllPlugin {
bindDllReferencePlugin(compiler) {
const rawDllConfig = this.dllCompiler.rawDllConfig;
const dllContext = rawDllConfig.context;
const dllManifestPath = this.dllCompiler.getManifestPath();
const dllManifestPaths = this.dllCompiler.getManifestPaths();
new webpack.DllReferencePlugin({
context: dllContext,
manifest: dllManifestPath,
}).apply(compiler);
dllManifestPaths.forEach(dllChunkManifestPath => {
new webpack.DllReferencePlugin({
context: dllContext,
manifest: dllChunkManifestPath,
}).apply(compiler);
});
}
registerInitBasicHooks(compiler) {
@ -192,7 +194,7 @@ export class DynamicDllPlugin {
// then will be set to false
compilation.needsDLLCompilation =
this.afterCompilationEntryPaths !== this.entryPaths ||
!this.dllCompiler.dllExistsSync() ||
!this.dllCompiler.dllsExistsSync() ||
(this.isToForceDLLCreation() && this.performedCompilations === 0);
this.entryPaths = this.afterCompilationEntryPaths;
@ -337,7 +339,9 @@ export class DynamicDllPlugin {
// We need to purge the cache into the inputFileSystem
// for every single built in previous compilation
// that we rely in next ones.
mainCompiler.inputFileSystem.purge(this.dllCompiler.getManifestPath());
this.dllCompiler
.getManifestPaths()
.forEach(chunkDllManifestPath => mainCompiler.inputFileSystem.purge(chunkDllManifestPath));
this.performedCompilations++;

10
tasks/config/karma.js
View file

@ -21,6 +21,7 @@ import { dirname } from 'path';
import { times } from 'lodash';
import { makeJunitReportPath } from '@kbn/test';
import * as UiSharedDeps from '@kbn/ui-shared-deps';
import { DllCompiler } from '../../src/optimize/dynamic_dll_plugin';
const TOTAL_CI_SHARDS = 4;
const ROOT = dirname(require.resolve('../../package.json'));
@ -54,7 +55,10 @@ module.exports = function(grunt) {
'http://localhost:5610/test_bundle/built_css.css',
`http://localhost:5610/bundles/kbn-ui-shared-deps/${UiSharedDeps.distFilename}`,
'http://localhost:5610/built_assets/dlls/vendors.bundle.dll.js',
'http://localhost:5610/built_assets/dlls/vendors_runtime.bundle.dll.js',
...DllCompiler.getRawDllConfig().chunks.map(
chunk => `http://localhost:5610/built_assets/dlls/vendors${chunk}.bundle.dll.js`
),
shardNum === undefined
? `http://localhost:5610/bundles/tests.bundle.js`
@ -63,7 +67,9 @@ module.exports = function(grunt) {
// this causes tilemap tests to fail, probably because the eui styles haven't been
// included in the karma harness a long some time, if ever
// `http://localhost:5610/bundles/kbn-ui-shared-deps/${UiSharedDeps.lightCssDistFilename}`,
'http://localhost:5610/built_assets/dlls/vendors.style.dll.css',
...DllCompiler.getRawDllConfig().chunks.map(
chunk => `http://localhost:5610/built_assets/dlls/vendors${chunk}.style.dll.css`
),
'http://localhost:5610/bundles/tests.style.css',
];
}

5
yarn.lock
View file

@ -25895,6 +25895,11 @@ scss-tokenizer@^0.2.3:
js-base64 "^2.1.8"
source-map "^0.4.2"
seedrandom@^3.0.5:
version "3.0.5"
resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7"
integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==
seek-bzip@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.5.tgz#cfe917cb3d274bcffac792758af53173eb1fabdc"