maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[src/dev/build] build Kibana Platform bundles from source (#73591)

Browse source 
Co-authored-by: spalger <spalger@users.noreply.github.com>
This commit is contained in:
Spencer 2020-08-04 15:34:01 -07:00 committed by GitHub
parent 372d8817a7
commit 409779d1ed
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 99 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts
View file

@ -19,9 +19,12 @@
import { REPO_ROOT } from '../repo_root';
export function createAbsolutePathSerializer(rootPath: string = REPO_ROOT) {
export function createAbsolutePathSerializer(
rootPath: string = REPO_ROOT,
replacement = '<absolute path>'
) {
return {
test: (value: any) => typeof value === 'string' && value.startsWith(rootPath),
serialize: (value: string) => value.replace(rootPath, '<absolute path>').replace(/\\/g, '/'),
serialize: (value: string) => value.replace(rootPath, replacement).replace(/\\/g, '/'),
};
}

14
packages/kbn-optimizer/src/common/bundle_cache.ts
View file

@ -104,4 +104,18 @@ export class BundleCache {
public getOptimizerCacheKey() {
return this.get().optimizerCacheKey;
}
public clear() {
this.state = undefined;
if (this.path) {
try {
Fs.unlinkSync(this.path);
} catch (error) {
if (error.code !== 'ENOENT') {
throw error;
}
}
}
}
}

31
packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts
View file

@ -21,7 +21,9 @@ import { createAbsolutePathSerializer } from '@kbn/dev-utils';
import { getPluginBundles } from './get_plugin_bundles';
expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo'));
expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo', '<repoRoot>'));
expect.addSnapshotSerializer(createAbsolutePathSerializer('/output', '<outputRoot>'));
expect.addSnapshotSerializer(createAbsolutePathSerializer('/outside/of/repo', '<outsideOfRepo>'));
it('returns a bundle for core and each plugin', () => {
expect(
@ -56,46 +58,47 @@ it('returns a bundle for core and each plugin', () => {
manifestPath: '/repo/x-pack/plugins/box/kibana.json',
},
],
'/repo'
'/repo',
'/output'
).map((b) => b.toSpec())
).toMatchInlineSnapshot(`
Array [
Object {
"banner": undefined,
"contextDir": <absolute path>/plugins/foo,
"contextDir": <repoRoot>/plugins/foo,
"id": "foo",
"manifestPath": <absolute path>/plugins/foo/kibana.json,
"outputDir": <absolute path>/plugins/foo/target/public,
"manifestPath": <repoRoot>/plugins/foo/kibana.json,
"outputDir": <outputRoot>/plugins/foo/target/public,
"publicDirNames": Array [
"public",
],
"sourceRoot": <absolute path>,
"sourceRoot": <repoRoot>,
"type": "plugin",
},
Object {
"banner": undefined,
"contextDir": "/outside/of/repo/plugins/baz",
"contextDir": <outsideOfRepo>/plugins/baz,
"id": "baz",
"manifestPath": "/outside/of/repo/plugins/baz/kibana.json",
"outputDir": "/outside/of/repo/plugins/baz/target/public",
"manifestPath": <outsideOfRepo>/plugins/baz/kibana.json,
"outputDir": <outsideOfRepo>/plugins/baz/target/public,
"publicDirNames": Array [
"public",
],
"sourceRoot": <absolute path>,
"sourceRoot": <repoRoot>,
"type": "plugin",
},
Object {
"banner": "/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.
* Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */
",
"contextDir": <absolute path>/x-pack/plugins/box,
"contextDir": <repoRoot>/x-pack/plugins/box,
"id": "box",
"manifestPath": <absolute path>/x-pack/plugins/box/kibana.json,
"outputDir": <absolute path>/x-pack/plugins/box/target/public,
"manifestPath": <repoRoot>/x-pack/plugins/box/kibana.json,
"outputDir": <outputRoot>/x-pack/plugins/box/target/public,
"publicDirNames": Array [
"public",
],
"sourceRoot": <absolute path>,
"sourceRoot": <repoRoot>,
"type": "plugin",
},
]

12
packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts
View file

@ -23,7 +23,11 @@ import { Bundle } from '../common';
import { KibanaPlatformPlugin } from './kibana_platform_plugins';
export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: string) {
export function getPluginBundles(
plugins: KibanaPlatformPlugin[],
repoRoot: string,
outputRoot: string
) {
const xpackDirSlash = Path.resolve(repoRoot, 'x-pack') + Path.sep;
return plugins
@ -36,7 +40,11 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri
publicDirNames: ['public', ...p.extraPublicDirs],
sourceRoot: repoRoot,
contextDir: p.directory,
outputDir: Path.resolve(p.directory, 'target/public'),
outputDir: Path.resolve(
outputRoot,
Path.relative(repoRoot, p.directory),
'target/public'
),
manifestPath: p.manifestPath,
banner: p.directory.startsWith(xpackDirSlash)
? `/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.\n` +

31
packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts
View file

@ -23,16 +23,20 @@ jest.mock('./get_plugin_bundles.ts');
jest.mock('../common/theme_tags.ts');
jest.mock('./filter_by_id.ts');
import Path from 'path';
import Os from 'os';
jest.mock('os', () => {
const realOs = jest.requireActual('os');
jest.spyOn(realOs, 'cpus').mockImplementation(() => {
return ['foo'] as any;
});
return realOs;
});
import Path from 'path';
import { REPO_ROOT, createAbsolutePathSerializer } from '@kbn/dev-utils';
import { OptimizerConfig } from './optimizer_config';
import { OptimizerConfig, ParsedOptions } from './optimizer_config';
import { parseThemeTags } from '../common';
jest.spyOn(Os, 'cpus').mockReturnValue(['foo'] as any);
expect.addSnapshotSerializer(createAbsolutePathSerializer());
beforeEach(() => {
@ -118,6 +122,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 2,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [
<absolute path>/src/plugins,
@ -145,6 +150,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 2,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [
<absolute path>/src/plugins,
@ -172,6 +178,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 2,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [
<absolute path>/src/plugins,
@ -201,6 +208,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 2,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [
<absolute path>/src/plugins,
@ -227,6 +235,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 2,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [
<absolute path>/x/y/z,
@ -253,6 +262,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 100,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [],
"profileWebpack": false,
@ -276,6 +286,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 100,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [],
"profileWebpack": false,
@ -299,6 +310,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 100,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [],
"profileWebpack": false,
@ -323,6 +335,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 100,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [],
"profileWebpack": false,
@ -347,6 +360,7 @@ describe('OptimizerConfig::parseOptions()', () => {
"includeCoreBundle": false,
"inspectWorkers": false,
"maxWorkerCount": 100,
"outputRoot": <absolute path>,
"pluginPaths": Array [],
"pluginScanDirs": Array [],
"profileWebpack": false,
@ -384,18 +398,22 @@ describe('OptimizerConfig::create()', () => {
getPluginBundles.mockReturnValue([Symbol('bundle1'), Symbol('bundle2')]);
filterById.mockReturnValue(Symbol('filtered bundles'));
jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): any => ({
jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): {
[key in keyof ParsedOptions]: any;
} => ({
cache: Symbol('parsed cache'),
dist: Symbol('parsed dist'),
maxWorkerCount: Symbol('parsed max worker count'),
pluginPaths: Symbol('parsed plugin paths'),
pluginScanDirs: Symbol('parsed plugin scan dirs'),
repoRoot: Symbol('parsed repo root'),
outputRoot: Symbol('parsed output root'),
watch: Symbol('parsed watch'),
themeTags: Symbol('theme tags'),
inspectWorkers: Symbol('parsed inspect workers'),
profileWebpack: Symbol('parsed profile webpack'),
filters: [],
includeCoreBundle: false,
}));
});
@ -474,6 +492,7 @@ describe('OptimizerConfig::create()', () => {
Array [
Symbol(new platform plugins),
Symbol(parsed repo root),
Symbol(parsed output root),
],
],
"instances": Array [

20
packages/kbn-optimizer/src/optimizer/optimizer_config.ts
View file

@ -55,6 +55,13 @@ function omit<T, K extends keyof T>(obj: T, keys: K[]): Omit<T, K> {
interface Options {
/** absolute path to root of the repo/build */
repoRoot: string;
/**
* absolute path to the root directory where output should be written to. This
* defaults to the repoRoot but can be customized to write output somewhere else.
*
* This is how we write output to the build directory in the Kibana build tasks.
*/
outputRoot?: string;
/** enable to run the optimizer in watch mode */
watch?: boolean;
/** the maximum number of workers that will be created */
@ -107,8 +114,9 @@ interface Options {
themes?: ThemeTag | '*' | ThemeTag[];
}
interface ParsedOptions {
export interface ParsedOptions {
repoRoot: string;
outputRoot: string;
watch: boolean;
maxWorkerCount: number;
profileWebpack: boolean;
@ -139,6 +147,11 @@ export class OptimizerConfig {
throw new TypeError('repoRoot must be an absolute path');
}
const outputRoot = options.outputRoot ?? repoRoot;
if (!Path.isAbsolute(outputRoot)) {
throw new TypeError('outputRoot must be an absolute path');
}
/**
* BEWARE: this needs to stay roughly synchronized with
* `src/core/server/config/env.ts` which determines which paths
@ -182,6 +195,7 @@ export class OptimizerConfig {
watch,
dist,
repoRoot,
outputRoot,
maxWorkerCount,
profileWebpack,
cache,
@ -206,11 +220,11 @@ export class OptimizerConfig {
publicDirNames: ['public', 'public/utils'],
sourceRoot: options.repoRoot,
contextDir: Path.resolve(options.repoRoot, 'src/core'),
outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'),
outputDir: Path.resolve(options.outputRoot, 'src/core/target/public'),
}),
]
: []),
...getPluginBundles(plugins, options.repoRoot),
...getPluginBundles(plugins, options.repoRoot, options.outputRoot),
];
return new OptimizerConfig(

18
src/dev/build/tasks/build_kibana_platform_plugins.ts
View file

@ -17,7 +17,7 @@
* under the License.
*/
import { CiStatsReporter } from '@kbn/dev-utils';
import { CiStatsReporter, REPO_ROOT } from '@kbn/dev-utils';
import {
runOptimizer,
OptimizerConfig,
@ -29,9 +29,10 @@ import { Task } from '../lib';
export const BuildKibanaPlatformPlugins: Task = {
description: 'Building distributable versions of Kibana platform plugins',
async run(config, log, build) {
const optimizerConfig = OptimizerConfig.create({
repoRoot: build.resolvePath(),
async run(_, log, build) {
const config = OptimizerConfig.create({
repoRoot: REPO_ROOT,
outputRoot: build.resolvePath(),
cache: false,
oss: build.isOss(),
examples: false,
@ -42,11 +43,10 @@ export const BuildKibanaPlatformPlugins: Task = {
const reporter = CiStatsReporter.fromEnv(log);
await runOptimizer(optimizerConfig)
.pipe(
reportOptimizerStats(reporter, optimizerConfig, log),
logOptimizerState(log, optimizerConfig)
)
await runOptimizer(config)
.pipe(reportOptimizerStats(reporter, config, log), logOptimizerState(log, config))
.toPromise();
await Promise.all(config.bundles.map((b) => b.cache.clear()));
},
};

2
src/dev/build/tasks/copy_source_task.ts
View file

@ -30,7 +30,7 @@ export const CopySource: Task = {
'src/**',
'!src/**/*.{test,test.mocks,mock}.{js,ts,tsx}',
'!src/**/mocks.ts', // special file who imports .mock files
'!src/**/{__tests__,__snapshots__,__mocks__}/**',
'!src/**/{target,__tests__,__snapshots__,__mocks__}/**',
'!src/test_utils/**',
'!src/fixtures/**',
'!src/legacy/core_plugins/console/public/tests/**',

1
vars/kibanaPipeline.groovy
View file

@ -170,6 +170,7 @@ def uploadCoverageArtifacts(prefix, pattern) {
def withGcsArtifactUpload(workerName, closure) {
def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}"
def ARTIFACT_PATTERNS = [
'**/target/public/.kbn-optimizer-cache',
'target/kibana-*',
'target/test-metrics/*',
'target/kibana-security-solution/**/*.png',