[ci-stats] ship timings, collect overall bootstrap time (#93557)
Co-authored-by: spalger <spalger@users.noreply.github.com>
This commit is contained in:
parent
8be1dd7c54
commit
054da62a7a
|
@ -12,6 +12,7 @@ running in no time. If you have any problems, file an issue in the https://githu
|
||||||
* <<kibana-architecture>>
|
* <<kibana-architecture>>
|
||||||
* <<advanced>>
|
* <<advanced>>
|
||||||
* <<plugin-list>>
|
* <<plugin-list>>
|
||||||
|
* <<development-telemetry>>
|
||||||
|
|
||||||
--
|
--
|
||||||
|
|
||||||
|
@ -29,3 +30,5 @@ include::advanced/index.asciidoc[]
|
||||||
|
|
||||||
include::plugin-list.asciidoc[]
|
include::plugin-list.asciidoc[]
|
||||||
|
|
||||||
|
include::telemetry.asciidoc[]
|
||||||
|
|
||||||
|
|
18
docs/developer/telemetry.asciidoc
Normal file
18
docs/developer/telemetry.asciidoc
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
[[development-telemetry]]
|
||||||
|
== Development Telemetry
|
||||||
|
|
||||||
|
To help us provide a good developer experience, we track some straightforward metrics when running certain tasks locally and ship them to a service that we run. To disable this functionality, specify `CI_STATS_DISABLED=true` in your environment.
|
||||||
|
|
||||||
|
The operations we current report timing data for:
|
||||||
|
|
||||||
|
* Total execution time of `yarn kbn bootstrap`
|
||||||
|
|
||||||
|
Along with the execution time of each execution, we ship the following information about your machine to the service:
|
||||||
|
|
||||||
|
* The `branch` property from the package.json file
|
||||||
|
* The value of the `data/uuid` file
|
||||||
|
* https://nodejs.org/docs/latest/api/os.html#os_os_platform[Operating system platform]
|
||||||
|
* https://nodejs.org/docs/latest/api/os.html#os_os_release[Operating system release]
|
||||||
|
* https://nodejs.org/docs/latest/api/os.html#os_os_cpus[Count, model, and speed of the CPUs]
|
||||||
|
* https://nodejs.org/docs/latest/api/os.html#os_os_arch[CPU architecture]
|
||||||
|
* https://nodejs.org/docs/latest/api/os.html#os_os_totalmem[Total memory] and https://nodejs.org/docs/latest/api/os.html#os_os_freemem[Free memory]
|
3
packages/kbn-dev-utils/ci_stats_reporter/package.json
Normal file
3
packages/kbn-dev-utils/ci_stats_reporter/package.json
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
{
|
||||||
|
"main": "../target/ci_stats_reporter/ci_stats_reporter"
|
||||||
|
}
|
|
@ -0,0 +1,52 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ToolingLog } from '../tooling_log';
|
||||||
|
|
||||||
|
export interface Config {
|
||||||
|
apiToken: string;
|
||||||
|
buildId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateConfig(log: ToolingLog, config: { [k in keyof Config]: unknown }) {
|
||||||
|
const validApiToken = typeof config.apiToken === 'string' && config.apiToken.length !== 0;
|
||||||
|
if (!validApiToken) {
|
||||||
|
log.warning('KIBANA_CI_STATS_CONFIG is missing a valid api token, stats will not be reported');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const validId = typeof config.buildId === 'string' && config.buildId.length !== 0;
|
||||||
|
if (!validId) {
|
||||||
|
log.warning('KIBANA_CI_STATS_CONFIG is missing a valid build id, stats will not be reported');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
return config as Config;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseConfig(log: ToolingLog) {
|
||||||
|
const configJson = process.env.KIBANA_CI_STATS_CONFIG;
|
||||||
|
if (!configJson) {
|
||||||
|
log.debug('KIBANA_CI_STATS_CONFIG environment variable not found, disabling CiStatsReporter');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let config: unknown;
|
||||||
|
try {
|
||||||
|
config = JSON.parse(configJson);
|
||||||
|
} catch (_) {
|
||||||
|
// handled below
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof config === 'object' && config !== null) {
|
||||||
|
return validateConfig(log, config as { [k in keyof Config]: unknown });
|
||||||
|
}
|
||||||
|
|
||||||
|
log.warning('KIBANA_CI_STATS_CONFIG is invalid, stats will not be reported');
|
||||||
|
return;
|
||||||
|
}
|
|
@ -7,69 +7,50 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { inspect } from 'util';
|
import { inspect } from 'util';
|
||||||
|
import Os from 'os';
|
||||||
|
import Fs from 'fs';
|
||||||
|
import Path from 'path';
|
||||||
|
|
||||||
import Axios from 'axios';
|
import Axios from 'axios';
|
||||||
|
|
||||||
import { ToolingLog } from '../tooling_log';
|
import { ToolingLog } from '../tooling_log';
|
||||||
|
import { parseConfig, Config } from './ci_stats_config';
|
||||||
|
|
||||||
interface Config {
|
const BASE_URL = 'https://ci-stats.kibana.dev';
|
||||||
apiUrl: string;
|
|
||||||
apiToken: string;
|
|
||||||
buildId: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type CiStatsMetrics = Array<{
|
export interface CiStatsMetric {
|
||||||
group: string;
|
group: string;
|
||||||
id: string;
|
id: string;
|
||||||
value: number;
|
value: number;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
limitConfigPath?: string;
|
limitConfigPath?: string;
|
||||||
}>;
|
|
||||||
|
|
||||||
function parseConfig(log: ToolingLog) {
|
|
||||||
const configJson = process.env.KIBANA_CI_STATS_CONFIG;
|
|
||||||
if (!configJson) {
|
|
||||||
log.debug('KIBANA_CI_STATS_CONFIG environment variable not found, disabling CiStatsReporter');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let config: unknown;
|
|
||||||
try {
|
|
||||||
config = JSON.parse(configJson);
|
|
||||||
} catch (_) {
|
|
||||||
// handled below
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof config === 'object' && config !== null) {
|
|
||||||
return validateConfig(log, config as { [k in keyof Config]: unknown });
|
|
||||||
}
|
|
||||||
|
|
||||||
log.warning('KIBANA_CI_STATS_CONFIG is invalid, stats will not be reported');
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateConfig(log: ToolingLog, config: { [k in keyof Config]: unknown }) {
|
export interface CiStatsTimingMetadata {
|
||||||
const validApiUrl = typeof config.apiUrl === 'string' && config.apiUrl.length !== 0;
|
[key: string]: string | string[] | number | boolean | undefined;
|
||||||
if (!validApiUrl) {
|
}
|
||||||
log.warning('KIBANA_CI_STATS_CONFIG is missing a valid api url, stats will not be reported');
|
export interface CiStatsTiming {
|
||||||
return;
|
group: string;
|
||||||
}
|
id: string;
|
||||||
|
ms: number;
|
||||||
const validApiToken = typeof config.apiToken === 'string' && config.apiToken.length !== 0;
|
meta?: CiStatsTimingMetadata;
|
||||||
if (!validApiToken) {
|
|
||||||
log.warning('KIBANA_CI_STATS_CONFIG is missing a valid api token, stats will not be reported');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const validId = typeof config.buildId === 'string' && config.buildId.length !== 0;
|
|
||||||
if (!validId) {
|
|
||||||
log.warning('KIBANA_CI_STATS_CONFIG is missing a valid build id, stats will not be reported');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
return config as Config;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ReqOptions {
|
||||||
|
auth: boolean;
|
||||||
|
path: string;
|
||||||
|
body: any;
|
||||||
|
bodyDesc: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TimingsOptions {
|
||||||
|
/** list of timings to record */
|
||||||
|
timings: CiStatsTiming[];
|
||||||
|
/** master, 7.x, etc, automatically detected from package.json if not specified */
|
||||||
|
upstreamBranch?: string;
|
||||||
|
/** value of data/uuid, automatically loaded if not specified */
|
||||||
|
kibanaUuid?: string | null;
|
||||||
|
}
|
||||||
export class CiStatsReporter {
|
export class CiStatsReporter {
|
||||||
static fromEnv(log: ToolingLog) {
|
static fromEnv(log: ToolingLog) {
|
||||||
return new CiStatsReporter(parseConfig(log), log);
|
return new CiStatsReporter(parseConfig(log), log);
|
||||||
|
@ -78,19 +59,126 @@ export class CiStatsReporter {
|
||||||
constructor(private config: Config | undefined, private log: ToolingLog) {}
|
constructor(private config: Config | undefined, private log: ToolingLog) {}
|
||||||
|
|
||||||
isEnabled() {
|
isEnabled() {
|
||||||
return !!this.config;
|
return process.env.CI_STATS_DISABLED !== 'true';
|
||||||
}
|
}
|
||||||
|
|
||||||
async metrics(metrics: CiStatsMetrics) {
|
hasBuildConfig() {
|
||||||
if (!this.config) {
|
return this.isEnabled() && !!this.config?.apiToken && !!this.config?.buildId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Report timings data to the ci-stats service. If running in CI then the reporter
|
||||||
|
* will include the buildId in the report with the access token, otherwise the timings
|
||||||
|
* data will be recorded as anonymous timing data.
|
||||||
|
*/
|
||||||
|
async timings(options: TimingsOptions) {
|
||||||
|
if (!this.isEnabled()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const buildId = this.config?.buildId;
|
||||||
|
const timings = options.timings;
|
||||||
|
const upstreamBranch = options.upstreamBranch ?? this.getUpstreamBranch();
|
||||||
|
const kibanaUuid = options.kibanaUuid === undefined ? this.getKibanaUuid() : options.kibanaUuid;
|
||||||
|
const defaultMetadata = {
|
||||||
|
osPlatform: Os.platform(),
|
||||||
|
osRelease: Os.release(),
|
||||||
|
osArch: Os.arch(),
|
||||||
|
cpuCount: Os.cpus()?.length,
|
||||||
|
cpuModel: Os.cpus()[0]?.model,
|
||||||
|
cpuSpeed: Os.cpus()[0]?.speed,
|
||||||
|
freeMem: Os.freemem(),
|
||||||
|
totalMem: Os.totalmem(),
|
||||||
|
kibanaUuid,
|
||||||
|
};
|
||||||
|
|
||||||
|
return await this.req({
|
||||||
|
auth: !!buildId,
|
||||||
|
path: '/v1/timings',
|
||||||
|
body: {
|
||||||
|
buildId,
|
||||||
|
upstreamBranch,
|
||||||
|
timings,
|
||||||
|
defaultMetadata,
|
||||||
|
},
|
||||||
|
bodyDesc: timings.length === 1 ? `${timings.length} timing` : `${timings.length} timings`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Report metrics data to the ci-stats service. If running outside of CI this method
|
||||||
|
* does nothing as metrics can only be reported when associated with a specific CI build.
|
||||||
|
*/
|
||||||
|
async metrics(metrics: CiStatsMetric[]) {
|
||||||
|
if (!this.hasBuildConfig()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const buildId = this.config?.buildId;
|
||||||
|
|
||||||
|
if (!buildId) {
|
||||||
|
throw new Error(`CiStatsReporter can't be authorized without a buildId`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return await this.req({
|
||||||
|
auth: true,
|
||||||
|
path: '/v1/metrics',
|
||||||
|
body: {
|
||||||
|
buildId,
|
||||||
|
metrics,
|
||||||
|
},
|
||||||
|
bodyDesc: `metrics: ${metrics
|
||||||
|
.map(({ group, id, value }) => `[${group}/${id}=${value}]`)
|
||||||
|
.join(' ')}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In order to allow this code to run before @kbn/utils is built, @kbn/pm will pass
|
||||||
|
* in the upstreamBranch when calling the timings() method. Outside of @kbn/pm
|
||||||
|
* we rely on @kbn/utils to find the package.json file.
|
||||||
|
*/
|
||||||
|
private getUpstreamBranch() {
|
||||||
|
// specify the module id in a way that will keep webpack from bundling extra code into @kbn/pm
|
||||||
|
const hideFromWebpack = ['@', 'kbn/utils'];
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||||
|
const { kibanaPackageJson } = require(hideFromWebpack.join(''));
|
||||||
|
return kibanaPackageJson.branch;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In order to allow this code to run before @kbn/utils is built, @kbn/pm will pass
|
||||||
|
* in the kibanaUuid when calling the timings() method. Outside of @kbn/pm
|
||||||
|
* we rely on @kbn/utils to find the repo root.
|
||||||
|
*/
|
||||||
|
private getKibanaUuid() {
|
||||||
|
// specify the module id in a way that will keep webpack from bundling extra code into @kbn/pm
|
||||||
|
const hideFromWebpack = ['@', 'kbn/utils'];
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||||
|
const { REPO_ROOT } = require(hideFromWebpack.join(''));
|
||||||
|
try {
|
||||||
|
return Fs.readFileSync(Path.resolve(REPO_ROOT, 'data/uuid'), 'utf-8').trim();
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async req({ auth, body, bodyDesc, path }: ReqOptions) {
|
||||||
let attempt = 0;
|
let attempt = 0;
|
||||||
const maxAttempts = 5;
|
const maxAttempts = 5;
|
||||||
const bodySummary = metrics
|
|
||||||
.map(({ group, id, value }) => `[${group}/${id}=${value}]`)
|
let headers;
|
||||||
.join(' ');
|
if (auth && this.config) {
|
||||||
|
headers = {
|
||||||
|
Authorization: `token ${this.config.apiToken}`,
|
||||||
|
};
|
||||||
|
} else if (auth) {
|
||||||
|
throw new Error('this.req() shouldnt be called with auth=true if this.config is defined');
|
||||||
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
attempt += 1;
|
attempt += 1;
|
||||||
|
@ -98,15 +186,10 @@ export class CiStatsReporter {
|
||||||
try {
|
try {
|
||||||
await Axios.request({
|
await Axios.request({
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
url: '/v1/metrics',
|
url: path,
|
||||||
baseURL: this.config.apiUrl,
|
baseURL: BASE_URL,
|
||||||
headers: {
|
headers,
|
||||||
Authorization: `token ${this.config.apiToken}`,
|
data: body,
|
||||||
},
|
|
||||||
data: {
|
|
||||||
buildId: this.config.buildId,
|
|
||||||
metrics,
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
|
@ -116,19 +199,19 @@ export class CiStatsReporter {
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (error?.response && error.response.status !== 502) {
|
if (error?.response && error.response.status < 502) {
|
||||||
// error response from service was received so warn the user and move on
|
// error response from service was received so warn the user and move on
|
||||||
this.log.warning(
|
this.log.warning(
|
||||||
`error recording metric [status=${error.response.status}] [resp=${inspect(
|
`error reporting ${bodyDesc} [status=${error.response.status}] [resp=${inspect(
|
||||||
error.response.data
|
error.response.data
|
||||||
)}] ${bodySummary}`
|
)}]`
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (attempt === maxAttempts) {
|
if (attempt === maxAttempts) {
|
||||||
this.log.warning(
|
this.log.warning(
|
||||||
`failed to reach kibana-ci-stats service too many times, unable to record metric ${bodySummary}`
|
`unable to report ${bodyDesc}, failed to reach ci-stats service too many times`
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -139,7 +222,7 @@ export class CiStatsReporter {
|
||||||
: 'no response';
|
: 'no response';
|
||||||
|
|
||||||
this.log.warning(
|
this.log.warning(
|
||||||
`failed to reach kibana-ci-stats service [reason=${reason}], retrying in ${attempt} seconds`
|
`failed to reach ci-stats service [reason=${reason}], retrying in ${attempt} seconds`
|
||||||
);
|
);
|
||||||
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, attempt * 1000));
|
await new Promise((resolve) => setTimeout(resolve, attempt * 1000));
|
||||||
|
|
|
@ -11,7 +11,7 @@ import Path from 'path';
|
||||||
|
|
||||||
import dedent from 'dedent';
|
import dedent from 'dedent';
|
||||||
import Yaml from 'js-yaml';
|
import Yaml from 'js-yaml';
|
||||||
import { createFailError, ToolingLog, CiStatsMetrics } from '@kbn/dev-utils';
|
import { createFailError, ToolingLog, CiStatsMetric } from '@kbn/dev-utils';
|
||||||
|
|
||||||
import { OptimizerConfig, Limits } from './optimizer';
|
import { OptimizerConfig, Limits } from './optimizer';
|
||||||
|
|
||||||
|
@ -86,7 +86,7 @@ export function updateBundleLimits({
|
||||||
limitsPath,
|
limitsPath,
|
||||||
}: UpdateBundleLimitsOptions) {
|
}: UpdateBundleLimitsOptions) {
|
||||||
const limits = readLimits(limitsPath);
|
const limits = readLimits(limitsPath);
|
||||||
const metrics: CiStatsMetrics = config.bundles
|
const metrics: CiStatsMetric[] = config.bundles
|
||||||
.map((bundle) =>
|
.map((bundle) =>
|
||||||
JSON.parse(Fs.readFileSync(Path.resolve(bundle.outputDir, 'metrics.json'), 'utf-8'))
|
JSON.parse(Fs.readFileSync(Path.resolve(bundle.outputDir, 'metrics.json'), 'utf-8'))
|
||||||
)
|
)
|
||||||
|
|
|
@ -10,7 +10,7 @@ import Path from 'path';
|
||||||
|
|
||||||
import webpack from 'webpack';
|
import webpack from 'webpack';
|
||||||
import { RawSource } from 'webpack-sources';
|
import { RawSource } from 'webpack-sources';
|
||||||
import { CiStatsMetrics } from '@kbn/dev-utils';
|
import { CiStatsMetric } from '@kbn/dev-utils';
|
||||||
|
|
||||||
import { Bundle } from '../common';
|
import { Bundle } from '../common';
|
||||||
|
|
||||||
|
@ -68,7 +68,7 @@ export class BundleMetricsPlugin {
|
||||||
throw new Error(`moduleCount wasn't populated by PopulateBundleCachePlugin`);
|
throw new Error(`moduleCount wasn't populated by PopulateBundleCachePlugin`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const bundleMetrics: CiStatsMetrics = [
|
const bundleMetrics: CiStatsMetric[] = [
|
||||||
{
|
{
|
||||||
group: `@kbn/optimizer bundle module count`,
|
group: `@kbn/optimizer bundle module count`,
|
||||||
id: bundle.id,
|
id: bundle.id,
|
||||||
|
|
5828
packages/kbn-pm/dist/index.js
vendored
5828
packages/kbn-pm/dist/index.js
vendored
File diff suppressed because it is too large
Load diff
|
@ -23,6 +23,11 @@ export const BootstrapCommand: ICommand = {
|
||||||
description: 'Install dependencies and crosslink projects',
|
description: 'Install dependencies and crosslink projects',
|
||||||
name: 'bootstrap',
|
name: 'bootstrap',
|
||||||
|
|
||||||
|
reportTiming: {
|
||||||
|
group: 'bootstrap',
|
||||||
|
id: 'overall time',
|
||||||
|
},
|
||||||
|
|
||||||
async run(projects, projectGraph, { options, kbn, rootPath }) {
|
async run(projects, projectGraph, { options, kbn, rootPath }) {
|
||||||
const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
|
const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
|
||||||
const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);
|
const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);
|
||||||
|
|
|
@ -18,6 +18,10 @@ export interface ICommandConfig {
|
||||||
export interface ICommand {
|
export interface ICommand {
|
||||||
name: string;
|
name: string;
|
||||||
description: string;
|
description: string;
|
||||||
|
reportTiming?: {
|
||||||
|
group: string;
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
|
||||||
run: (projects: ProjectMap, projectGraph: ProjectGraph, config: ICommandConfig) => Promise<void>;
|
run: (projects: ProjectMap, projectGraph: ProjectGraph, config: ICommandConfig) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,8 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { CiStatsReporter } from '@kbn/dev-utils/ci_stats_reporter';
|
||||||
|
|
||||||
import { ICommand, ICommandConfig } from './commands';
|
import { ICommand, ICommandConfig } from './commands';
|
||||||
import { CliError } from './utils/errors';
|
import { CliError } from './utils/errors';
|
||||||
import { log } from './utils/log';
|
import { log } from './utils/log';
|
||||||
|
@ -14,10 +16,13 @@ import { renderProjectsTree } from './utils/projects_tree';
|
||||||
import { Kibana } from './utils/kibana';
|
import { Kibana } from './utils/kibana';
|
||||||
|
|
||||||
export async function runCommand(command: ICommand, config: Omit<ICommandConfig, 'kbn'>) {
|
export async function runCommand(command: ICommand, config: Omit<ICommandConfig, 'kbn'>) {
|
||||||
|
const runStartTime = Date.now();
|
||||||
|
let kbn;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
log.debug(`Running [${command.name}] command from [${config.rootPath}]`);
|
log.debug(`Running [${command.name}] command from [${config.rootPath}]`);
|
||||||
|
|
||||||
const kbn = await Kibana.loadFrom(config.rootPath);
|
kbn = await Kibana.loadFrom(config.rootPath);
|
||||||
const projects = kbn.getFilteredProjects({
|
const projects = kbn.getFilteredProjects({
|
||||||
skipKibanaPlugins: Boolean(config.options['skip-kibana-plugins']),
|
skipKibanaPlugins: Boolean(config.options['skip-kibana-plugins']),
|
||||||
ossOnly: Boolean(config.options.oss),
|
ossOnly: Boolean(config.options.oss),
|
||||||
|
@ -41,7 +46,46 @@ export async function runCommand(command: ICommand, config: Omit<ICommandConfig,
|
||||||
...config,
|
...config,
|
||||||
kbn,
|
kbn,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (command.reportTiming) {
|
||||||
|
const reporter = CiStatsReporter.fromEnv(log);
|
||||||
|
await reporter.timings({
|
||||||
|
upstreamBranch: kbn.kibanaProject.json.branch,
|
||||||
|
// prevent loading @kbn/utils by passing null
|
||||||
|
kibanaUuid: kbn.getUuid() || null,
|
||||||
|
timings: [
|
||||||
|
{
|
||||||
|
group: command.reportTiming.group,
|
||||||
|
id: command.reportTiming.id,
|
||||||
|
ms: Date.now() - runStartTime,
|
||||||
|
meta: {
|
||||||
|
success: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (command.reportTiming) {
|
||||||
|
// if we don't have a kbn object then things are too broken to report on
|
||||||
|
if (kbn) {
|
||||||
|
const reporter = CiStatsReporter.fromEnv(log);
|
||||||
|
await reporter.timings({
|
||||||
|
upstreamBranch: kbn.kibanaProject.json.branch,
|
||||||
|
timings: [
|
||||||
|
{
|
||||||
|
group: command.reportTiming.group,
|
||||||
|
id: command.reportTiming.id,
|
||||||
|
ms: Date.now() - runStartTime,
|
||||||
|
meta: {
|
||||||
|
success: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.error(`[${command.name}] failed:`);
|
log.error(`[${command.name}] failed:`);
|
||||||
|
|
||||||
if (error instanceof CliError) {
|
if (error instanceof CliError) {
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import Path from 'path';
|
import Path from 'path';
|
||||||
|
import Fs from 'fs';
|
||||||
|
|
||||||
import multimatch from 'multimatch';
|
import multimatch from 'multimatch';
|
||||||
import isPathInside from 'is-path-inside';
|
import isPathInside from 'is-path-inside';
|
||||||
|
@ -146,4 +147,16 @@ export class Kibana {
|
||||||
|
|
||||||
return new Map([...kibanaDeps.entries(), ...xpackDeps.entries()]);
|
return new Map([...kibanaDeps.entries(), ...xpackDeps.entries()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getUuid() {
|
||||||
|
try {
|
||||||
|
return Fs.readFileSync(this.getAbsolute('data/uuid'), 'utf-8').trim();
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'ENOENT') {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ import Path from 'path';
|
||||||
|
|
||||||
import { REPO_ROOT } from '@kbn/utils';
|
import { REPO_ROOT } from '@kbn/utils';
|
||||||
import { lastValueFrom } from '@kbn/std';
|
import { lastValueFrom } from '@kbn/std';
|
||||||
import { CiStatsMetrics } from '@kbn/dev-utils';
|
import { CiStatsMetric } from '@kbn/dev-utils';
|
||||||
import { runOptimizer, OptimizerConfig, logOptimizerState } from '@kbn/optimizer';
|
import { runOptimizer, OptimizerConfig, logOptimizerState } from '@kbn/optimizer';
|
||||||
|
|
||||||
import { Task, deleteAll, write, read } from '../lib';
|
import { Task, deleteAll, write, read } from '../lib';
|
||||||
|
@ -32,11 +32,11 @@ export const BuildKibanaPlatformPlugins: Task = {
|
||||||
|
|
||||||
await lastValueFrom(runOptimizer(config).pipe(logOptimizerState(log, config)));
|
await lastValueFrom(runOptimizer(config).pipe(logOptimizerState(log, config)));
|
||||||
|
|
||||||
const combinedMetrics: CiStatsMetrics = [];
|
const combinedMetrics: CiStatsMetric[] = [];
|
||||||
const metricFilePaths: string[] = [];
|
const metricFilePaths: string[] = [];
|
||||||
for (const bundle of config.bundles) {
|
for (const bundle of config.bundles) {
|
||||||
const path = Path.resolve(bundle.outputDir, 'metrics.json');
|
const path = Path.resolve(bundle.outputDir, 'metrics.json');
|
||||||
const metrics: CiStatsMetrics = JSON.parse(await read(path));
|
const metrics: CiStatsMetric[] = JSON.parse(await read(path));
|
||||||
combinedMetrics.push(...metrics);
|
combinedMetrics.push(...metrics);
|
||||||
metricFilePaths.push(path);
|
metricFilePaths.push(path);
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ import Path from 'path';
|
||||||
import Fs from 'fs';
|
import Fs from 'fs';
|
||||||
import { promisify } from 'util';
|
import { promisify } from 'util';
|
||||||
|
|
||||||
import { CiStatsMetrics } from '@kbn/dev-utils';
|
import { CiStatsMetric } from '@kbn/dev-utils';
|
||||||
|
|
||||||
import { mkdirp, compressTar, compressZip, Task } from '../lib';
|
import { mkdirp, compressTar, compressZip, Task } from '../lib';
|
||||||
|
|
||||||
|
@ -72,7 +72,7 @@ export const CreateArchives: Task = {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const metrics: CiStatsMetrics = [];
|
const metrics: CiStatsMetric[] = [];
|
||||||
for (const { format, path, fileCount } of archives) {
|
for (const { format, path, fileCount } of archives) {
|
||||||
metrics.push({
|
metrics.push({
|
||||||
group: `${build.isOss() ? 'oss ' : ''}distributable size`,
|
group: `${build.isOss() ? 'oss ' : ''}distributable size`,
|
||||||
|
|
Loading…
Reference in a new issue