[ci-stats] ship timings, collect overall bootstrap time (#93557)

Co-authored-by: spalger <spalger@users.noreply.github.com>

docs/developer/index.asciidoc

@@ -12,6 +12,7 @@ running in no time. If you have any problems, file an issue in the https://githu
 * <<kibana-architecture>>
 * <<advanced>>
 * <<plugin-list>>
+* <<development-telemetry>>
 
 --
 
@@ -29,3 +30,5 @@ include::advanced/index.asciidoc[]
 
 include::plugin-list.asciidoc[]
 
+include::telemetry.asciidoc[]
+

docs/developer/telemetry.asciidoc

@@ -0,0 +1,18 @@
+[[development-telemetry]]
+== Development Telemetry
+
+To help us provide a good developer experience, we track some straightforward metrics when running certain tasks locally and ship them to a service that we run. To disable this functionality, specify `CI_STATS_DISABLED=true` in your environment.
+
+The operations we current report timing data for:
+
+* Total execution time of `yarn kbn bootstrap`
+
+Along with the execution time of each execution, we ship the following information about your machine to the service:
+
+* The `branch` property from the package.json file
+* The value of the `data/uuid` file
+* https://nodejs.org/docs/latest/api/os.html#os_os_platform[Operating system platform]
+* https://nodejs.org/docs/latest/api/os.html#os_os_release[Operating system release]
+* https://nodejs.org/docs/latest/api/os.html#os_os_cpus[Count, model, and speed of the CPUs]
+* https://nodejs.org/docs/latest/api/os.html#os_os_arch[CPU architecture]
+* https://nodejs.org/docs/latest/api/os.html#os_os_totalmem[Total memory] and https://nodejs.org/docs/latest/api/os.html#os_os_freemem[Free memory]

packages/kbn-dev-utils/ci_stats_reporter/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "../target/ci_stats_reporter/ci_stats_reporter"
+}

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_config.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ToolingLog } from '../tooling_log';
+
+export interface Config {
+  apiToken: string;
+  buildId: string;
+}
+
+function validateConfig(log: ToolingLog, config: { [k in keyof Config]: unknown }) {
+  const validApiToken = typeof config.apiToken === 'string' && config.apiToken.length !== 0;
+  if (!validApiToken) {
+    log.warning('KIBANA_CI_STATS_CONFIG is missing a valid api token, stats will not be reported');
+    return;
+  }
+
+  const validId = typeof config.buildId === 'string' && config.buildId.length !== 0;
+  if (!validId) {
+    log.warning('KIBANA_CI_STATS_CONFIG is missing a valid build id, stats will not be reported');
+    return;
+  }
+
+  return config as Config;
+}
+
+export function parseConfig(log: ToolingLog) {
+  const configJson = process.env.KIBANA_CI_STATS_CONFIG;
+  if (!configJson) {
+    log.debug('KIBANA_CI_STATS_CONFIG environment variable not found, disabling CiStatsReporter');
+    return;
+  }
+
+  let config: unknown;
+  try {
+    config = JSON.parse(configJson);
+  } catch (_) {
+    // handled below
+  }
+
+  if (typeof config === 'object' && config !== null) {
+    return validateConfig(log, config as { [k in keyof Config]: unknown });
+  }
+
+  log.warning('KIBANA_CI_STATS_CONFIG is invalid, stats will not be reported');
+  return;
+}

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -7,69 +7,50 @@
  */
 
 import { inspect } from 'util';
+import Os from 'os';
+import Fs from 'fs';
+import Path from 'path';
 
 import Axios from 'axios';
 
 import { ToolingLog } from '../tooling_log';
+import { parseConfig, Config } from './ci_stats_config';
 
-interface Config {
-  apiUrl: string;
-  apiToken: string;
-  buildId: string;
-}
+const BASE_URL = 'https://ci-stats.kibana.dev';
 
-export type CiStatsMetrics = Array<{
+export interface CiStatsMetric {
   group: string;
   id: string;
   value: number;
   limit?: number;
   limitConfigPath?: string;
-}>;
-
-function parseConfig(log: ToolingLog) {
-  const configJson = process.env.KIBANA_CI_STATS_CONFIG;
-  if (!configJson) {
-    log.debug('KIBANA_CI_STATS_CONFIG environment variable not found, disabling CiStatsReporter');
-    return;
-  }
-
-  let config: unknown;
-  try {
-    config = JSON.parse(configJson);
-  } catch (_) {
-    // handled below
-  }
-
-  if (typeof config === 'object' && config !== null) {
-    return validateConfig(log, config as { [k in keyof Config]: unknown });
-  }
-
-  log.warning('KIBANA_CI_STATS_CONFIG is invalid, stats will not be reported');
-  return;
 }
 
-function validateConfig(log: ToolingLog, config: { [k in keyof Config]: unknown }) {
-  const validApiUrl = typeof config.apiUrl === 'string' && config.apiUrl.length !== 0;
-  if (!validApiUrl) {
-    log.warning('KIBANA_CI_STATS_CONFIG is missing a valid api url, stats will not be reported');
-    return;
-  }
-
-  const validApiToken = typeof config.apiToken === 'string' && config.apiToken.length !== 0;
-  if (!validApiToken) {
-    log.warning('KIBANA_CI_STATS_CONFIG is missing a valid api token, stats will not be reported');
-    return;
-  }
-
-  const validId = typeof config.buildId === 'string' && config.buildId.length !== 0;
-  if (!validId) {
-    log.warning('KIBANA_CI_STATS_CONFIG is missing a valid build id, stats will not be reported');
-    return;
-  }
-
-  return config as Config;
+export interface CiStatsTimingMetadata {
+  [key: string]: string | string[] | number | boolean | undefined;
+}
+export interface CiStatsTiming {
+  group: string;
+  id: string;
+  ms: number;
+  meta?: CiStatsTimingMetadata;
 }
 
+export interface ReqOptions {
+  auth: boolean;
+  path: string;
+  body: any;
+  bodyDesc: string;
+}
+
+export interface TimingsOptions {
+  /** list of timings to record */
+  timings: CiStatsTiming[];
+  /** master, 7.x, etc, automatically detected from package.json if not specified */
+  upstreamBranch?: string;
+  /** value of data/uuid, automatically loaded if not specified */
+  kibanaUuid?: string | null;
+}
 export class CiStatsReporter {
   static fromEnv(log: ToolingLog) {
     return new CiStatsReporter(parseConfig(log), log);
@@ -78,19 +59,126 @@ export class CiStatsReporter {
   constructor(private config: Config | undefined, private log: ToolingLog) {}
 
   isEnabled() {
-    return !!this.config;
+    return process.env.CI_STATS_DISABLED !== 'true';
   }
 
-  async metrics(metrics: CiStatsMetrics) {
-    if (!this.config) {
+  hasBuildConfig() {
+    return this.isEnabled() && !!this.config?.apiToken && !!this.config?.buildId;
+  }
+
+  /**
+   * Report timings data to the ci-stats service. If running in CI then the reporter
+   * will include the buildId in the report with the access token, otherwise the timings
+   * data will be recorded as anonymous timing data.
+   */
+  async timings(options: TimingsOptions) {
+    if (!this.isEnabled()) {
       return;
     }
 
+    const buildId = this.config?.buildId;
+    const timings = options.timings;
+    const upstreamBranch = options.upstreamBranch ?? this.getUpstreamBranch();
+    const kibanaUuid = options.kibanaUuid === undefined ? this.getKibanaUuid() : options.kibanaUuid;
+    const defaultMetadata = {
+      osPlatform: Os.platform(),
+      osRelease: Os.release(),
+      osArch: Os.arch(),
+      cpuCount: Os.cpus()?.length,
+      cpuModel: Os.cpus()[0]?.model,
+      cpuSpeed: Os.cpus()[0]?.speed,
+      freeMem: Os.freemem(),
+      totalMem: Os.totalmem(),
+      kibanaUuid,
+    };
+
+    return await this.req({
+      auth: !!buildId,
+      path: '/v1/timings',
+      body: {
+        buildId,
+        upstreamBranch,
+        timings,
+        defaultMetadata,
+      },
+      bodyDesc: timings.length === 1 ? `${timings.length} timing` : `${timings.length} timings`,
+    });
+  }
+
+  /**
+   * Report metrics data to the ci-stats service. If running outside of CI this method
+   * does nothing as metrics can only be reported when associated with a specific CI build.
+   */
+  async metrics(metrics: CiStatsMetric[]) {
+    if (!this.hasBuildConfig()) {
+      return;
+    }
+
+    const buildId = this.config?.buildId;
+
+    if (!buildId) {
+      throw new Error(`CiStatsReporter can't be authorized without a buildId`);
+    }
+
+    return await this.req({
+      auth: true,
+      path: '/v1/metrics',
+      body: {
+        buildId,
+        metrics,
+      },
+      bodyDesc: `metrics: ${metrics
+        .map(({ group, id, value }) => `[${group}/${id}=${value}]`)
+        .join(' ')}`,
+    });
+  }
+
+  /**
+   * In order to allow this code to run before @kbn/utils is built, @kbn/pm will pass
+   * in the upstreamBranch when calling the timings() method. Outside of @kbn/pm
+   * we rely on @kbn/utils to find the package.json file.
+   */
+  private getUpstreamBranch() {
+    // specify the module id in a way that will keep webpack from bundling extra code into @kbn/pm
+    const hideFromWebpack = ['@', 'kbn/utils'];
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { kibanaPackageJson } = require(hideFromWebpack.join(''));
+    return kibanaPackageJson.branch;
+  }
+
+  /**
+   * In order to allow this code to run before @kbn/utils is built, @kbn/pm will pass
+   * in the kibanaUuid when calling the timings() method. Outside of @kbn/pm
+   * we rely on @kbn/utils to find the repo root.
+   */
+  private getKibanaUuid() {
+    // specify the module id in a way that will keep webpack from bundling extra code into @kbn/pm
+    const hideFromWebpack = ['@', 'kbn/utils'];
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { REPO_ROOT } = require(hideFromWebpack.join(''));
+    try {
+      return Fs.readFileSync(Path.resolve(REPO_ROOT, 'data/uuid'), 'utf-8').trim();
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return undefined;
+      }
+
+      throw error;
+    }
+  }
+
+  private async req({ auth, body, bodyDesc, path }: ReqOptions) {
     let attempt = 0;
     const maxAttempts = 5;
-    const bodySummary = metrics
-      .map(({ group, id, value }) => `[${group}/${id}=${value}]`)
-      .join(' ');
+
+    let headers;
+    if (auth && this.config) {
+      headers = {
+        Authorization: `token ${this.config.apiToken}`,
+      };
+    } else if (auth) {
+      throw new Error('this.req() shouldnt be called with auth=true if this.config is defined');
+    }
 
     while (true) {
       attempt += 1;
@@ -98,15 +186,10 @@ export class CiStatsReporter {
       try {
         await Axios.request({
           method: 'POST',
-          url: '/v1/metrics',
-          baseURL: this.config.apiUrl,
-          headers: {
-            Authorization: `token ${this.config.apiToken}`,
-          },
-          data: {
-            buildId: this.config.buildId,
-            metrics,
-          },
+          url: path,
+          baseURL: BASE_URL,
+          headers,
+          data: body,
         });
 
         return true;
@@ -116,19 +199,19 @@ export class CiStatsReporter {
           throw error;
         }
 
-        if (error?.response && error.response.status !== 502) {
+        if (error?.response && error.response.status < 502) {
           // error response from service was received so warn the user and move on
           this.log.warning(
-            `error recording metric [status=${error.response.status}] [resp=${inspect(
+            `error reporting ${bodyDesc} [status=${error.response.status}] [resp=${inspect(
               error.response.data
-            )}] ${bodySummary}`
+            )}]`
           );
           return;
         }
 
         if (attempt === maxAttempts) {
           this.log.warning(
-            `failed to reach kibana-ci-stats service too many times, unable to record metric ${bodySummary}`
+            `unable to report ${bodyDesc}, failed to reach ci-stats service too many times`
           );
           return;
         }
@@ -139,7 +222,7 @@ export class CiStatsReporter {
           : 'no response';
 
         this.log.warning(
-          `failed to reach kibana-ci-stats service [reason=${reason}], retrying in ${attempt} seconds`
+          `failed to reach ci-stats service [reason=${reason}], retrying in ${attempt} seconds`
         );
 
         await new Promise((resolve) => setTimeout(resolve, attempt * 1000));

packages/kbn-optimizer/src/limits.ts

@@ -11,7 +11,7 @@ import Path from 'path';
 
 import dedent from 'dedent';
 import Yaml from 'js-yaml';
-import { createFailError, ToolingLog, CiStatsMetrics } from '@kbn/dev-utils';
+import { createFailError, ToolingLog, CiStatsMetric } from '@kbn/dev-utils';
 
 import { OptimizerConfig, Limits } from './optimizer';
 
@@ -86,7 +86,7 @@ export function updateBundleLimits({
   limitsPath,
 }: UpdateBundleLimitsOptions) {
   const limits = readLimits(limitsPath);
-  const metrics: CiStatsMetrics = config.bundles
+  const metrics: CiStatsMetric[] = config.bundles
     .map((bundle) =>
       JSON.parse(Fs.readFileSync(Path.resolve(bundle.outputDir, 'metrics.json'), 'utf-8'))
     )

packages/kbn-optimizer/src/worker/bundle_metrics_plugin.ts

@@ -10,7 +10,7 @@ import Path from 'path';
 
 import webpack from 'webpack';
 import { RawSource } from 'webpack-sources';
-import { CiStatsMetrics } from '@kbn/dev-utils';
+import { CiStatsMetric } from '@kbn/dev-utils';
 
 import { Bundle } from '../common';
 
@@ -68,7 +68,7 @@ export class BundleMetricsPlugin {
         throw new Error(`moduleCount wasn't populated by PopulateBundleCachePlugin`);
       }
 
-      const bundleMetrics: CiStatsMetrics = [
+      const bundleMetrics: CiStatsMetric[] = [
         {
           group: `@kbn/optimizer bundle module count`,
           id: bundle.id,

packages/kbn-pm/src/commands/bootstrap.ts

@@ -23,6 +23,11 @@ export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
   name: 'bootstrap',
 
+  reportTiming: {
+    group: 'bootstrap',
+    id: 'overall time',
+  },
+
   async run(projects, projectGraph, { options, kbn, rootPath }) {
     const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
     const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);

packages/kbn-pm/src/commands/index.ts

@@ -18,6 +18,10 @@ export interface ICommandConfig {
 export interface ICommand {
   name: string;
   description: string;
+  reportTiming?: {
+    group: string;
+    id: string;
+  };
 
   run: (projects: ProjectMap, projectGraph: ProjectGraph, config: ICommandConfig) => Promise<void>;
 }

packages/kbn-pm/src/run.ts

@@ -6,6 +6,8 @@
  * Side Public License, v 1.
  */
 
+import { CiStatsReporter } from '@kbn/dev-utils/ci_stats_reporter';
+
 import { ICommand, ICommandConfig } from './commands';
 import { CliError } from './utils/errors';
 import { log } from './utils/log';
@@ -14,10 +16,13 @@ import { renderProjectsTree } from './utils/projects_tree';
 import { Kibana } from './utils/kibana';
 
 export async function runCommand(command: ICommand, config: Omit<ICommandConfig, 'kbn'>) {
+  const runStartTime = Date.now();
+  let kbn;
+
   try {
     log.debug(`Running [${command.name}] command from [${config.rootPath}]`);
 
-    const kbn = await Kibana.loadFrom(config.rootPath);
+    kbn = await Kibana.loadFrom(config.rootPath);
     const projects = kbn.getFilteredProjects({
       skipKibanaPlugins: Boolean(config.options['skip-kibana-plugins']),
       ossOnly: Boolean(config.options.oss),
@@ -41,7 +46,46 @@ export async function runCommand(command: ICommand, config: Omit<ICommandConfig,
       ...config,
       kbn,
     });
+
+    if (command.reportTiming) {
+      const reporter = CiStatsReporter.fromEnv(log);
+      await reporter.timings({
+        upstreamBranch: kbn.kibanaProject.json.branch,
+        // prevent loading @kbn/utils by passing null
+        kibanaUuid: kbn.getUuid() || null,
+        timings: [
+          {
+            group: command.reportTiming.group,
+            id: command.reportTiming.id,
+            ms: Date.now() - runStartTime,
+            meta: {
+              success: true,
+            },
+          },
+        ],
+      });
+    }
   } catch (error) {
+    if (command.reportTiming) {
+      // if we don't have a kbn object then things are too broken to report on
+      if (kbn) {
+        const reporter = CiStatsReporter.fromEnv(log);
+        await reporter.timings({
+          upstreamBranch: kbn.kibanaProject.json.branch,
+          timings: [
+            {
+              group: command.reportTiming.group,
+              id: command.reportTiming.id,
+              ms: Date.now() - runStartTime,
+              meta: {
+                success: false,
+              },
+            },
+          ],
+        });
+      }
+    }
+
     log.error(`[${command.name}] failed:`);
 
     if (error instanceof CliError) {

packages/kbn-pm/src/utils/kibana.ts

@@ -7,6 +7,7 @@
  */
 
 import Path from 'path';
+import Fs from 'fs';
 
 import multimatch from 'multimatch';
 import isPathInside from 'is-path-inside';
@@ -146,4 +147,16 @@ export class Kibana {
 
     return new Map([...kibanaDeps.entries(), ...xpackDeps.entries()]);
   }
+
+  getUuid() {
+    try {
+      return Fs.readFileSync(this.getAbsolute('data/uuid'), 'utf-8').trim();
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return undefined;
+      }
+
+      throw error;
+    }
+  }
 }

src/dev/build/tasks/build_kibana_platform_plugins.ts

@@ -10,7 +10,7 @@ import Path from 'path';
 
 import { REPO_ROOT } from '@kbn/utils';
 import { lastValueFrom } from '@kbn/std';
-import { CiStatsMetrics } from '@kbn/dev-utils';
+import { CiStatsMetric } from '@kbn/dev-utils';
 import { runOptimizer, OptimizerConfig, logOptimizerState } from '@kbn/optimizer';
 
 import { Task, deleteAll, write, read } from '../lib';
@@ -32,11 +32,11 @@ export const BuildKibanaPlatformPlugins: Task = {
 
     await lastValueFrom(runOptimizer(config).pipe(logOptimizerState(log, config)));
 
-    const combinedMetrics: CiStatsMetrics = [];
+    const combinedMetrics: CiStatsMetric[] = [];
     const metricFilePaths: string[] = [];
     for (const bundle of config.bundles) {
       const path = Path.resolve(bundle.outputDir, 'metrics.json');
-      const metrics: CiStatsMetrics = JSON.parse(await read(path));
+      const metrics: CiStatsMetric[] = JSON.parse(await read(path));
       combinedMetrics.push(...metrics);
       metricFilePaths.push(path);
     }

src/dev/build/tasks/create_archives_task.ts

@@ -10,7 +10,7 @@ import Path from 'path';
 import Fs from 'fs';
 import { promisify } from 'util';
 
-import { CiStatsMetrics } from '@kbn/dev-utils';
+import { CiStatsMetric } from '@kbn/dev-utils';
 
 import { mkdirp, compressTar, compressZip, Task } from '../lib';
 
@@ -72,7 +72,7 @@ export const CreateArchives: Task = {
       }
     }
 
-    const metrics: CiStatsMetrics = [];
+    const metrics: CiStatsMetric[] = [];
     for (const { format, path, fileCount } of archives) {
       metrics.push({
         group: `${build.isOss() ? 'oss ' : ''}distributable size`,