[Telemetry] Add possibility of registering exclusive collectors for each collection (#62665)

* [Telemetry] Add posibility of regitering exclusive collectors for collections

* [Telemetry] Filter unwanted fields from the kibana.os telemetry payload

* Filter the collectors properly in bulkFetch

* Move "kibana" usage collector from Monitoring to OSS Telemetry

* Remove exclusivity of the "kibana_settings" collector

* Unify "kibana_stats" collector from Monitoring and Legacy

* Remove unused legacy constants

* Proper type for UsageCollectionSetup in monitoring

* Missed one undo

* Add unit tests to the migrated collectors

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

src/legacy/server/status/collectors/get_ops_stats_collector.js

@@ -1,56 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { KIBANA_STATS_TYPE } from '../constants';
-import { getKibanaInfoForStats } from '../lib';
-
-/*
- * Initialize a collector for Kibana Ops Stats
- *
- * NOTE this collector's fetch method returns the latest stats from the
- * Hapi/Good/Even-Better ops event listener. Therefore, the stats reset
- * every 5 seconds (the default value of the ops.interval configuration
- * setting). That makes it geared for providing the latest "real-time"
- * stats. In the long-term, fetch should return stats that constantly
- * accumulate over the server's uptime for better machine readability.
- * Since the data is captured, timestamped and stored, the historical
- * data can provide "real-time" stats by calculating a derivative of
- * the metrics.
- * See PR comment in https://github.com/elastic/kibana/pull/20577/files#r202416647
- */
-export function getOpsStatsCollector(usageCollection, server, kbnServer) {
-  return usageCollection.makeStatsCollector({
-    type: KIBANA_STATS_TYPE,
-    fetch: () => {
-      return {
-        kibana: getKibanaInfoForStats(server, kbnServer),
-        ...kbnServer.metrics, // latest metrics captured from the ops event listener in src/legacy/server/status/index
-      };
-    },
-    isReady: () => true,
-    ignoreForInternalUploader: true, // Ignore this one from internal uploader. A different stats collector is used there.
-  });
-}
-
-export function registerOpsStatsCollector(usageCollection, server, kbnServer) {
-  if (usageCollection) {
-    const collector = getOpsStatsCollector(usageCollection, server, kbnServer);
-    usageCollection.registerCollector(collector);
-  }
-}

src/legacy/server/status/index.js

@@ -20,7 +20,6 @@
 import ServerStatus from './server_status';
 import { Metrics } from './lib/metrics';
 import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes';
-import { registerOpsStatsCollector } from './collectors';
 import Oppsy from 'oppsy';
 import { cloneDeep } from 'lodash';
 import { getOSInfo } from './lib/get_os_info';
@@ -28,7 +27,6 @@ import { getOSInfo } from './lib/get_os_info';
 export function statusMixin(kbnServer, server, config) {
   kbnServer.status = new ServerStatus(kbnServer.server);
   const { usageCollection } = server.newPlatform.setup.plugins;
-  registerOpsStatsCollector(usageCollection, server, kbnServer);
 
   const metrics = new Metrics(config, server);
 

src/plugins/telemetry/common/constants.ts

@@ -80,3 +80,14 @@ export const APPLICATION_USAGE_TYPE = 'application_usage';
  * The type name used within the Monitoring index to publish management stats.
  */
 export const KIBANA_STACK_MANAGEMENT_STATS_TYPE = 'stack_management';
+
+/**
+ * The type name used to publish Kibana usage stats.
+ * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats
+ */
+export const KIBANA_USAGE_TYPE = 'kibana';
+
+/**
+ * The type name used to publish Kibana usage stats in the formatted as bulk.
+ */
+export const KIBANA_STATS_TYPE = 'kibana_stats';

src/plugins/telemetry/server/collectors/index.ts

@@ -22,3 +22,5 @@ export { registerUiMetricUsageCollector } from './ui_metric';
 export { registerTelemetryPluginUsageCollector } from './telemetry_plugin';
 export { registerManagementUsageCollector } from './management';
 export { registerApplicationUsageCollector } from './application_usage';
+export { registerKibanaUsageCollector } from './kibana';
+export { registerOpsStatsCollector } from './ops_stats';

src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getSavedObjectsCounts } from './get_saved_object_counts';
+
+describe('getSavedObjectsCounts', () => {
+  test('Get all the saved objects equal to 0 because no results were found', async () => {
+    const callCluster = jest.fn(() => ({}));
+
+    const results = await getSavedObjectsCounts(callCluster as any, '.kibana');
+    expect(results).toStrictEqual({
+      dashboard: { total: 0 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 0 },
+      timelion_sheet: { total: 0 },
+    });
+  });
+
+  test('Merge the zeros with the results', async () => {
+    const callCluster = jest.fn(() => ({
+      aggregations: {
+        types: {
+          buckets: [
+            { key: 'dashboard', doc_count: 1 },
+            { key: 'timelion-sheet', doc_count: 2 },
+            { key: 'index-pattern', value: 2 }, // Malformed on purpose
+            { key: 'graph_workspace', doc_count: 3 }, // already snake_cased
+          ],
+        },
+      },
+    }));
+
+    const results = await getSavedObjectsCounts(callCluster as any, '.kibana');
+    expect(results).toStrictEqual({
+      dashboard: { total: 1 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 3 },
+      timelion_sheet: { total: 2 },
+    });
+  });
+});

src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.ts

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Moved from /x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts
+ *
+ * The PR https://github.com/elastic/kibana/pull/62665 proved what the issue https://github.com/elastic/kibana/issues/58249
+ * was claiming: the structure and payload for common telemetry bits differs between Monitoring and OSS/X-Pack collections.
+ *
+ * Unifying this logic from Monitoring that makes sense to have in OSS here and we will import it on the monitoring side to reuse it.
+ */
+
+import { snakeCase } from 'lodash';
+import { APICaller } from 'kibana/server';
+
+const TYPES = [
+  'dashboard',
+  'visualization',
+  'search',
+  'index-pattern',
+  'graph-workspace',
+  'timelion-sheet',
+];
+
+export interface KibanaSavedObjectCounts {
+  [pluginName: string]: {
+    total: number;
+  };
+}
+
+export async function getSavedObjectsCounts(
+  callCluster: APICaller,
+  kibanaIndex: string // Typically '.kibana'. We might need a way to obtain it from the SavedObjects client (or the SavedObjects client to provide a way to run aggregations?)
+): Promise<KibanaSavedObjectCounts> {
+  const savedObjectCountSearchParams = {
+    index: kibanaIndex,
+    ignoreUnavailable: true,
+    filterPath: 'aggregations.types.buckets',
+    body: {
+      size: 0,
+      query: {
+        terms: { type: TYPES },
+      },
+      aggs: {
+        types: {
+          terms: { field: 'type', size: TYPES.length },
+        },
+      },
+    },
+  };
+  const resp = await callCluster('search', savedObjectCountSearchParams);
+  const buckets: Array<{ key: string; doc_count: number }> =
+    resp.aggregations?.types?.buckets || [];
+
+  // Initialise the object with all zeros for all the types
+  const allZeros: KibanaSavedObjectCounts = TYPES.reduce(
+    (acc, type) => ({ ...acc, [snakeCase(type)]: { total: 0 } }),
+    {}
+  );
+
+  // Add the doc_count from each bucket
+  return buckets.reduce(
+    (acc, { key, doc_count: total }) => (total ? { ...acc, [snakeCase(key)]: { total } } : acc),
+    allZeros
+  );
+}

src/plugins/telemetry/server/collectors/kibana/index.test.ts

@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { UsageCollectionSetup } from '../../../../../plugins/usage_collection/server';
+import { pluginInitializerContextConfigMock } from '../../../../../core/server/mocks';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { CollectorOptions } from '../../../../../plugins/usage_collection/server/collector/collector';
+
+import { registerKibanaUsageCollector } from './';
+
+describe('telemetry_kibana', () => {
+  let collector: CollectorOptions;
+
+  const usageCollectionMock: jest.Mocked<UsageCollectionSetup> = {
+    makeUsageCollector: jest.fn().mockImplementation(config => (collector = config)),
+    registerCollector: jest.fn(),
+  } as any;
+
+  const legacyConfig$ = pluginInitializerContextConfigMock({}).legacy.globalConfig$;
+  const callCluster = jest.fn().mockImplementation(() => ({}));
+
+  beforeAll(() => registerKibanaUsageCollector(usageCollectionMock, legacyConfig$));
+  afterAll(() => jest.clearAllTimers());
+
+  test('registered collector is set', () => {
+    expect(collector).not.toBeUndefined();
+    expect(collector.type).toBe('kibana');
+  });
+
+  test('fetch', async () => {
+    expect(await collector.fetch(callCluster)).toStrictEqual({
+      index: '.kibana-tests',
+      dashboard: { total: 0 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 0 },
+      timelion_sheet: { total: 0 },
+    });
+  });
+
+  test('formatForBulkUpload', async () => {
+    const resultFromFetch = {
+      index: '.kibana-tests',
+      dashboard: { total: 0 },
+      visualization: { total: 0 },
+      search: { total: 0 },
+      index_pattern: { total: 0 },
+      graph_workspace: { total: 0 },
+      timelion_sheet: { total: 0 },
+    };
+
+    expect(collector.formatForBulkUpload!(resultFromFetch)).toStrictEqual({
+      type: 'kibana_stats',
+      payload: {
+        usage: resultFromFetch,
+      },
+    });
+  });
+});

src/plugins/telemetry/server/collectors/kibana/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export const KIBANA_STATS_TYPE = 'oss_kibana_stats'; // kibana stats per 5s intervals
+export { registerKibanaUsageCollector } from './kibana_usage_collector';

src/plugins/telemetry/server/collectors/kibana/kibana_usage_collector.ts

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Observable } from 'rxjs';
+import { take } from 'rxjs/operators';
+import { SharedGlobalConfig } from 'kibana/server';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { KIBANA_STATS_TYPE, KIBANA_USAGE_TYPE } from '../../../common/constants';
+import { getSavedObjectsCounts } from './get_saved_object_counts';
+
+export function getKibanaUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  legacyConfig$: Observable<SharedGlobalConfig>
+) {
+  return usageCollection.makeUsageCollector({
+    type: KIBANA_USAGE_TYPE,
+    isReady: () => true,
+    async fetch(callCluster) {
+      const {
+        kibana: { index },
+      } = await legacyConfig$.pipe(take(1)).toPromise();
+      return {
+        index,
+        ...(await getSavedObjectsCounts(callCluster, index)),
+      };
+    },
+
+    /*
+     * Format the response data into a model for internal upload
+     * 1. Make this data part of the "kibana_stats" type
+     * 2. Organize the payload in the usage namespace of the data payload (usage.index, etc)
+     */
+    formatForBulkUpload: result => {
+      return {
+        type: KIBANA_STATS_TYPE,
+        payload: {
+          usage: result,
+        },
+      };
+    },
+  });
+}
+
+export function registerKibanaUsageCollector(
+  usageCollection: UsageCollectionSetup,
+  legacyConfig$: Observable<SharedGlobalConfig>
+) {
+  usageCollection.registerCollector(getKibanaUsageCollector(usageCollection, legacyConfig$));
+}

src/plugins/telemetry/server/collectors/ops_stats/__snapshots__/index.test.ts.snap

@@ -0,0 +1,43 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`telemetry_ops_stats should return something when there is a metric 1`] = `
+Object {
+  "concurrent_connections": 20,
+  "os": Object {
+    "load": Object {
+      "15m": 3,
+      "1m": 0.5,
+      "5m": 1,
+    },
+    "memory": Object {
+      "free_in_bytes": 10,
+      "total_in_bytes": 10,
+      "used_in_bytes": 10,
+    },
+    "platform": "darwin",
+    "platformRelease": "test",
+    "uptime_in_millis": 1000,
+  },
+  "process": Object {
+    "event_loop_delay": 10,
+    "memory": Object {
+      "heap": Object {
+        "size_limit": 0,
+        "total_in_bytes": 0,
+        "used_in_bytes": 0,
+      },
+      "resident_set_size_in_bytes": 0,
+    },
+    "uptime_in_millis": 1000,
+  },
+  "requests": Object {
+    "disconnects": 10,
+    "total": 100,
+  },
+  "response_times": Object {
+    "average": 100,
+    "max": 200,
+  },
+  "timestamp": Any<String>,
+}
+`;

src/plugins/telemetry/server/collectors/ops_stats/index.test.ts

@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Subject } from 'rxjs';
+import { UsageCollectionSetup } from '../../../../../plugins/usage_collection/server';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { CollectorOptions } from '../../../../../plugins/usage_collection/server/collector/collector';
+
+import { registerOpsStatsCollector } from './';
+import { OpsMetrics } from '../../../../../core/server';
+
+describe('telemetry_ops_stats', () => {
+  let collector: CollectorOptions;
+
+  const usageCollectionMock: jest.Mocked<UsageCollectionSetup> = {
+    makeStatsCollector: jest.fn().mockImplementation(config => (collector = config)),
+    registerCollector: jest.fn(),
+  } as any;
+
+  const metrics$ = new Subject<OpsMetrics>();
+  const callCluster = jest.fn();
+
+  const metric: OpsMetrics = {
+    process: {
+      memory: {
+        heap: {
+          total_in_bytes: 0,
+          used_in_bytes: 0,
+          size_limit: 0,
+        },
+        resident_set_size_in_bytes: 0,
+      },
+      event_loop_delay: 10,
+      pid: 10,
+      uptime_in_millis: 1000,
+    },
+    os: {
+      platform: 'darwin',
+      platformRelease: 'test',
+      load: {
+        '1m': 0.5,
+        '5m': 1,
+        '15m': 3,
+      },
+      memory: {
+        total_in_bytes: 10,
+        free_in_bytes: 10,
+        used_in_bytes: 10,
+      },
+      uptime_in_millis: 1000,
+    },
+    response_times: { avg_in_millis: 100, max_in_millis: 200 },
+    requests: {
+      disconnects: 10,
+      total: 100,
+      statusCodes: { 200: 100 },
+    },
+    concurrent_connections: 20,
+  };
+
+  beforeAll(() => registerOpsStatsCollector(usageCollectionMock, metrics$));
+  afterAll(() => jest.clearAllTimers());
+
+  test('registered collector is set', () => {
+    expect(collector).not.toBeUndefined();
+    expect(collector.type).toBe('kibana_stats');
+  });
+
+  test('isReady should return false because no metrics have been provided yet', () => {
+    expect(collector.isReady()).toBe(false);
+  });
+
+  test('should return something when there is a metric', async () => {
+    metrics$.next(metric);
+    expect(collector.isReady()).toBe(true);
+    expect(await collector.fetch(callCluster)).toMatchSnapshot({
+      concurrent_connections: 20,
+      os: {
+        load: {
+          '15m': 3,
+          '1m': 0.5,
+          '5m': 1,
+        },
+        memory: {
+          free_in_bytes: 10,
+          total_in_bytes: 10,
+          used_in_bytes: 10,
+        },
+        platform: 'darwin',
+        platformRelease: 'test',
+        uptime_in_millis: 1000,
+      },
+      process: {
+        event_loop_delay: 10,
+        memory: {
+          heap: {
+            size_limit: 0,
+            total_in_bytes: 0,
+            used_in_bytes: 0,
+          },
+          resident_set_size_in_bytes: 0,
+        },
+        uptime_in_millis: 1000,
+      },
+      requests: {
+        disconnects: 10,
+        total: 100,
+      },
+      response_times: {
+        average: 100,
+        max: 200,
+      },
+      timestamp: expect.any(String),
+    });
+  });
+});

src/plugins/telemetry/server/collectors/ops_stats/index.ts

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { registerOpsStatsCollector } from './get_ops_stats_collector';
+export { registerOpsStatsCollector } from './ops_stats_collector';

src/plugins/telemetry/server/collectors/ops_stats/ops_stats_collector.ts

@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Observable } from 'rxjs';
+import { cloneDeep } from 'lodash';
+import moment from 'moment';
+import { OpsMetrics } from 'kibana/server';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { KIBANA_STATS_TYPE } from '../../../common/constants';
+
+interface OpsStatsMetrics extends Omit<OpsMetrics, 'response_times'> {
+  timestamp: string;
+  response_times: {
+    average: number;
+    max: number;
+  };
+}
+
+/**
+ * Initialize a collector for Kibana Ops Stats
+ */
+export function getOpsStatsCollector(
+  usageCollection: UsageCollectionSetup,
+  metrics$: Observable<OpsMetrics>
+) {
+  let lastMetrics: OpsStatsMetrics | null = null;
+  metrics$.subscribe(_metrics => {
+    const metrics = cloneDeep(_metrics);
+    // Ensure we only include the same data that Metricbeat collection would get
+    delete metrics.process.pid;
+    const responseTimes = {
+      average: metrics.response_times.avg_in_millis,
+      max: metrics.response_times.max_in_millis,
+    };
+    delete metrics.requests.statusCodes;
+    lastMetrics = {
+      ...metrics,
+      response_times: responseTimes,
+      timestamp: moment.utc().toISOString(),
+    };
+  });
+
+  return usageCollection.makeStatsCollector({
+    type: KIBANA_STATS_TYPE,
+    isReady: () => !!lastMetrics,
+    fetch: () => lastMetrics,
+  });
+}
+
+export function registerOpsStatsCollector(
+  usageCollection: UsageCollectionSetup,
+  metrics$: Observable<OpsMetrics>
+) {
+  usageCollection.registerCollector(getOpsStatsCollector(usageCollection, metrics$));
+}

src/plugins/telemetry/server/plugin.ts

@@ -32,6 +32,8 @@ import {
   SavedObjectsClient,
   Plugin,
   Logger,
+  SharedGlobalConfig,
+  MetricsServiceSetup,
 } from '../../../core/server';
 import { registerRoutes } from './routes';
 import { registerCollection } from './telemetry_collection';
@@ -41,6 +43,8 @@ import {
   registerTelemetryPluginUsageCollector,
   registerManagementUsageCollector,
   registerApplicationUsageCollector,
+  registerKibanaUsageCollector,
+  registerOpsStatsCollector,
 } from './collectors';
 import { TelemetryConfigType } from './config';
 import { FetcherTask } from './fetcher';
@@ -61,6 +65,7 @@ export class TelemetryPlugin implements Plugin {
   private readonly logger: Logger;
   private readonly currentKibanaVersion: string;
   private readonly config$: Observable<TelemetryConfigType>;
+  private readonly legacyConfig$: Observable<SharedGlobalConfig>;
   private readonly isDev: boolean;
   private readonly fetcherTask: FetcherTask;
   private savedObjectsClient?: ISavedObjectsRepository;
@@ -71,6 +76,7 @@ export class TelemetryPlugin implements Plugin {
     this.isDev = initializerContext.env.mode.dev;
     this.currentKibanaVersion = initializerContext.env.packageInfo.version;
     this.config$ = initializerContext.config.create();
+    this.legacyConfig$ = initializerContext.config.legacy.globalConfig$;
     this.fetcherTask = new FetcherTask({
       ...initializerContext,
       logger: this.logger,
@@ -78,15 +84,15 @@ export class TelemetryPlugin implements Plugin {
   }
 
   public async setup(
-    core: CoreSetup,
+    { elasticsearch, http, savedObjects, metrics }: CoreSetup,
     { usageCollection, telemetryCollectionManager }: TelemetryPluginsSetup
   ) {
     const currentKibanaVersion = this.currentKibanaVersion;
     const config$ = this.config$;
     const isDev = this.isDev;
 
-    registerCollection(telemetryCollectionManager, core.elasticsearch.dataClient);
-    const router = core.http.createRouter();
+    registerCollection(telemetryCollectionManager, elasticsearch.dataClient);
+    const router = http.createRouter();
 
     registerRoutes({
       config$,
@@ -96,8 +102,8 @@ export class TelemetryPlugin implements Plugin {
       telemetryCollectionManager,
     });
 
-    this.registerMappings(opts => core.savedObjects.registerType(opts));
-    this.registerUsageCollectors(usageCollection, opts => core.savedObjects.registerType(opts));
+    this.registerMappings(opts => savedObjects.registerType(opts));
+    this.registerUsageCollectors(usageCollection, metrics, opts => savedObjects.registerType(opts));
   }
 
   public async start(core: CoreStart, { telemetryCollectionManager }: TelemetryPluginsStart) {
@@ -153,11 +159,14 @@ export class TelemetryPlugin implements Plugin {
 
   private registerUsageCollectors(
     usageCollection: UsageCollectionSetup,
+    metrics: MetricsServiceSetup,
     registerType: SavedObjectsRegisterType
   ) {
     const getSavedObjectsClient = () => this.savedObjectsClient;
     const getUiSettingsClient = () => this.uiSettingsClient;
 
+    registerOpsStatsCollector(usageCollection, metrics.getOpsMetrics$());
+    registerKibanaUsageCollector(usageCollection, this.legacyConfig$);
     registerTelemetryPluginUsageCollector(usageCollection, {
       currentKibanaVersion: this.currentKibanaVersion,
       config$: this.config$,

src/plugins/telemetry/server/telemetry_collection/get_kibana.ts

@@ -55,6 +55,10 @@ export function handleKibanaStats(
     ...kibanaStats.os,
   };
   const formattedOsStats = Object.entries(os).reduce((acc, [key, value]) => {
+    if (typeof value !== 'string') {
+      // There are new fields reported now from the "os" property like "load", "memory", etc. They are objects.
+      return acc;
+    }
     return {
       ...acc,
       [`${key}s`]: [{ [key]: value, count: 1 }],

x-pack/plugins/monitoring/common/constants.ts

@@ -28,12 +28,6 @@ export const KIBANA_STATS_TYPE_MONITORING = 'kibana_stats'; // similar to KIBANA
  * @type {string}
  */
 export const KIBANA_SETTINGS_TYPE = 'kibana_settings';
-/**
- * The type name used within the Monitoring index to publish Kibana usage stats.
- * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats
- * @type {string}
- */
-export const KIBANA_USAGE_TYPE = 'kibana';
 
 /*
  * Key for the localStorage service

x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts

@@ -1,86 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { get, snakeCase } from 'lodash';
-import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
-import { KIBANA_USAGE_TYPE, KIBANA_STATS_TYPE_MONITORING } from '../../../common/constants';
-
-const TYPES = [
-  'dashboard',
-  'visualization',
-  'search',
-  'index-pattern',
-  'graph-workspace',
-  'timelion-sheet',
-];
-
-/**
- * Fetches saved object counts by querying the .kibana index
- */
-export function getKibanaUsageCollector(usageCollection: any, kibanaIndex: string) {
-  return usageCollection.makeUsageCollector({
-    type: KIBANA_USAGE_TYPE,
-    isReady: () => true,
-    async fetch(callCluster: CallCluster) {
-      const savedObjectCountSearchParams = {
-        index: kibanaIndex,
-        ignoreUnavailable: true,
-        filterPath: 'aggregations.types.buckets',
-        body: {
-          size: 0,
-          query: {
-            terms: { type: TYPES },
-          },
-          aggs: {
-            types: {
-              terms: { field: 'type', size: TYPES.length },
-            },
-          },
-        },
-      };
-
-      const resp = await callCluster('search', savedObjectCountSearchParams);
-      const buckets: any = get(resp, 'aggregations.types.buckets', []);
-
-      // get the doc_count from each bucket
-      const bucketCounts = buckets.reduce(
-        (acc: any, bucket: any) => ({
-          ...acc,
-          [bucket.key]: bucket.doc_count,
-        }),
-        {}
-      );
-
-      return {
-        index: kibanaIndex,
-        ...TYPES.reduce(
-          (acc, type) => ({
-            // combine the bucketCounts and 0s for types that don't have documents
-            ...acc,
-            [snakeCase(type)]: {
-              total: bucketCounts[type] || 0,
-            },
-          }),
-          {}
-        ),
-      };
-    },
-
-    /*
-     * Format the response data into a model for internal upload
-     * 1. Make this data part of the "kibana_stats" type
-     * 2. Organize the payload in the usage namespace of the data payload (usage.index, etc)
-     */
-    formatForBulkUpload: (result: any) => {
-      return {
-        type: KIBANA_STATS_TYPE_MONITORING,
-        payload: {
-          usage: result,
-        },
-      };
-    },
-  });
-}

x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.ts

@@ -1,46 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { Observable } from 'rxjs';
-import { cloneDeep } from 'lodash';
-import moment from 'moment';
-import { OpsMetrics } from 'kibana/server';
-import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
-import { KIBANA_STATS_TYPE_MONITORING } from '../../../common/constants';
-
-interface MonitoringOpsMetrics extends OpsMetrics {
-  timestamp: string;
-}
-
-/*
- * Initialize a collector for Kibana Ops Stats
- */
-export function getOpsStatsCollector(
-  usageCollection: UsageCollectionSetup,
-  metrics$: Observable<OpsMetrics>
-) {
-  let lastMetrics: MonitoringOpsMetrics | null = null;
-  metrics$.subscribe(_metrics => {
-    const metrics: any = cloneDeep(_metrics);
-    // Ensure we only include the same data that Metricbeat collection would get
-    delete metrics.process.pid;
-    metrics.response_times = {
-      average: metrics.response_times.avg_in_millis,
-      max: metrics.response_times.max_in_millis,
-    };
-    delete metrics.requests.statusCodes;
-    lastMetrics = {
-      ...metrics,
-      timestamp: moment.utc().toISOString(),
-    };
-  });
-
-  return usageCollection.makeStatsCollector({
-    type: KIBANA_STATS_TYPE_MONITORING,
-    isReady: () => !!lastMetrics,
-    fetch: () => lastMetrics,
-  });
-}

x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts

@@ -3,20 +3,14 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { Observable } from 'rxjs';
-import { OpsMetrics } from 'kibana/server';
-import { getKibanaUsageCollector } from './get_kibana_usage_collector';
-import { getOpsStatsCollector } from './get_ops_stats_collector';
+
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { getSettingsCollector } from './get_settings_collector';
 import { MonitoringConfig } from '../../config';
 
 export function registerCollectors(
-  usageCollection: any,
-  config: MonitoringConfig,
-  opsMetrics$: Observable<OpsMetrics>,
-  kibanaIndex: string
+  usageCollection: UsageCollectionSetup,
+  config: MonitoringConfig
 ) {
-  usageCollection.registerCollector(getOpsStatsCollector(usageCollection, opsMetrics$));
-  usageCollection.registerCollector(getKibanaUsageCollector(usageCollection, kibanaIndex));
   usageCollection.registerCollector(getSettingsCollector(usageCollection, config));
 }

x-pack/plugins/monitoring/server/plugin.ts

@@ -177,12 +177,7 @@ export class Plugin {
 
     // Register collector objects for stats to show up in the APIs
     if (plugins.usageCollection) {
-      registerCollectors(
-        plugins.usageCollection,
-        config,
-        core.metrics.getOpsMetrics$(),
-        get(legacyConfig, 'kibana.index')
-      );
+      registerCollectors(plugins.usageCollection, config);
     }
 
     // If collection is enabled, create the bulk uploader