From 028313a8fe9a03ec79182a9bd510245c872c4d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Wed, 8 Apr 2020 09:08:59 +0100 Subject: [PATCH] [Telemetry] Add possibility of registering exclusive collectors for each collection (#62665) * [Telemetry] Add posibility of regitering exclusive collectors for collections * [Telemetry] Filter unwanted fields from the kibana.os telemetry payload * Filter the collectors properly in bulkFetch * Move "kibana" usage collector from Monitoring to OSS Telemetry * Remove exclusivity of the "kibana_settings" collector * Unify "kibana_stats" collector from Monitoring and Legacy * Remove unused legacy constants * Proper type for UsageCollectionSetup in monitoring * Missed one undo * Add unit tests to the migrated collectors Co-authored-by: Elastic Machine --- .../collectors/get_ops_stats_collector.js | 56 -------- src/legacy/server/status/index.js | 2 - src/plugins/telemetry/common/constants.ts | 11 ++ .../telemetry/server/collectors/index.ts | 2 + .../kibana/get_saved_object_counts.test.ts | 61 ++++++++ .../kibana/get_saved_object_counts.ts | 82 +++++++++++ .../server/collectors/kibana/index.test.ts | 76 ++++++++++ .../server/collectors/kibana/index.ts} | 2 +- .../kibana/kibana_usage_collector.ts | 65 +++++++++ .../__snapshots__/index.test.ts.snap | 43 ++++++ .../server/collectors/ops_stats/index.test.ts | 132 ++++++++++++++++++ .../server/collectors/ops_stats/index.ts} | 2 +- .../ops_stats/ops_stats_collector.ts | 71 ++++++++++ src/plugins/telemetry/server/plugin.ts | 19 ++- .../server/telemetry_collection/get_kibana.ts | 4 + x-pack/plugins/monitoring/common/constants.ts | 6 - .../collectors/get_kibana_usage_collector.ts | 86 ------------ .../collectors/get_ops_stats_collector.ts | 46 ------ .../kibana_monitoring/collectors/index.ts | 14 +- x-pack/plugins/monitoring/server/plugin.ts | 7 +- 20 files changed, 568 insertions(+), 219 deletions(-) delete mode 100644 src/legacy/server/status/collectors/get_ops_stats_collector.js create mode 100644 src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.test.ts create mode 100644 src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.ts create mode 100644 src/plugins/telemetry/server/collectors/kibana/index.test.ts rename src/{legacy/server/status/constants.js => plugins/telemetry/server/collectors/kibana/index.ts} (90%) create mode 100644 src/plugins/telemetry/server/collectors/kibana/kibana_usage_collector.ts create mode 100644 src/plugins/telemetry/server/collectors/ops_stats/__snapshots__/index.test.ts.snap create mode 100644 src/plugins/telemetry/server/collectors/ops_stats/index.test.ts rename src/{legacy/server/status/collectors/index.js => plugins/telemetry/server/collectors/ops_stats/index.ts} (91%) create mode 100644 src/plugins/telemetry/server/collectors/ops_stats/ops_stats_collector.ts delete mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts delete mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.ts diff --git a/src/legacy/server/status/collectors/get_ops_stats_collector.js b/src/legacy/server/status/collectors/get_ops_stats_collector.js deleted file mode 100644 index b733e2e721e3..000000000000 --- a/src/legacy/server/status/collectors/get_ops_stats_collector.js +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { KIBANA_STATS_TYPE } from '../constants'; -import { getKibanaInfoForStats } from '../lib'; - -/* - * Initialize a collector for Kibana Ops Stats - * - * NOTE this collector's fetch method returns the latest stats from the - * Hapi/Good/Even-Better ops event listener. Therefore, the stats reset - * every 5 seconds (the default value of the ops.interval configuration - * setting). That makes it geared for providing the latest "real-time" - * stats. In the long-term, fetch should return stats that constantly - * accumulate over the server's uptime for better machine readability. - * Since the data is captured, timestamped and stored, the historical - * data can provide "real-time" stats by calculating a derivative of - * the metrics. - * See PR comment in https://github.com/elastic/kibana/pull/20577/files#r202416647 - */ -export function getOpsStatsCollector(usageCollection, server, kbnServer) { - return usageCollection.makeStatsCollector({ - type: KIBANA_STATS_TYPE, - fetch: () => { - return { - kibana: getKibanaInfoForStats(server, kbnServer), - ...kbnServer.metrics, // latest metrics captured from the ops event listener in src/legacy/server/status/index - }; - }, - isReady: () => true, - ignoreForInternalUploader: true, // Ignore this one from internal uploader. A different stats collector is used there. - }); -} - -export function registerOpsStatsCollector(usageCollection, server, kbnServer) { - if (usageCollection) { - const collector = getOpsStatsCollector(usageCollection, server, kbnServer); - usageCollection.registerCollector(collector); - } -} diff --git a/src/legacy/server/status/index.js b/src/legacy/server/status/index.js index df02b3c45ec2..5bd1efa99eb2 100644 --- a/src/legacy/server/status/index.js +++ b/src/legacy/server/status/index.js @@ -20,7 +20,6 @@ import ServerStatus from './server_status'; import { Metrics } from './lib/metrics'; import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes'; -import { registerOpsStatsCollector } from './collectors'; import Oppsy from 'oppsy'; import { cloneDeep } from 'lodash'; import { getOSInfo } from './lib/get_os_info'; @@ -28,7 +27,6 @@ import { getOSInfo } from './lib/get_os_info'; export function statusMixin(kbnServer, server, config) { kbnServer.status = new ServerStatus(kbnServer.server); const { usageCollection } = server.newPlatform.setup.plugins; - registerOpsStatsCollector(usageCollection, server, kbnServer); const metrics = new Metrics(config, server); diff --git a/src/plugins/telemetry/common/constants.ts b/src/plugins/telemetry/common/constants.ts index babd009143c5..fd3286289652 100644 --- a/src/plugins/telemetry/common/constants.ts +++ b/src/plugins/telemetry/common/constants.ts @@ -80,3 +80,14 @@ export const APPLICATION_USAGE_TYPE = 'application_usage'; * The type name used within the Monitoring index to publish management stats. */ export const KIBANA_STACK_MANAGEMENT_STATS_TYPE = 'stack_management'; + +/** + * The type name used to publish Kibana usage stats. + * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats + */ +export const KIBANA_USAGE_TYPE = 'kibana'; + +/** + * The type name used to publish Kibana usage stats in the formatted as bulk. + */ +export const KIBANA_STATS_TYPE = 'kibana_stats'; diff --git a/src/plugins/telemetry/server/collectors/index.ts b/src/plugins/telemetry/server/collectors/index.ts index 6eeda080bb3a..a874f8dd6202 100644 --- a/src/plugins/telemetry/server/collectors/index.ts +++ b/src/plugins/telemetry/server/collectors/index.ts @@ -22,3 +22,5 @@ export { registerUiMetricUsageCollector } from './ui_metric'; export { registerTelemetryPluginUsageCollector } from './telemetry_plugin'; export { registerManagementUsageCollector } from './management'; export { registerApplicationUsageCollector } from './application_usage'; +export { registerKibanaUsageCollector } from './kibana'; +export { registerOpsStatsCollector } from './ops_stats'; diff --git a/src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.test.ts b/src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.test.ts new file mode 100644 index 000000000000..a7681e176642 --- /dev/null +++ b/src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.test.ts @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getSavedObjectsCounts } from './get_saved_object_counts'; + +describe('getSavedObjectsCounts', () => { + test('Get all the saved objects equal to 0 because no results were found', async () => { + const callCluster = jest.fn(() => ({})); + + const results = await getSavedObjectsCounts(callCluster as any, '.kibana'); + expect(results).toStrictEqual({ + dashboard: { total: 0 }, + visualization: { total: 0 }, + search: { total: 0 }, + index_pattern: { total: 0 }, + graph_workspace: { total: 0 }, + timelion_sheet: { total: 0 }, + }); + }); + + test('Merge the zeros with the results', async () => { + const callCluster = jest.fn(() => ({ + aggregations: { + types: { + buckets: [ + { key: 'dashboard', doc_count: 1 }, + { key: 'timelion-sheet', doc_count: 2 }, + { key: 'index-pattern', value: 2 }, // Malformed on purpose + { key: 'graph_workspace', doc_count: 3 }, // already snake_cased + ], + }, + }, + })); + + const results = await getSavedObjectsCounts(callCluster as any, '.kibana'); + expect(results).toStrictEqual({ + dashboard: { total: 1 }, + visualization: { total: 0 }, + search: { total: 0 }, + index_pattern: { total: 0 }, + graph_workspace: { total: 3 }, + timelion_sheet: { total: 2 }, + }); + }); +}); diff --git a/src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.ts b/src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.ts new file mode 100644 index 000000000000..804c8b0ed202 --- /dev/null +++ b/src/plugins/telemetry/server/collectors/kibana/get_saved_object_counts.ts @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Moved from /x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts + * + * The PR https://github.com/elastic/kibana/pull/62665 proved what the issue https://github.com/elastic/kibana/issues/58249 + * was claiming: the structure and payload for common telemetry bits differs between Monitoring and OSS/X-Pack collections. + * + * Unifying this logic from Monitoring that makes sense to have in OSS here and we will import it on the monitoring side to reuse it. + */ + +import { snakeCase } from 'lodash'; +import { APICaller } from 'kibana/server'; + +const TYPES = [ + 'dashboard', + 'visualization', + 'search', + 'index-pattern', + 'graph-workspace', + 'timelion-sheet', +]; + +export interface KibanaSavedObjectCounts { + [pluginName: string]: { + total: number; + }; +} + +export async function getSavedObjectsCounts( + callCluster: APICaller, + kibanaIndex: string // Typically '.kibana'. We might need a way to obtain it from the SavedObjects client (or the SavedObjects client to provide a way to run aggregations?) +): Promise { + const savedObjectCountSearchParams = { + index: kibanaIndex, + ignoreUnavailable: true, + filterPath: 'aggregations.types.buckets', + body: { + size: 0, + query: { + terms: { type: TYPES }, + }, + aggs: { + types: { + terms: { field: 'type', size: TYPES.length }, + }, + }, + }, + }; + const resp = await callCluster('search', savedObjectCountSearchParams); + const buckets: Array<{ key: string; doc_count: number }> = + resp.aggregations?.types?.buckets || []; + + // Initialise the object with all zeros for all the types + const allZeros: KibanaSavedObjectCounts = TYPES.reduce( + (acc, type) => ({ ...acc, [snakeCase(type)]: { total: 0 } }), + {} + ); + + // Add the doc_count from each bucket + return buckets.reduce( + (acc, { key, doc_count: total }) => (total ? { ...acc, [snakeCase(key)]: { total } } : acc), + allZeros + ); +} diff --git a/src/plugins/telemetry/server/collectors/kibana/index.test.ts b/src/plugins/telemetry/server/collectors/kibana/index.test.ts new file mode 100644 index 000000000000..91ede686ded3 --- /dev/null +++ b/src/plugins/telemetry/server/collectors/kibana/index.test.ts @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { UsageCollectionSetup } from '../../../../../plugins/usage_collection/server'; +import { pluginInitializerContextConfigMock } from '../../../../../core/server/mocks'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { CollectorOptions } from '../../../../../plugins/usage_collection/server/collector/collector'; + +import { registerKibanaUsageCollector } from './'; + +describe('telemetry_kibana', () => { + let collector: CollectorOptions; + + const usageCollectionMock: jest.Mocked = { + makeUsageCollector: jest.fn().mockImplementation(config => (collector = config)), + registerCollector: jest.fn(), + } as any; + + const legacyConfig$ = pluginInitializerContextConfigMock({}).legacy.globalConfig$; + const callCluster = jest.fn().mockImplementation(() => ({})); + + beforeAll(() => registerKibanaUsageCollector(usageCollectionMock, legacyConfig$)); + afterAll(() => jest.clearAllTimers()); + + test('registered collector is set', () => { + expect(collector).not.toBeUndefined(); + expect(collector.type).toBe('kibana'); + }); + + test('fetch', async () => { + expect(await collector.fetch(callCluster)).toStrictEqual({ + index: '.kibana-tests', + dashboard: { total: 0 }, + visualization: { total: 0 }, + search: { total: 0 }, + index_pattern: { total: 0 }, + graph_workspace: { total: 0 }, + timelion_sheet: { total: 0 }, + }); + }); + + test('formatForBulkUpload', async () => { + const resultFromFetch = { + index: '.kibana-tests', + dashboard: { total: 0 }, + visualization: { total: 0 }, + search: { total: 0 }, + index_pattern: { total: 0 }, + graph_workspace: { total: 0 }, + timelion_sheet: { total: 0 }, + }; + + expect(collector.formatForBulkUpload!(resultFromFetch)).toStrictEqual({ + type: 'kibana_stats', + payload: { + usage: resultFromFetch, + }, + }); + }); +}); diff --git a/src/legacy/server/status/constants.js b/src/plugins/telemetry/server/collectors/kibana/index.ts similarity index 90% rename from src/legacy/server/status/constants.js rename to src/plugins/telemetry/server/collectors/kibana/index.ts index 3bb23749bae8..695d972346b8 100644 --- a/src/legacy/server/status/constants.js +++ b/src/plugins/telemetry/server/collectors/kibana/index.ts @@ -17,4 +17,4 @@ * under the License. */ -export const KIBANA_STATS_TYPE = 'oss_kibana_stats'; // kibana stats per 5s intervals +export { registerKibanaUsageCollector } from './kibana_usage_collector'; diff --git a/src/plugins/telemetry/server/collectors/kibana/kibana_usage_collector.ts b/src/plugins/telemetry/server/collectors/kibana/kibana_usage_collector.ts new file mode 100644 index 000000000000..ccf6f7b1033c --- /dev/null +++ b/src/plugins/telemetry/server/collectors/kibana/kibana_usage_collector.ts @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Observable } from 'rxjs'; +import { take } from 'rxjs/operators'; +import { SharedGlobalConfig } from 'kibana/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { KIBANA_STATS_TYPE, KIBANA_USAGE_TYPE } from '../../../common/constants'; +import { getSavedObjectsCounts } from './get_saved_object_counts'; + +export function getKibanaUsageCollector( + usageCollection: UsageCollectionSetup, + legacyConfig$: Observable +) { + return usageCollection.makeUsageCollector({ + type: KIBANA_USAGE_TYPE, + isReady: () => true, + async fetch(callCluster) { + const { + kibana: { index }, + } = await legacyConfig$.pipe(take(1)).toPromise(); + return { + index, + ...(await getSavedObjectsCounts(callCluster, index)), + }; + }, + + /* + * Format the response data into a model for internal upload + * 1. Make this data part of the "kibana_stats" type + * 2. Organize the payload in the usage namespace of the data payload (usage.index, etc) + */ + formatForBulkUpload: result => { + return { + type: KIBANA_STATS_TYPE, + payload: { + usage: result, + }, + }; + }, + }); +} + +export function registerKibanaUsageCollector( + usageCollection: UsageCollectionSetup, + legacyConfig$: Observable +) { + usageCollection.registerCollector(getKibanaUsageCollector(usageCollection, legacyConfig$)); +} diff --git a/src/plugins/telemetry/server/collectors/ops_stats/__snapshots__/index.test.ts.snap b/src/plugins/telemetry/server/collectors/ops_stats/__snapshots__/index.test.ts.snap new file mode 100644 index 000000000000..678237ffb6ea --- /dev/null +++ b/src/plugins/telemetry/server/collectors/ops_stats/__snapshots__/index.test.ts.snap @@ -0,0 +1,43 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`telemetry_ops_stats should return something when there is a metric 1`] = ` +Object { + "concurrent_connections": 20, + "os": Object { + "load": Object { + "15m": 3, + "1m": 0.5, + "5m": 1, + }, + "memory": Object { + "free_in_bytes": 10, + "total_in_bytes": 10, + "used_in_bytes": 10, + }, + "platform": "darwin", + "platformRelease": "test", + "uptime_in_millis": 1000, + }, + "process": Object { + "event_loop_delay": 10, + "memory": Object { + "heap": Object { + "size_limit": 0, + "total_in_bytes": 0, + "used_in_bytes": 0, + }, + "resident_set_size_in_bytes": 0, + }, + "uptime_in_millis": 1000, + }, + "requests": Object { + "disconnects": 10, + "total": 100, + }, + "response_times": Object { + "average": 100, + "max": 200, + }, + "timestamp": Any, +} +`; diff --git a/src/plugins/telemetry/server/collectors/ops_stats/index.test.ts b/src/plugins/telemetry/server/collectors/ops_stats/index.test.ts new file mode 100644 index 000000000000..92e0e40776eb --- /dev/null +++ b/src/plugins/telemetry/server/collectors/ops_stats/index.test.ts @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Subject } from 'rxjs'; +import { UsageCollectionSetup } from '../../../../../plugins/usage_collection/server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { CollectorOptions } from '../../../../../plugins/usage_collection/server/collector/collector'; + +import { registerOpsStatsCollector } from './'; +import { OpsMetrics } from '../../../../../core/server'; + +describe('telemetry_ops_stats', () => { + let collector: CollectorOptions; + + const usageCollectionMock: jest.Mocked = { + makeStatsCollector: jest.fn().mockImplementation(config => (collector = config)), + registerCollector: jest.fn(), + } as any; + + const metrics$ = new Subject(); + const callCluster = jest.fn(); + + const metric: OpsMetrics = { + process: { + memory: { + heap: { + total_in_bytes: 0, + used_in_bytes: 0, + size_limit: 0, + }, + resident_set_size_in_bytes: 0, + }, + event_loop_delay: 10, + pid: 10, + uptime_in_millis: 1000, + }, + os: { + platform: 'darwin', + platformRelease: 'test', + load: { + '1m': 0.5, + '5m': 1, + '15m': 3, + }, + memory: { + total_in_bytes: 10, + free_in_bytes: 10, + used_in_bytes: 10, + }, + uptime_in_millis: 1000, + }, + response_times: { avg_in_millis: 100, max_in_millis: 200 }, + requests: { + disconnects: 10, + total: 100, + statusCodes: { 200: 100 }, + }, + concurrent_connections: 20, + }; + + beforeAll(() => registerOpsStatsCollector(usageCollectionMock, metrics$)); + afterAll(() => jest.clearAllTimers()); + + test('registered collector is set', () => { + expect(collector).not.toBeUndefined(); + expect(collector.type).toBe('kibana_stats'); + }); + + test('isReady should return false because no metrics have been provided yet', () => { + expect(collector.isReady()).toBe(false); + }); + + test('should return something when there is a metric', async () => { + metrics$.next(metric); + expect(collector.isReady()).toBe(true); + expect(await collector.fetch(callCluster)).toMatchSnapshot({ + concurrent_connections: 20, + os: { + load: { + '15m': 3, + '1m': 0.5, + '5m': 1, + }, + memory: { + free_in_bytes: 10, + total_in_bytes: 10, + used_in_bytes: 10, + }, + platform: 'darwin', + platformRelease: 'test', + uptime_in_millis: 1000, + }, + process: { + event_loop_delay: 10, + memory: { + heap: { + size_limit: 0, + total_in_bytes: 0, + used_in_bytes: 0, + }, + resident_set_size_in_bytes: 0, + }, + uptime_in_millis: 1000, + }, + requests: { + disconnects: 10, + total: 100, + }, + response_times: { + average: 100, + max: 200, + }, + timestamp: expect.any(String), + }); + }); +}); diff --git a/src/legacy/server/status/collectors/index.js b/src/plugins/telemetry/server/collectors/ops_stats/index.ts similarity index 91% rename from src/legacy/server/status/collectors/index.js rename to src/plugins/telemetry/server/collectors/ops_stats/index.ts index 92d9e601bbb3..443a25749d20 100644 --- a/src/legacy/server/status/collectors/index.js +++ b/src/plugins/telemetry/server/collectors/ops_stats/index.ts @@ -17,4 +17,4 @@ * under the License. */ -export { registerOpsStatsCollector } from './get_ops_stats_collector'; +export { registerOpsStatsCollector } from './ops_stats_collector'; diff --git a/src/plugins/telemetry/server/collectors/ops_stats/ops_stats_collector.ts b/src/plugins/telemetry/server/collectors/ops_stats/ops_stats_collector.ts new file mode 100644 index 000000000000..4967e20006dd --- /dev/null +++ b/src/plugins/telemetry/server/collectors/ops_stats/ops_stats_collector.ts @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Observable } from 'rxjs'; +import { cloneDeep } from 'lodash'; +import moment from 'moment'; +import { OpsMetrics } from 'kibana/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { KIBANA_STATS_TYPE } from '../../../common/constants'; + +interface OpsStatsMetrics extends Omit { + timestamp: string; + response_times: { + average: number; + max: number; + }; +} + +/** + * Initialize a collector for Kibana Ops Stats + */ +export function getOpsStatsCollector( + usageCollection: UsageCollectionSetup, + metrics$: Observable +) { + let lastMetrics: OpsStatsMetrics | null = null; + metrics$.subscribe(_metrics => { + const metrics = cloneDeep(_metrics); + // Ensure we only include the same data that Metricbeat collection would get + delete metrics.process.pid; + const responseTimes = { + average: metrics.response_times.avg_in_millis, + max: metrics.response_times.max_in_millis, + }; + delete metrics.requests.statusCodes; + lastMetrics = { + ...metrics, + response_times: responseTimes, + timestamp: moment.utc().toISOString(), + }; + }); + + return usageCollection.makeStatsCollector({ + type: KIBANA_STATS_TYPE, + isReady: () => !!lastMetrics, + fetch: () => lastMetrics, + }); +} + +export function registerOpsStatsCollector( + usageCollection: UsageCollectionSetup, + metrics$: Observable +) { + usageCollection.registerCollector(getOpsStatsCollector(usageCollection, metrics$)); +} diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts index 77036b4ea7dd..1df6a665e4d7 100644 --- a/src/plugins/telemetry/server/plugin.ts +++ b/src/plugins/telemetry/server/plugin.ts @@ -32,6 +32,8 @@ import { SavedObjectsClient, Plugin, Logger, + SharedGlobalConfig, + MetricsServiceSetup, } from '../../../core/server'; import { registerRoutes } from './routes'; import { registerCollection } from './telemetry_collection'; @@ -41,6 +43,8 @@ import { registerTelemetryPluginUsageCollector, registerManagementUsageCollector, registerApplicationUsageCollector, + registerKibanaUsageCollector, + registerOpsStatsCollector, } from './collectors'; import { TelemetryConfigType } from './config'; import { FetcherTask } from './fetcher'; @@ -61,6 +65,7 @@ export class TelemetryPlugin implements Plugin { private readonly logger: Logger; private readonly currentKibanaVersion: string; private readonly config$: Observable; + private readonly legacyConfig$: Observable; private readonly isDev: boolean; private readonly fetcherTask: FetcherTask; private savedObjectsClient?: ISavedObjectsRepository; @@ -71,6 +76,7 @@ export class TelemetryPlugin implements Plugin { this.isDev = initializerContext.env.mode.dev; this.currentKibanaVersion = initializerContext.env.packageInfo.version; this.config$ = initializerContext.config.create(); + this.legacyConfig$ = initializerContext.config.legacy.globalConfig$; this.fetcherTask = new FetcherTask({ ...initializerContext, logger: this.logger, @@ -78,15 +84,15 @@ export class TelemetryPlugin implements Plugin { } public async setup( - core: CoreSetup, + { elasticsearch, http, savedObjects, metrics }: CoreSetup, { usageCollection, telemetryCollectionManager }: TelemetryPluginsSetup ) { const currentKibanaVersion = this.currentKibanaVersion; const config$ = this.config$; const isDev = this.isDev; - registerCollection(telemetryCollectionManager, core.elasticsearch.dataClient); - const router = core.http.createRouter(); + registerCollection(telemetryCollectionManager, elasticsearch.dataClient); + const router = http.createRouter(); registerRoutes({ config$, @@ -96,8 +102,8 @@ export class TelemetryPlugin implements Plugin { telemetryCollectionManager, }); - this.registerMappings(opts => core.savedObjects.registerType(opts)); - this.registerUsageCollectors(usageCollection, opts => core.savedObjects.registerType(opts)); + this.registerMappings(opts => savedObjects.registerType(opts)); + this.registerUsageCollectors(usageCollection, metrics, opts => savedObjects.registerType(opts)); } public async start(core: CoreStart, { telemetryCollectionManager }: TelemetryPluginsStart) { @@ -153,11 +159,14 @@ export class TelemetryPlugin implements Plugin { private registerUsageCollectors( usageCollection: UsageCollectionSetup, + metrics: MetricsServiceSetup, registerType: SavedObjectsRegisterType ) { const getSavedObjectsClient = () => this.savedObjectsClient; const getUiSettingsClient = () => this.uiSettingsClient; + registerOpsStatsCollector(usageCollection, metrics.getOpsMetrics$()); + registerKibanaUsageCollector(usageCollection, this.legacyConfig$); registerTelemetryPluginUsageCollector(usageCollection, { currentKibanaVersion: this.currentKibanaVersion, config$: this.config$, diff --git a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts index 86c6731e11d3..a17f1b8232a2 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_kibana.ts @@ -55,6 +55,10 @@ export function handleKibanaStats( ...kibanaStats.os, }; const formattedOsStats = Object.entries(os).reduce((acc, [key, value]) => { + if (typeof value !== 'string') { + // There are new fields reported now from the "os" property like "load", "memory", etc. They are objects. + return acc; + } return { ...acc, [`${key}s`]: [{ [key]: value, count: 1 }], diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts index 3a4c7b71dcd0..edd6142455df 100644 --- a/x-pack/plugins/monitoring/common/constants.ts +++ b/x-pack/plugins/monitoring/common/constants.ts @@ -28,12 +28,6 @@ export const KIBANA_STATS_TYPE_MONITORING = 'kibana_stats'; // similar to KIBANA * @type {string} */ export const KIBANA_SETTINGS_TYPE = 'kibana_settings'; -/** - * The type name used within the Monitoring index to publish Kibana usage stats. - * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats - * @type {string} - */ -export const KIBANA_USAGE_TYPE = 'kibana'; /* * Key for the localStorage service diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts deleted file mode 100644 index 2c40ac56e19e..000000000000 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { get, snakeCase } from 'lodash'; -import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; -import { KIBANA_USAGE_TYPE, KIBANA_STATS_TYPE_MONITORING } from '../../../common/constants'; - -const TYPES = [ - 'dashboard', - 'visualization', - 'search', - 'index-pattern', - 'graph-workspace', - 'timelion-sheet', -]; - -/** - * Fetches saved object counts by querying the .kibana index - */ -export function getKibanaUsageCollector(usageCollection: any, kibanaIndex: string) { - return usageCollection.makeUsageCollector({ - type: KIBANA_USAGE_TYPE, - isReady: () => true, - async fetch(callCluster: CallCluster) { - const savedObjectCountSearchParams = { - index: kibanaIndex, - ignoreUnavailable: true, - filterPath: 'aggregations.types.buckets', - body: { - size: 0, - query: { - terms: { type: TYPES }, - }, - aggs: { - types: { - terms: { field: 'type', size: TYPES.length }, - }, - }, - }, - }; - - const resp = await callCluster('search', savedObjectCountSearchParams); - const buckets: any = get(resp, 'aggregations.types.buckets', []); - - // get the doc_count from each bucket - const bucketCounts = buckets.reduce( - (acc: any, bucket: any) => ({ - ...acc, - [bucket.key]: bucket.doc_count, - }), - {} - ); - - return { - index: kibanaIndex, - ...TYPES.reduce( - (acc, type) => ({ - // combine the bucketCounts and 0s for types that don't have documents - ...acc, - [snakeCase(type)]: { - total: bucketCounts[type] || 0, - }, - }), - {} - ), - }; - }, - - /* - * Format the response data into a model for internal upload - * 1. Make this data part of the "kibana_stats" type - * 2. Organize the payload in the usage namespace of the data payload (usage.index, etc) - */ - formatForBulkUpload: (result: any) => { - return { - type: KIBANA_STATS_TYPE_MONITORING, - payload: { - usage: result, - }, - }; - }, - }); -} diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.ts deleted file mode 100644 index 85357f786ddc..000000000000 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Observable } from 'rxjs'; -import { cloneDeep } from 'lodash'; -import moment from 'moment'; -import { OpsMetrics } from 'kibana/server'; -import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { KIBANA_STATS_TYPE_MONITORING } from '../../../common/constants'; - -interface MonitoringOpsMetrics extends OpsMetrics { - timestamp: string; -} - -/* - * Initialize a collector for Kibana Ops Stats - */ -export function getOpsStatsCollector( - usageCollection: UsageCollectionSetup, - metrics$: Observable -) { - let lastMetrics: MonitoringOpsMetrics | null = null; - metrics$.subscribe(_metrics => { - const metrics: any = cloneDeep(_metrics); - // Ensure we only include the same data that Metricbeat collection would get - delete metrics.process.pid; - metrics.response_times = { - average: metrics.response_times.avg_in_millis, - max: metrics.response_times.max_in_millis, - }; - delete metrics.requests.statusCodes; - lastMetrics = { - ...metrics, - timestamp: moment.utc().toISOString(), - }; - }); - - return usageCollection.makeStatsCollector({ - type: KIBANA_STATS_TYPE_MONITORING, - isReady: () => !!lastMetrics, - fetch: () => lastMetrics, - }); -} diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts index e41b1512f1b2..dcd35b0d323e 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/index.ts @@ -3,20 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Observable } from 'rxjs'; -import { OpsMetrics } from 'kibana/server'; -import { getKibanaUsageCollector } from './get_kibana_usage_collector'; -import { getOpsStatsCollector } from './get_ops_stats_collector'; + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { getSettingsCollector } from './get_settings_collector'; import { MonitoringConfig } from '../../config'; export function registerCollectors( - usageCollection: any, - config: MonitoringConfig, - opsMetrics$: Observable, - kibanaIndex: string + usageCollection: UsageCollectionSetup, + config: MonitoringConfig ) { - usageCollection.registerCollector(getOpsStatsCollector(usageCollection, opsMetrics$)); - usageCollection.registerCollector(getKibanaUsageCollector(usageCollection, kibanaIndex)); usageCollection.registerCollector(getSettingsCollector(usageCollection, config)); } diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 784226dca66f..920accd17ec8 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -177,12 +177,7 @@ export class Plugin { // Register collector objects for stats to show up in the APIs if (plugins.usageCollection) { - registerCollectors( - plugins.usageCollection, - config, - core.metrics.getOpsMetrics$(), - get(legacyConfig, 'kibana.index') - ); + registerCollectors(plugins.usageCollection, config); } // If collection is enabled, create the bulk uploader