From 630d4bbfcf7aedee1369a71a8a6db18218ef3682 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 14 Apr 2021 10:23:45 -0400 Subject: [PATCH] [Usage collection] Usage counters (#96696) (#97103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alejandro Fernández Haro Co-authored-by: Ahmad Bamieh Co-authored-by: Alejandro Fernández Haro --- ...rver.indexpatternsserviceprovider.start.md | 4 +- src/plugins/data/server/server.api.md | 1 + .../server/__snapshots__/index.test.ts.snap | 21 -- ...emetry_application_usage_collector.test.ts | 2 +- .../cloud/cloud_provider_collector.test.ts | 14 +- .../server/collectors/core/index.test.ts | 2 +- .../server/collectors/index.ts | 4 + .../server/collectors/kibana/index.test.ts | 2 +- .../telemetry_management_collector.test.ts | 2 +- .../server/collectors/ops_stats/index.test.ts | 2 +- .../__fixtures__/ui_counter_saved_objects.ts | 51 ++++ .../usage_counter_saved_objects.ts | 104 +++++++ .../register_ui_counters_collector.test.ts | 266 +++++++++++++---- .../register_ui_counters_collector.ts | 156 ++++++++-- .../ui_counters/rollups/register_rollups.ts | 12 +- .../ui_counters/rollups/rollups.test.ts | 38 ++- .../collectors/ui_counters/rollups/rollups.ts | 16 + .../server/collectors/ui_metric/index.test.ts | 2 +- .../usage_counter_saved_objects.ts | 104 +++++++ .../server/collectors/usage_counters/index.ts | 10 + .../register_usage_counters_collector.test.ts | 55 ++++ .../register_usage_counters_collector.ts | 116 ++++++++ .../usage_counters/rollups/constants.ts | 22 ++ .../usage_counters/rollups/index.ts | 9 + .../rollups/register_rollups.ts | 21 ++ .../usage_counters/rollups/rollups.test.ts | 170 +++++++++++ .../usage_counters/rollups/rollups.ts | 73 +++++ .../server/{index.test.mocks.ts => mocks.ts} | 0 .../server/{index.test.ts => plugin.test.ts} | 66 ++++- .../kibana_usage_collection/server/plugin.ts | 20 +- src/plugins/telemetry/schema/oss_plugins.json | 47 +++ src/plugins/usage_collection/README.mdx | 95 ++++++ .../usage_collection/common/ui_counters.ts | 23 ++ .../server/collector/collector_set.ts | 32 +- .../server/collector/index.ts | 1 - src/plugins/usage_collection/server/config.ts | 5 + src/plugins/usage_collection/server/index.ts | 13 + src/plugins/usage_collection/server/mocks.ts | 65 ++++- src/plugins/usage_collection/server/plugin.ts | 85 +++++- .../server/report/store_report.test.ts | 84 ++++-- .../server/report/store_report.ts | 18 +- .../usage_collection/server/routes/index.ts | 6 +- .../server/routes/ui_counters.ts | 6 +- .../server/usage_collection.mock.ts | 58 ---- .../server/usage_counters/index.ts | 15 + .../usage_counters/saved_objects.test.ts | 71 +++++ .../server/usage_counters/saved_objects.ts | 86 ++++++ .../usage_counters/usage_counter.test.ts | 38 +++ .../server/usage_counters/usage_counter.ts | 48 +++ .../usage_counters_service.mock.ts | 40 +++ .../usage_counters_service.test.ts | 241 +++++++++++++++ .../usage_counters/usage_counters_service.ts | 185 ++++++++++++ .../register_usage_collector.test.ts | 6 +- .../register_timeseries_collector.test.ts | 2 +- .../register_vega_collector.test.ts | 2 +- .../register_visualizations_collector.test.ts | 2 +- .../telemetry/__fixtures__/ui_counters.ts | 8 + .../telemetry/__fixtures__/usage_counters.ts | 36 +++ .../apis/telemetry/telemetry_local.ts | 15 + .../apis/ui_counters/ui_counters.ts | 50 ++-- test/api_integration/config.js | 2 + .../saved_objects/ui_counters/data.json | 111 +++++++ .../saved_objects/ui_counters/data.json.gz | Bin 236 -> 0 bytes .../saved_objects/ui_counters/mappings.json | 9 + .../saved_objects/usage_counters/data.json | 89 ++++++ .../usage_counters/mappings.json | 276 ++++++++++++++++++ test/plugin_functional/config.ts | 3 + .../plugins/usage_collection/kibana.json | 9 + .../plugins/usage_collection/package.json | 14 + .../plugins/usage_collection/server/index.ts | 10 + .../plugins/usage_collection/server/plugin.ts | 43 +++ .../plugins/usage_collection/server/routes.ts | 24 ++ .../plugins/usage_collection/tsconfig.json | 18 ++ .../test_suites/usage_collection/index.ts | 15 + .../usage_collection/usage_counters.ts | 67 +++++ 75 files changed, 3120 insertions(+), 318 deletions(-) delete mode 100644 src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap create mode 100644 src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts rename src/plugins/kibana_usage_collection/server/{index.test.mocks.ts => mocks.ts} (100%) rename src/plugins/kibana_usage_collection/server/{index.test.ts => plugin.test.ts} (59%) create mode 100644 src/plugins/usage_collection/common/ui_counters.ts delete mode 100644 src/plugins/usage_collection/server/usage_collection.mock.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/index.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/saved_objects.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counter.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts create mode 100644 test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts create mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json delete mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz create mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json create mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json create mode 100644 test/plugin_functional/plugins/usage_collection/kibana.json create mode 100644 test/plugin_functional/plugins/usage_collection/package.json create mode 100644 test/plugin_functional/plugins/usage_collection/server/index.ts create mode 100644 test/plugin_functional/plugins/usage_collection/server/plugin.ts create mode 100644 test/plugin_functional/plugins/usage_collection/server/routes.ts create mode 100644 test/plugin_functional/plugins/usage_collection/tsconfig.json create mode 100644 test/plugin_functional/test_suites/usage_collection/index.ts create mode 100644 test/plugin_functional/test_suites/usage_collection/usage_counters.ts diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md index 88079bb2fa3c..118b0104fbee 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md @@ -8,7 +8,7 @@ ```typescript start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; }; ``` @@ -22,6 +22,6 @@ start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): Returns: `{ - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; }` diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index f0a370639cbd..6432109beebd 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -56,6 +56,7 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestHandlerContext } from 'src/core/server'; +import * as Rx from 'rxjs'; import { SavedObject } from 'kibana/server'; import { SavedObject as SavedObject_2 } from 'src/core/server'; import { SavedObjectsClientContract } from 'src/core/server'; diff --git a/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap b/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap deleted file mode 100644 index 939e90d2f258..000000000000 --- a/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap +++ /dev/null @@ -1,21 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`kibana_usage_collection Runs the setup method without issues 1`] = `true`; - -exports[`kibana_usage_collection Runs the setup method without issues 2`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 3`] = `true`; - -exports[`kibana_usage_collection Runs the setup method without issues 4`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 5`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 6`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 7`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 8`] = `true`; - -exports[`kibana_usage_collection Runs the setup method without issues 9`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 10`] = `true`; diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts index f1b21af5506e..da4e1b101914 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts @@ -10,7 +10,7 @@ import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../co import { Collector, createUsageCollectionSetupMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { diff --git a/src/plugins/kibana_usage_collection/server/collectors/cloud/cloud_provider_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/cloud/cloud_provider_collector.test.ts index 1f7617a0e69c..a2f08ddb465c 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/cloud/cloud_provider_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/cloud/cloud_provider_collector.test.ts @@ -12,25 +12,23 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerCloudProviderUsageCollector } from './cloud_provider_collector'; describe('registerCloudProviderUsageCollector', () => { let collector: Collector; const logger = loggingSystemMock.createLogger(); - - const usageCollectionMock = createUsageCollectionSetupMock(); - usageCollectionMock.makeUsageCollector.mockImplementation((config) => { - collector = new Collector(logger, config); - return createUsageCollectionSetupMock().makeUsageCollector(config); - }); - const mockedFetchContext = createCollectorFetchContextMock(); beforeEach(() => { cloudDetailsMock.mockClear(); detectCloudServiceMock.mockClear(); + const usageCollectionMock = createUsageCollectionSetupMock(); + usageCollectionMock.makeUsageCollector.mockImplementation((config) => { + collector = new Collector(logger, config); + return createUsageCollectionSetupMock().makeUsageCollector(config); + }); registerCloudProviderUsageCollector(usageCollectionMock); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts index 4409442f4c70..cbc38129fddd 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts @@ -9,7 +9,7 @@ import { Collector, createUsageCollectionSetupMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { registerCoreUsageCollector } from '.'; import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/index.ts b/src/plugins/kibana_usage_collection/server/collectors/index.ts index 89e1e6e79482..522860e58918 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/index.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/index.ts @@ -20,3 +20,7 @@ export { registerUiCounterSavedObjectType, registerUiCountersRollups, } from './ui_counters'; +export { + registerUsageCountersRollups, + registerUsageCountersUsageCollector, +} from './usage_counters'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts index 1d0329cb01d6..e1afbfbcecc4 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts @@ -15,7 +15,7 @@ import { Collector, createCollectorFetchContextMock, createUsageCollectionSetupMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerKibanaUsageCollector } from './'; const logger = loggingSystemMock.createLogger(); diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts index a8ac77822608..cb0b1c045397 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts @@ -11,7 +11,7 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerManagementUsageCollector, diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts index a90197e7a25a..dfd6a93b7ea1 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts @@ -11,7 +11,7 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerOpsStatsCollector } from './'; import { OpsMetrics } from '../../../../../core/server'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts new file mode 100644 index 000000000000..ebc958c7be8c --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UICounterSavedObject } from '../ui_counter_saved_object_type'; +export const rawUiCounters: UICounterSavedObject[] = [ + { + type: 'ui-counter', + id: 'Kibana_home:23102020:click:different_type', + attributes: { + count: 1, + }, + references: [], + updated_at: '2020-11-24T11:27:57.067Z', + version: 'WzI5NDRd', + }, + { + type: 'ui-counter', + id: 'Kibana_home:25102020:loaded:intersecting_event', + attributes: { + count: 1, + }, + references: [], + updated_at: '2020-10-25T11:27:57.067Z', + version: 'WzI5NDRd', + }, + { + type: 'ui-counter', + id: 'Kibana_home:23102020:loaded:intersecting_event', + attributes: { + count: 3, + }, + references: [], + updated_at: '2020-10-23T11:27:57.067Z', + version: 'WzI5NDRd', + }, + { + type: 'ui-counter', + id: 'Kibana_home:24112020:click:only_reported_in_ui_counters', + attributes: { + count: 1, + }, + references: [], + updated_at: '2020-11-24T11:27:57.067Z', + version: 'WzI5NDRd', + }, +]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts new file mode 100644 index 000000000000..6b70a8c97e65 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UsageCountersSavedObject } from '../../../../../usage_collection/server'; + +export const rawUsageCounters: UsageCountersSavedObject[] = [ + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event', + attributes: { + count: 1, + counterName: 'myApp:my_event', + counterType: 'count', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:17:57.693Z', + }, + { + type: 'usage-counters', + id: 'uiCounter:23102020:loaded:Kibana_home:intersecting_event', + attributes: { + count: 60, + counterName: 'Kibana_home:intersecting_event', + counterType: 'loaded', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2020-10-23T11:27:57.067Z', + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_4457914848544', + attributes: { + count: 0, + counterName: 'myApp:my_event_4457914848544', + counterType: 'count', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_malformed', + attributes: { + // @ts-expect-error + count: 'malformed', + counterName: 'myApp:my_event_malformed', + counterType: 'count', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId:09042021:count:some_event_name', + attributes: { + count: 4, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_4457914848544_2', + attributes: { + count: 8, + counterName: 'myApp:my_event_4457914848544_2', + counterType: 'count', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.031Z', + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:only_reported_in_usage_counters', + attributes: { + count: 1, + counterName: 'myApp:only_reported_in_usage_counters', + counterType: 'count', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.031Z', + }, +]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts index 7e84bc852c9b..122e637d2b20 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts @@ -6,70 +6,208 @@ * Side Public License, v 1. */ -import { transformRawCounter } from './register_ui_counters_collector'; -import { UICounterSavedObject } from './ui_counter_saved_object_type'; +import { + transformRawUiCounterObject, + transformRawUsageCounterObject, + createFetchUiCounters, +} from './register_ui_counters_collector'; +import { BehaviorSubject } from 'rxjs'; +import { rawUiCounters } from './__fixtures__/ui_counter_saved_objects'; +import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; +import { savedObjectsClientMock } from '../../../../../core/server/mocks'; +import { UI_COUNTER_SAVED_OBJECT_TYPE } from './ui_counter_saved_object_type'; +import { USAGE_COUNTERS_SAVED_OBJECT_TYPE } from '../../../../usage_collection/server'; -describe('transformRawCounter', () => { - const mockRawUiCounters = [ - { - type: 'ui-counter', - id: 'Kibana_home:24112020:click:ingest_data_card_home_tutorial_directory', - attributes: { - count: 3, - }, - references: [], - updated_at: '2020-11-24T11:27:57.067Z', - version: 'WzI5LDFd', - }, - { - type: 'ui-counter', - id: 'Kibana_home:24112020:click:home_tutorial_directory', - attributes: { - count: 1, - }, - references: [], - updated_at: '2020-11-24T11:27:57.067Z', - version: 'WzI5NDRd', - }, - { - type: 'ui-counter', - id: 'Kibana_home:24112020:loaded:home_tutorial_directory', - attributes: { - count: 3, - }, - references: [], - updated_at: '2020-10-23T11:27:57.067Z', - version: 'WzI5NDRd', - }, - ] as UICounterSavedObject[]; - - it('transforms saved object raw entries', () => { - const result = mockRawUiCounters.map(transformRawCounter); - expect(result).toEqual([ - { - appName: 'Kibana_home', - eventName: 'ingest_data_card_home_tutorial_directory', - lastUpdatedAt: '2020-11-24T11:27:57.067Z', - fromTimestamp: '2020-11-24T00:00:00Z', - counterType: 'click', - total: 3, - }, - { - appName: 'Kibana_home', - eventName: 'home_tutorial_directory', - lastUpdatedAt: '2020-11-24T11:27:57.067Z', - fromTimestamp: '2020-11-24T00:00:00Z', - counterType: 'click', - total: 1, - }, - { - appName: 'Kibana_home', - eventName: 'home_tutorial_directory', - lastUpdatedAt: '2020-10-23T11:27:57.067Z', - fromTimestamp: '2020-10-23T00:00:00Z', - counterType: 'loaded', - total: 3, - }, - ]); +describe('transformRawUsageCounterObject', () => { + it('transforms usage counters savedObject raw entries', () => { + const result = rawUsageCounters.map(transformRawUsageCounterObject); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "my_event", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:17:57.693Z", + "total": 1, + }, + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "intersecting_event", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 60, + }, + undefined, + undefined, + undefined, + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "my_event_4457914848544_2", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 8, + }, + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "only_reported_in_usage_counters", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 1, + }, + ] + `); + }); +}); + +describe('transformRawUiCounterObject', () => { + it('transforms ui counters savedObject raw entries', () => { + const result = rawUiCounters.map(transformRawUiCounterObject); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "different_type", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 1, + }, + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "intersecting_event", + "fromTimestamp": "2020-10-25T00:00:00Z", + "lastUpdatedAt": "2020-10-25T11:27:57.067Z", + "total": 1, + }, + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "intersecting_event", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 3, + }, + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "only_reported_in_ui_counters", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 1, + }, + ] + `); + }); +}); + +describe('createFetchUiCounters', () => { + let stopUsingUiCounterIndicies$: BehaviorSubject; + const soClientMock = savedObjectsClientMock.create(); + beforeEach(() => { + jest.clearAllMocks(); + stopUsingUiCounterIndicies$ = new BehaviorSubject(false); + }); + + it('does not query ui_counters saved objects if stopUsingUiCounterIndicies$ is complete', async () => { + // @ts-expect-error incomplete mock implementation + soClientMock.find.mockImplementation(async ({ type }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: rawUsageCounters }; + default: + throw new Error(`unexpected type ${type}`); + } + }); + + stopUsingUiCounterIndicies$.complete(); + // @ts-expect-error incomplete mock implementation + const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({ + soClient: soClientMock, + }); + + const transforemdUsageCounters = rawUsageCounters.map(transformRawUsageCounterObject); + expect(soClientMock.find).toBeCalledTimes(1); + expect(dailyEvents).toEqual(transforemdUsageCounters.filter(Boolean)); + }); + + it('merges saved objects from both ui_counters and usage_counters saved objects', async () => { + // @ts-expect-error incomplete mock implementation + soClientMock.find.mockImplementation(async ({ type }) => { + switch (type) { + case UI_COUNTER_SAVED_OBJECT_TYPE: + return { saved_objects: rawUiCounters }; + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: rawUsageCounters }; + default: + throw new Error(`unexpected type ${type}`); + } + }); + + // @ts-expect-error incomplete mock implementation + const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({ + soClient: soClientMock, + }); + expect(dailyEvents).toHaveLength(7); + const intersectingEntry = dailyEvents.find( + ({ eventName, fromTimestamp }) => + eventName === 'intersecting_event' && fromTimestamp === '2020-10-23T00:00:00Z' + ); + + const onlyFromUICountersEntry = dailyEvents.find( + ({ eventName }) => eventName === 'only_reported_in_ui_counters' + ); + + const onlyFromUsageCountersEntry = dailyEvents.find( + ({ eventName }) => eventName === 'only_reported_in_usage_counters' + ); + + const invalidCountEntry = dailyEvents.find( + ({ eventName }) => eventName === 'my_event_malformed' + ); + + const zeroCountEntry = dailyEvents.find( + ({ eventName }) => eventName === 'my_event_4457914848544' + ); + + const nonUiCountersEntry = dailyEvents.find(({ eventName }) => eventName === 'some_event_name'); + + expect(invalidCountEntry).toBe(undefined); + expect(nonUiCountersEntry).toBe(undefined); + expect(zeroCountEntry).toBe(undefined); + expect(onlyFromUICountersEntry).toMatchInlineSnapshot(` + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "only_reported_in_ui_counters", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 1, + } + `); + expect(onlyFromUsageCountersEntry).toMatchInlineSnapshot(` + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "only_reported_in_usage_counters", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 1, + } + `); + expect(intersectingEntry).toMatchInlineSnapshot(` + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "intersecting_event", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 63, + } + `); }); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts index dc3fac738209..19190de45d96 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts @@ -7,13 +7,28 @@ */ import moment from 'moment'; -import { CollectorFetchContext, UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { mergeWith } from 'lodash'; +import type { Subject } from 'rxjs'; import { UICounterSavedObject, UICounterSavedObjectAttributes, UI_COUNTER_SAVED_OBJECT_TYPE, } from './ui_counter_saved_object_type'; +import { + CollectorFetchContext, + UsageCollectionSetup, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, + serializeCounterKey, +} from '../../../../usage_collection/server'; + +import { + deserializeUiCounterName, + serializeUiCounterName, +} from '../../../../usage_collection/common/ui_counters'; + interface UiCounterEvent { appName: string; eventName: string; @@ -27,12 +42,20 @@ export interface UiCountersUsage { dailyEvents: UiCounterEvent[]; } -export function transformRawCounter(rawUiCounter: UICounterSavedObject) { - const { id, attributes, updated_at: lastUpdatedAt } = rawUiCounter; +export function transformRawUiCounterObject( + rawUiCounter: UICounterSavedObject +): UiCounterEvent | undefined { + const { + id, + attributes: { count }, + updated_at: lastUpdatedAt, + } = rawUiCounter; + if (typeof count !== 'number' || count < 1) { + return; + } + const [appName, , counterType, ...restId] = id.split(':'); const eventName = restId.join(':'); - const counterTotal: unknown = attributes.count; - const total = typeof counterTotal === 'number' ? counterTotal : 0; const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); return { @@ -41,11 +64,110 @@ export function transformRawCounter(rawUiCounter: UICounterSavedObject) { lastUpdatedAt, fromTimestamp, counterType, - total, + total: count, }; } -export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) { +export function transformRawUsageCounterObject( + rawUsageCounter: UsageCountersSavedObject +): UiCounterEvent | undefined { + const { + attributes: { count, counterName, counterType, domainId }, + updated_at: lastUpdatedAt, + } = rawUsageCounter; + + if (domainId !== 'uiCounter' || typeof count !== 'number' || count < 1) { + return; + } + + const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); + const { appName, eventName } = deserializeUiCounterName(counterName); + + return { + appName, + eventName, + lastUpdatedAt, + fromTimestamp, + counterType, + total: count, + }; +} + +export const createFetchUiCounters = (stopUsingUiCounterIndicies$: Subject) => + async function fetchUiCounters({ soClient }: CollectorFetchContext) { + const { + saved_objects: rawUsageCounters, + } = await soClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + fields: ['count', 'counterName', 'counterType', 'domainId'], + filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, + perPage: 10000, + }); + + const skipFetchingUiCounters = stopUsingUiCounterIndicies$.isStopped; + const result = + skipFetchingUiCounters || + (await soClient.find({ + type: UI_COUNTER_SAVED_OBJECT_TYPE, + fields: ['count'], + perPage: 10000, + })); + + const rawUiCounters = typeof result === 'object' ? result.saved_objects : []; + const dailyEventsFromUiCounters = rawUiCounters.reduce((acc, raw) => { + try { + const event = transformRawUiCounterObject(raw); + if (event) { + const { appName, eventName, counterType } = event; + const key = serializeCounterKey({ + domainId: 'uiCounter', + counterName: serializeUiCounterName({ appName, eventName }), + counterType, + date: event.lastUpdatedAt, + }); + + acc[key] = event; + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. + } + return acc; + }, {} as Record); + + const dailyEventsFromUsageCounters = rawUsageCounters.reduce((acc, raw) => { + try { + const event = transformRawUsageCounterObject(raw); + if (event) { + acc[raw.id] = event; + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. + } + return acc; + }, {} as Record); + + const mergedDailyCounters = mergeWith( + dailyEventsFromUsageCounters, + dailyEventsFromUiCounters, + (value: UiCounterEvent | undefined, srcValue: UiCounterEvent): UiCounterEvent => { + if (!value) { + return srcValue; + } + + return { + ...srcValue, + total: srcValue.total + value.total, + }; + } + ); + + return { dailyEvents: Object.values(mergedDailyCounters) }; + }; + +export function registerUiCountersUsageCollector( + usageCollection: UsageCollectionSetup, + stopUsingUiCounterIndicies$: Subject +) { const collector = usageCollection.makeUsageCollector({ type: 'ui_counters', schema: { @@ -76,25 +198,7 @@ export function registerUiCountersUsageCollector(usageCollection: UsageCollectio }, }, }, - fetch: async ({ soClient }: CollectorFetchContext) => { - const { saved_objects: rawUiCounters } = await soClient.find({ - type: UI_COUNTER_SAVED_OBJECT_TYPE, - fields: ['count'], - perPage: 10000, - }); - - return { - dailyEvents: rawUiCounters.reduce((acc, raw) => { - try { - const aggEvent = transformRawCounter(raw); - acc.push(aggEvent); - } catch (_) { - // swallow error; allows sending successfully transformed objects. - } - return acc; - }, [] as UiCounterEvent[]), - }; - }, + fetch: createFetchUiCounters(stopUsingUiCounterIndicies$), isReady: () => true, }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts index 9595101efb63..55da239d8ef2 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts @@ -6,16 +6,20 @@ * Side Public License, v 1. */ -import { timer } from 'rxjs'; +import { Subject, timer } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; import { Logger, ISavedObjectsRepository } from 'kibana/server'; import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants'; import { rollUiCounterIndices } from './rollups'; export function registerUiCountersRollups( logger: Logger, + stopRollingUiCounterIndicies$: Subject, getSavedObjectsClient: () => ISavedObjectsRepository | undefined ) { - timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL).subscribe(() => - rollUiCounterIndices(logger, getSavedObjectsClient()) - ); + timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL) + .pipe(takeUntil(stopRollingUiCounterIndicies$)) + .subscribe(() => + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, getSavedObjectsClient()) + ); } diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts index 5cb91f7f898c..f69ddde6a65b 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts @@ -7,9 +7,11 @@ */ import moment from 'moment'; +import * as Rx from 'rxjs'; import { isSavedObjectOlderThan, rollUiCounterIndices } from './rollups'; import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks'; import { SavedObjectsFindResult } from 'kibana/server'; + import { UICounterSavedObjectAttributes, UI_COUNTER_SAVED_OBJECT_TYPE, @@ -70,14 +72,18 @@ describe('isSavedObjectOlderThan', () => { describe('rollUiCounterIndices', () => { let logger: ReturnType; let savedObjectClient: ReturnType; + let stopUsingUiCounterIndicies$: Rx.Subject; beforeEach(() => { logger = loggingSystemMock.createLogger(); savedObjectClient = savedObjectsRepositoryMock.create(); + stopUsingUiCounterIndicies$ = new Rx.Subject(); }); it('returns undefined if no savedObjectsClient initialised yet', async () => { - await expect(rollUiCounterIndices(logger, undefined)).resolves.toBe(undefined); + await expect( + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, undefined) + ).resolves.toBe(undefined); expect(logger.warn).toHaveBeenCalledTimes(0); }); @@ -90,11 +96,27 @@ describe('rollUiCounterIndices', () => { throw new Error(`Unexpected type [${type}]`); } }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual([]); + await expect( + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual([]); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).not.toBeCalled(); expect(logger.warn).toHaveBeenCalledTimes(0); }); + it('calls Subject complete() on empty saved objects', async () => { + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case UI_COUNTER_SAVED_OBJECT_TYPE: + return { saved_objects: [], total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + await expect( + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual([]); + expect(stopUsingUiCounterIndicies$.isStopped).toBe(true); + }); it(`deletes documents older than ${UI_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => { const mockSavedObjects = [ @@ -111,7 +133,9 @@ describe('rollUiCounterIndices', () => { throw new Error(`Unexpected type [${type}]`); } }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toHaveLength(2); + await expect( + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) + ).resolves.toHaveLength(2); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).toHaveBeenCalledTimes(2); expect(savedObjectClient.delete).toHaveBeenNthCalledWith( @@ -131,7 +155,9 @@ describe('rollUiCounterIndices', () => { savedObjectClient.find.mockImplementation(async () => { throw new Error(`Expected error!`); }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + await expect( + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual(undefined); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).not.toBeCalled(); expect(logger.warn).toHaveBeenCalledTimes(2); @@ -151,7 +177,9 @@ describe('rollUiCounterIndices', () => { savedObjectClient.delete.mockImplementation(async () => { throw new Error(`Expected error!`); }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + await expect( + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual(undefined); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).toHaveBeenCalledTimes(1); expect(savedObjectClient.delete).toHaveBeenNthCalledWith( diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts index 3a092f845c3a..79e7d3e07ba4 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts @@ -8,6 +8,7 @@ import { ISavedObjectsRepository, Logger } from 'kibana/server'; import moment from 'moment'; +import type { Subject } from 'rxjs'; import { UI_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; import { @@ -38,6 +39,7 @@ export function isSavedObjectOlderThan({ export async function rollUiCounterIndices( logger: Logger, + stopUsingUiCounterIndicies$: Subject, savedObjectsClient?: ISavedObjectsRepository ) { if (!savedObjectsClient) { @@ -54,6 +56,20 @@ export async function rollUiCounterIndices( } ); + if (rawUiCounterDocs.length === 0) { + /** + * @deprecated 7.13 to be removed in 8.0.0 + * Stop triggering rollups when we've rolled up all documents. + * + * This Saved Object registry is no longer used. + * Migration from one SO registry to another is not yet supported. + * In a future release we can remove this piece of code and + * migrate any docs to the Usage Counters Saved object. + */ + + stopUsingUiCounterIndicies$.complete(); + } + const docsToDelete = rawUiCounterDocs.filter((doc) => isSavedObjectOlderThan({ numberOfDays: UI_COUNTERS_KEEP_DOCS_FOR_DAYS, diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts index 77413cc7d7d9..51ecbf736bfc 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts @@ -11,7 +11,7 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerUiMetricUsageCollector } from './'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts new file mode 100644 index 000000000000..d0a45fb86b1f --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UsageCountersSavedObject } from '../../../../../usage_collection/server'; + +export const rawUsageCounters: UsageCountersSavedObject[] = [ + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event', + attributes: { + count: 13, + counterName: 'my_event', + counterType: 'count', + domainId: 'uiCounter', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId:09042021:count:some_event_name', + attributes: { + count: 4, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId:09042021:count:some_event_name', + attributes: { + count: 4, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-11T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId2:09042021:count:some_event_name', + attributes: { + count: 1, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId2', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId2:09042021:count:malformed_event', + attributes: { + // @ts-expect-error + count: 'malformed', + counterName: 'malformed_event', + counterType: 'count', + domainId: 'anotherDomainId2', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId2:09042021:custom_type:some_event_name', + attributes: { + count: 3, + counterName: 'some_event_name', + counterType: 'custom_type', + domainId: 'anotherDomainId2', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + }, + { + type: 'usage-counters', + id: 'anotherDomainId3:09042021:custom_type:zero_count', + attributes: { + count: 0, + counterName: 'zero_count', + counterType: 'custom_type', + domainId: 'anotherDomainId3', + }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + }, +]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts new file mode 100644 index 000000000000..1873fae42e54 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { registerUsageCountersUsageCollector } from './register_usage_counters_collector'; +export { registerUsageCountersRollups } from './rollups'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts new file mode 100644 index 000000000000..945eb007fe23 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { transformRawCounter } from './register_usage_counters_collector'; +import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; + +describe('transformRawCounter', () => { + it('transforms saved object raw entries', () => { + const result = rawUsageCounters.map(transformRawCounter); + expect(result).toMatchInlineSnapshot(` + Array [ + undefined, + Object { + "counterName": "some_event_name", + "counterType": "count", + "domainId": "anotherDomainId", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.030Z", + "total": 4, + }, + Object { + "counterName": "some_event_name", + "counterType": "count", + "domainId": "anotherDomainId", + "fromTimestamp": "2021-04-11T00:00:00Z", + "lastUpdatedAt": "2021-04-11T08:18:03.030Z", + "total": 4, + }, + Object { + "counterName": "some_event_name", + "counterType": "count", + "domainId": "anotherDomainId2", + "fromTimestamp": "2021-04-20T00:00:00Z", + "lastUpdatedAt": "2021-04-20T08:18:03.030Z", + "total": 1, + }, + undefined, + Object { + "counterName": "some_event_name", + "counterType": "custom_type", + "domainId": "anotherDomainId2", + "fromTimestamp": "2021-04-20T00:00:00Z", + "lastUpdatedAt": "2021-04-20T08:18:03.030Z", + "total": 3, + }, + undefined, + ] + `); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts new file mode 100644 index 000000000000..9c6db00fb359 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import { + CollectorFetchContext, + UsageCollectionSetup, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, +} from '../../../../usage_collection/server'; + +interface UsageCounterEvent { + domainId: string; + counterName: string; + counterType: string; + lastUpdatedAt?: string; + fromTimestamp?: string; + total: number; +} + +export interface UiCountersUsage { + dailyEvents: UsageCounterEvent[]; +} + +export function transformRawCounter( + rawUsageCounter: UsageCountersSavedObject +): UsageCounterEvent | undefined { + const { + attributes: { count, counterName, counterType, domainId }, + updated_at: lastUpdatedAt, + } = rawUsageCounter; + const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); + + if (domainId === 'uiCounter' || typeof count !== 'number' || count < 1) { + return; + } + + return { + domainId, + counterName, + counterType, + lastUpdatedAt, + fromTimestamp, + total: count, + }; +} + +export function registerUsageCountersUsageCollector(usageCollection: UsageCollectionSetup) { + const collector = usageCollection.makeUsageCollector({ + type: 'usage_counters', + schema: { + dailyEvents: { + type: 'array', + items: { + domainId: { + type: 'keyword', + _meta: { description: 'Domain name of the metric (ie plugin name).' }, + }, + counterName: { + type: 'keyword', + _meta: { description: 'Name of the counter that happened.' }, + }, + lastUpdatedAt: { + type: 'date', + _meta: { description: 'Time at which the metric was last updated.' }, + }, + fromTimestamp: { + type: 'date', + _meta: { description: 'Time at which the metric was captured.' }, + }, + counterType: { + type: 'keyword', + _meta: { description: 'The type of counter used.' }, + }, + total: { + type: 'integer', + _meta: { description: 'The total number of times the event happened.' }, + }, + }, + }, + }, + fetch: async ({ soClient }: CollectorFetchContext) => { + const { + saved_objects: rawUsageCounters, + } = await soClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + fields: ['count', 'counterName', 'counterType', 'domainId'], + filter: `NOT ${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, + perPage: 10000, + }); + + return { + dailyEvents: rawUsageCounters.reduce((acc, rawUsageCounter) => { + try { + const event = transformRawCounter(rawUsageCounter); + if (event) { + acc.push(event); + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. + } + return acc; + }, [] as UsageCounterEvent[]), + }; + }, + isReady: () => true, + }); + + usageCollection.registerCollector(collector); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts new file mode 100644 index 000000000000..1c1ca3f466df --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Roll indices every 24h + */ +export const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000; + +/** + * Start rolling indices after 5 minutes up + */ +export const ROLL_INDICES_START = 5 * 60 * 1000; + +/** + * Number of days to keep the Usage counters saved object documents + */ +export const USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS = 5; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts new file mode 100644 index 000000000000..bf15f4d87586 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { registerUsageCountersRollups } from './register_rollups'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts new file mode 100644 index 000000000000..30ad993d54a8 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { timer } from 'rxjs'; +import { Logger, ISavedObjectsRepository } from 'kibana/server'; +import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants'; +import { rollUsageCountersIndices } from './rollups'; + +export function registerUsageCountersRollups( + logger: Logger, + getSavedObjectsClient: () => ISavedObjectsRepository | undefined +) { + timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL).subscribe(() => + rollUsageCountersIndices(logger, getSavedObjectsClient()) + ); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts new file mode 100644 index 000000000000..c6cdaae20a8b --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import { isSavedObjectOlderThan, rollUsageCountersIndices } from './rollups'; +import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks'; +import { SavedObjectsFindResult } from '../../../../../../core/server'; + +import { + UsageCountersSavedObjectAttributes, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '../../../../../usage_collection/server'; + +import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; + +const createMockSavedObjectDoc = (updatedAt: moment.Moment, id: string) => + ({ + id, + type: 'usage-counter', + attributes: { + count: 3, + counterName: 'testName', + counterType: 'count', + domainId: 'testDomain', + }, + references: [], + updated_at: updatedAt.format(), + version: 'WzI5LDFd', + score: 0, + } as SavedObjectsFindResult); + +describe('isSavedObjectOlderThan', () => { + it(`returns true if doc is older than x days`, () => { + const numberOfDays = 1; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(2, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(true); + }); + + it(`returns false if doc is exactly x days old`, () => { + const numberOfDays = 1; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(false); + }); + + it(`returns false if doc is younger than x days`, () => { + const numberOfDays = 2; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(false); + }); +}); + +describe('rollUsageCountersIndices', () => { + let logger: ReturnType; + let savedObjectClient: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + savedObjectClient = savedObjectsRepositoryMock.create(); + }); + + it('returns undefined if no savedObjectsClient initialised yet', async () => { + await expect(rollUsageCountersIndices(logger, undefined)).resolves.toBe(undefined); + expect(logger.warn).toHaveBeenCalledTimes(0); + }); + + it('does not delete any documents on empty saved objects', async () => { + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: [], total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual([]); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).not.toBeCalled(); + expect(logger.warn).toHaveBeenCalledTimes(0); + }); + + it(`deletes documents older than ${USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => { + const mockSavedObjects = [ + createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1'), + createMockSavedObjectDoc(moment().subtract(9, 'days'), 'doc-id-1'), + createMockSavedObjectDoc(moment().subtract(1, 'days'), 'doc-id-2'), + createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3'), + ]; + + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toHaveLength(2); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).toHaveBeenCalledTimes(2); + expect(savedObjectClient.delete).toHaveBeenNthCalledWith( + 1, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + 'doc-id-1' + ); + expect(savedObjectClient.delete).toHaveBeenNthCalledWith( + 2, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + 'doc-id-3' + ); + expect(logger.warn).toHaveBeenCalledTimes(0); + }); + + it(`logs warnings on savedObject.find failure`, async () => { + savedObjectClient.find.mockImplementation(async () => { + throw new Error(`Expected error!`); + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).not.toBeCalled(); + expect(logger.warn).toHaveBeenCalledTimes(2); + }); + + it(`logs warnings on savedObject.delete failure`, async () => { + const mockSavedObjects = [createMockSavedObjectDoc(moment().subtract(7, 'days'), 'doc-id-1')]; + + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + savedObjectClient.delete.mockImplementation(async () => { + throw new Error(`Expected error!`); + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).toHaveBeenCalledTimes(1); + expect(savedObjectClient.delete).toHaveBeenNthCalledWith( + 1, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + 'doc-id-1' + ); + expect(logger.warn).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts new file mode 100644 index 000000000000..c07ea37536f2 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ISavedObjectsRepository, Logger } from 'kibana/server'; +import moment from 'moment'; + +import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; + +import { + UsageCountersSavedObject, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '../../../../../usage_collection/server'; + +export function isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, +}: { + numberOfDays: number; + startDate: moment.Moment | string | number; + doc: Pick; +}): boolean { + const { updated_at: updatedAt } = doc; + const today = moment(startDate).startOf('day'); + const updateDay = moment(updatedAt).startOf('day'); + + const diffInDays = today.diff(updateDay, 'days'); + if (diffInDays > numberOfDays) { + return true; + } + + return false; +} + +export async function rollUsageCountersIndices( + logger: Logger, + savedObjectsClient?: ISavedObjectsRepository +) { + if (!savedObjectsClient) { + return; + } + + const now = moment(); + + try { + const { + saved_objects: rawUiCounterDocs, + } = await savedObjectsClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + perPage: 1000, // Process 1000 at a time as a compromise of speed and overload + }); + + const docsToDelete = rawUiCounterDocs.filter((doc) => + isSavedObjectOlderThan({ + numberOfDays: USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS, + startDate: now, + doc, + }) + ); + + return await Promise.all( + docsToDelete.map(({ id }) => savedObjectsClient.delete(USAGE_COUNTERS_SAVED_OBJECT_TYPE, id)) + ); + } catch (err) { + logger.warn(`Failed to rollup Usage Counters saved objects.`); + logger.warn(err); + } +} diff --git a/src/plugins/kibana_usage_collection/server/index.test.mocks.ts b/src/plugins/kibana_usage_collection/server/mocks.ts similarity index 100% rename from src/plugins/kibana_usage_collection/server/index.test.mocks.ts rename to src/plugins/kibana_usage_collection/server/mocks.ts diff --git a/src/plugins/kibana_usage_collection/server/index.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts similarity index 59% rename from src/plugins/kibana_usage_collection/server/index.test.ts rename to src/plugins/kibana_usage_collection/server/plugin.test.ts index b4c52f8353d7..86204ed30e65 100644 --- a/src/plugins/kibana_usage_collection/server/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -14,8 +14,8 @@ import { import { CollectorOptions, createUsageCollectionSetupMock, -} from '../../usage_collection/server/usage_collection.mock'; -import { cloudDetailsMock } from './index.test.mocks'; +} from '../../usage_collection/server/mocks'; +import { cloudDetailsMock } from './mocks'; import { plugin } from './'; @@ -38,13 +38,67 @@ describe('kibana_usage_collection', () => { cloudDetailsMock.mockClear(); }); - test('Runs the setup method without issues', () => { + test('Runs the setup method without issues', async () => { const coreSetup = coreMock.createSetup(); expect(pluginInstance.setup(coreSetup, { usageCollection })).toBe(undefined); - usageCollectors.forEach(({ isReady }) => { - expect(isReady()).toMatchSnapshot(); // Some should return false at this stage - }); + + await expect( + Promise.all( + usageCollectors.map(async (usageCollector) => { + const isReady = await usageCollector.isReady(); + const type = usageCollector.type; + return { type, isReady }; + }) + ) + ).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "isReady": true, + "type": "ui_counters", + }, + Object { + "isReady": true, + "type": "usage_counters", + }, + Object { + "isReady": false, + "type": "kibana_stats", + }, + Object { + "isReady": true, + "type": "kibana", + }, + Object { + "isReady": false, + "type": "stack_management", + }, + Object { + "isReady": false, + "type": "ui_metric", + }, + Object { + "isReady": false, + "type": "application_usage", + }, + Object { + "isReady": false, + "type": "cloud_provider", + }, + Object { + "isReady": true, + "type": "csp", + }, + Object { + "isReady": false, + "type": "core", + }, + Object { + "isReady": true, + "type": "localization", + }, + ] + `); }); test('Runs the start method without issues', () => { diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index 74d2d281ff8f..a27b8dff57b6 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -35,6 +35,8 @@ import { registerUiCountersUsageCollector, registerUiCounterSavedObjectType, registerUiCountersRollups, + registerUsageCountersRollups, + registerUsageCountersUsageCollector, } from './collectors'; interface KibanaUsageCollectionPluginsDepsSetup { @@ -50,18 +52,23 @@ export class KibanaUsageCollectionPlugin implements Plugin { private uiSettingsClient?: IUiSettingsClient; private metric$: Subject; private coreUsageData?: CoreUsageDataStart; + private stopUsingUiCounterIndicies$: Subject; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); this.legacyConfig$ = initializerContext.config.legacy.globalConfig$; this.metric$ = new Subject(); + this.stopUsingUiCounterIndicies$ = new Subject(); } public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { + usageCollection.createUsageCounter('uiCounters'); + this.registerUsageCollectors( usageCollection, coreSetup, this.metric$, + this.stopUsingUiCounterIndicies$, coreSetup.savedObjects.registerType.bind(coreSetup.savedObjects) ); } @@ -77,12 +84,14 @@ export class KibanaUsageCollectionPlugin implements Plugin { public stop() { this.metric$.complete(); + this.stopUsingUiCounterIndicies$.complete(); } private registerUsageCollectors( usageCollection: UsageCollectionSetup, coreSetup: CoreSetup, metric$: Subject, + stopUsingUiCounterIndicies$: Subject, registerType: SavedObjectsRegisterType ) { const getSavedObjectsClient = () => this.savedObjectsClient; @@ -90,8 +99,15 @@ export class KibanaUsageCollectionPlugin implements Plugin { const getCoreUsageDataService = () => this.coreUsageData!; registerUiCounterSavedObjectType(coreSetup.savedObjects); - registerUiCountersRollups(this.logger.get('ui-counters'), getSavedObjectsClient); - registerUiCountersUsageCollector(usageCollection); + registerUiCountersRollups( + this.logger.get('ui-counters'), + stopUsingUiCounterIndicies$, + getSavedObjectsClient + ); + registerUiCountersUsageCollector(usageCollection, stopUsingUiCounterIndicies$); + + registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient); + registerUsageCountersUsageCollector(usageCollection); registerOpsStatsCollector(usageCollection, metric$); registerKibanaUsageCollector(usageCollection, this.legacyConfig$); diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 9a13540f8ba0..761a620c4f37 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -9314,6 +9314,53 @@ } } }, + "usage_counters": { + "properties": { + "dailyEvents": { + "type": "array", + "items": { + "properties": { + "domainId": { + "type": "keyword", + "_meta": { + "description": "Domain name of the metric (ie plugin name)." + } + }, + "counterName": { + "type": "keyword", + "_meta": { + "description": "Name of the counter that happened." + } + }, + "lastUpdatedAt": { + "type": "date", + "_meta": { + "description": "Time at which the metric was last updated." + } + }, + "fromTimestamp": { + "type": "date", + "_meta": { + "description": "Time at which the metric was captured." + } + }, + "counterType": { + "type": "keyword", + "_meta": { + "description": "The type of counter used." + } + }, + "total": { + "type": "integer", + "_meta": { + "description": "The total number of times the event happened." + } + } + } + } + } + } + }, "telemetry": { "properties": { "opt_in_status": { diff --git a/src/plugins/usage_collection/README.mdx b/src/plugins/usage_collection/README.mdx index 04e1e0fbb500..a6f6f6c8e597 100644 --- a/src/plugins/usage_collection/README.mdx +++ b/src/plugins/usage_collection/README.mdx @@ -20,6 +20,7 @@ The way to report the usage of any feature depends on whether the actions to tra In any case, to use any of these APIs, the plugin must optionally require the plugin `usageCollection`: + ```json // plugin/kibana.json { @@ -112,6 +113,100 @@ Not an API as such. However, Data Telemetry collects the usage of known patterns This collector does not report the name of the indices nor any content. It only provides stats about usage of known shippers/ingest tools. +#### Usage Counters + +Usage counters allows plugins to report user triggered events from the server. This api has feature parity with UI Counters on the `public` plugin side of usage_collection. + +Usage counters provide instrumentation on the server to count triggered events such as "api called", "threshold reached", and miscellaneous events count. + +It is useful for gathering _semi-aggregated_ events with a per day granularity. +This allows tracking trends in usage and provides enough granularity for this type of telemetry to provide insights such as +- "How many times this threshold has been reached?" +- "What is the trend in usage of this api?" +- "How frequent are users hitting this error per day?" +- "What is the success rate of this operation?" +- "Which option is being selected the most/least?" + +##### How to use it + +To create a usage counter for your plugin, use the API `usageCollection.createUsageCounter` as follows: + +```ts +// server/plugin.ts +import type { Plugin, CoreStart } from '../../../core/server'; +import type { UsageCollectionSetup, UsageCounter } from '../../../plugins/usage_collection/server'; + +export class MyPlugin implements Plugin { + private usageCounter?: UsageCounter; + public setup( + core: CoreStart, + { usageCollection }: { usageCollection?: UsageCollectionSetup } + ) { + + /** + * Create a usage counter for this plugin. Domain ID must be unique. + * It is advised to use the plugin name as the domain ID for most cases. + */ + this.usageCounter = usageCollection?.createUsageCounter(''); + try { + doSomeOperation(); + this.usageCounter?.incrementCounter({ + counterName: 'doSomeOperation_success', + incrementBy: 1, + }); + } catch (err) { + this.usageCounter?.incrementCounter({ + counterName: 'doSomeOperation_error', + counterType: 'error', + incrementBy: 1, + }); + logger.error(err); + } + } +} +``` + +Pass the created `usageCounter` around in your service to instrument usage. + +That's all you need to do! The Usage counters service will handle piping these counters all the way to the telemetry service. + +##### Telemetry reported usage + +Usage counters are reported inside the telemetry usage payload under `stack_stats.kibana.plugins.usage_counters`. + +```ts +{ + usage_counters: { + dailyEvents: [ + { + domainId: '', + counterName: 'doSomeOperation_success', + counterType: 'count', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + total: 3, + }, + { + domainId: '', + counterName: 'doSomeOperation_success', + counterType: 'count', + lastUpdatedAt: '2021-11-21T10:30:00.961Z', + fromTimestamp: '2021-11-21T00:00:00Z', + total: 5, + }, + { + domainId: '', + counterName: 'doSomeOperation_error', + counterType: 'error', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + total: 1, + }, + ], + }, +} +``` + #### Custom collector In many cases, plugins need to report the custom usage of a feature. In this cases, the plugins must complete the following 2 steps in the `setup` lifecycle step: diff --git a/src/plugins/usage_collection/common/ui_counters.ts b/src/plugins/usage_collection/common/ui_counters.ts new file mode 100644 index 000000000000..3ed6e44aee41 --- /dev/null +++ b/src/plugins/usage_collection/common/ui_counters.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const serializeUiCounterName = ({ + appName, + eventName, +}: { + appName: string; + eventName: string; +}) => { + return `${appName}:${eventName}`; +}; + +export const deserializeUiCounterName = (key: string) => { + const [appName, ...restKey] = key.split(':'); + const eventName = restKey.join(':'); + return { appName, eventName }; +}; diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 32a58a6657ee..4de5691eaaa7 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -25,22 +25,6 @@ interface CollectorSetConfig { collectors?: AnyCollector[]; } -/** - * Public interface of the CollectorSet (makes it easier to mock only the public methods) - */ -export type CollectorSetPublic = Pick< - CollectorSet, - | 'makeStatsCollector' - | 'makeUsageCollector' - | 'registerCollector' - | 'getCollectorByType' - | 'areAllCollectorsReady' - | 'bulkFetch' - | 'bulkFetchUsage' - | 'toObject' - | 'toApiFieldNames' ->; - export class CollectorSet { private _waitingForAllCollectorsTimestamp?: number; private readonly logger: Logger; @@ -215,19 +199,19 @@ export class CollectorSet { * Convert an array of fetched stats results into key/object * @param statsData Array of fetched stats results */ - public toObject, T = unknown>( + public toObject = , T = unknown>( statsData: Array<{ type: string; result: T }> = [] - ): Result { + ): Result => { return Object.fromEntries(statsData.map(({ type, result }) => [type, result])) as Result; - } + }; /** * Rename fields to use API conventions * @param apiData Data to be normalized */ - public toApiFieldNames( + public toApiFieldNames = ( apiData: Record | unknown[] - ): Record | unknown[] { + ): Record | unknown[] => { // handle array and return early, or return a reduced object if (Array.isArray(apiData)) { return apiData.map((value) => this.getValueOrRecurse(value)); @@ -244,14 +228,14 @@ export class CollectorSet { return [newName, this.getValueOrRecurse(value)]; }) ); - } + }; - private getValueOrRecurse(value: unknown) { + private getValueOrRecurse = (value: unknown) => { if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { return this.toApiFieldNames(value as Record | unknown[]); // recurse } return value; - } + }; private makeCollectorSetFromArray = (collectors: AnyCollector[]) => { return new CollectorSet({ diff --git a/src/plugins/usage_collection/server/collector/index.ts b/src/plugins/usage_collection/server/collector/index.ts index d5e0d95659e5..594455f70fdf 100644 --- a/src/plugins/usage_collection/server/collector/index.ts +++ b/src/plugins/usage_collection/server/collector/index.ts @@ -7,7 +7,6 @@ */ export { CollectorSet } from './collector_set'; -export type { CollectorSetPublic } from './collector_set'; export { Collector } from './collector'; export type { AllowedSchemaTypes, diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index ff6ea8424ba6..cd6f6b9d8139 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -11,6 +11,11 @@ import { PluginConfigDescriptor } from 'src/core/server'; import { DEFAULT_MAXIMUM_WAIT_TIME_FOR_ALL_COLLECTORS_IN_S } from '../common/constants'; export const configSchema = schema.object({ + usageCounters: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + retryCount: schema.number({ defaultValue: 1 }), + bufferDuration: schema.duration({ defaultValue: '5s' }), + }), uiCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), debug: schema.boolean({ defaultValue: schema.contextRef('dev') }), diff --git a/src/plugins/usage_collection/server/index.ts b/src/plugins/usage_collection/server/index.ts index dd9e6644a827..b5441a8b7b34 100644 --- a/src/plugins/usage_collection/server/index.ts +++ b/src/plugins/usage_collection/server/index.ts @@ -18,6 +18,19 @@ export type { UsageCollectorOptions, CollectorFetchContext, } from './collector'; + +export type { + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, + IncrementCounterParams, +} from './usage_counters'; + +export { + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + serializeCounterKey, + UsageCounter, +} from './usage_counters'; + export type { UsageCollectionSetup } from './plugin'; export { config } from './config'; export const plugin = (initializerContext: PluginInitializerContext) => diff --git a/src/plugins/usage_collection/server/mocks.ts b/src/plugins/usage_collection/server/mocks.ts index e5ad10226362..b84fa0f0aab7 100644 --- a/src/plugins/usage_collection/server/mocks.ts +++ b/src/plugins/usage_collection/server/mocks.ts @@ -6,20 +6,61 @@ * Side Public License, v 1. */ -import { loggingSystemMock } from '../../../core/server/mocks'; -import { UsageCollectionSetup } from './plugin'; -import { CollectorSet } from './collector'; -export { Collector, createCollectorFetchContextMock } from './usage_collection.mock'; +import { + elasticsearchServiceMock, + httpServerMock, + loggingSystemMock, + savedObjectsClientMock, +} from '../../../../src/core/server/mocks'; -const createSetupContract = () => { - return { - ...new CollectorSet({ - logger: loggingSystemMock.createLogger(), - maximumWaitTimeForAllCollectorsInS: 1, - }), - } as UsageCollectionSetup; +import { CollectorOptions, Collector, CollectorSet } from './collector'; +import { UsageCollectionSetup, CollectorFetchContext } from './index'; + +export type { CollectorOptions }; +export { Collector }; + +export const createUsageCollectionSetupMock = () => { + const collectorSet = new CollectorSet({ + logger: loggingSystemMock.createLogger(), + maximumWaitTimeForAllCollectorsInS: 1, + }); + + const usageCollectionSetupMock: jest.Mocked = { + createUsageCounter: jest.fn(), + getUsageCounterByType: jest.fn(), + areAllCollectorsReady: jest.fn().mockImplementation(collectorSet.areAllCollectorsReady), + bulkFetch: jest.fn().mockImplementation(collectorSet.bulkFetch), + getCollectorByType: jest.fn().mockImplementation(collectorSet.getCollectorByType), + toApiFieldNames: jest.fn().mockImplementation(collectorSet.toApiFieldNames), + toObject: jest.fn().mockImplementation(collectorSet.toObject), + makeStatsCollector: jest.fn().mockImplementation(collectorSet.makeStatsCollector), + makeUsageCollector: jest.fn().mockImplementation(collectorSet.makeUsageCollector), + registerCollector: jest.fn().mockImplementation(collectorSet.registerCollector), + }; + + usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true); + return usageCollectionSetupMock; }; +export function createCollectorFetchContextMock(): jest.Mocked> { + const collectorFetchClientsMock: jest.Mocked> = { + esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, + soClient: savedObjectsClientMock.create(), + }; + return collectorFetchClientsMock; +} + +export function createCollectorFetchContextWithKibanaMock(): jest.Mocked< + CollectorFetchContext +> { + const collectorFetchClientsMock: jest.Mocked> = { + esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, + soClient: savedObjectsClientMock.create(), + kibanaRequest: httpServerMock.createKibanaRequest(), + }; + return collectorFetchClientsMock; +} + export const usageCollectionPluginMock = { - createSetupContract, + createSetupContract: createUsageCollectionSetupMock, }; diff --git a/src/plugins/usage_collection/server/plugin.ts b/src/plugins/usage_collection/server/plugin.ts index a44365ae9be9..37d7327aed66 100644 --- a/src/plugins/usage_collection/server/plugin.ts +++ b/src/plugins/usage_collection/server/plugin.ts @@ -15,30 +15,78 @@ import { Plugin, } from 'src/core/server'; import { ConfigType } from './config'; -import { CollectorSet, CollectorSetPublic } from './collector'; +import { CollectorSet } from './collector'; import { setupRoutes } from './routes'; -export type UsageCollectionSetup = CollectorSetPublic; -export class UsageCollectionPlugin implements Plugin { +import { UsageCountersService } from './usage_counters'; +import type { UsageCountersServiceSetup } from './usage_counters'; + +export interface UsageCollectionSetup { + /** + * Creates and registers a usage counter to collect daily aggregated plugin counter events + */ + createUsageCounter: UsageCountersServiceSetup['createUsageCounter']; + /** + * Returns a usage counter by type + */ + getUsageCounterByType: UsageCountersServiceSetup['getUsageCounterByType']; + /** + * Creates a usage collector to collect plugin telemetry data. + * registerCollector must be called to connect the created collecter with the service. + */ + makeUsageCollector: CollectorSet['makeUsageCollector']; + /** + * Register a usage collector or a stats collector. + * Used to connect the created collector to telemetry. + */ + registerCollector: CollectorSet['registerCollector']; + /** + * Returns a usage collector by type + */ + getCollectorByType: CollectorSet['getCollectorByType']; + /* internal: telemetry use */ + areAllCollectorsReady: CollectorSet['areAllCollectorsReady']; + /* internal: telemetry use */ + bulkFetch: CollectorSet['bulkFetch']; + /* internal: telemetry use */ + toObject: CollectorSet['toObject']; + /* internal: monitoring use */ + toApiFieldNames: CollectorSet['toApiFieldNames']; + /* internal: telemtery and monitoring use */ + makeStatsCollector: CollectorSet['makeStatsCollector']; +} + +export class UsageCollectionPlugin implements Plugin { private readonly logger: Logger; private savedObjects?: ISavedObjectsRepository; + private usageCountersService?: UsageCountersService; + constructor(private readonly initializerContext: PluginInitializerContext) { this.logger = this.initializerContext.logger.get(); } - public setup(core: CoreSetup) { + public setup(core: CoreSetup): UsageCollectionSetup { const config = this.initializerContext.config.get(); const collectorSet = new CollectorSet({ - logger: this.logger.get('collector-set'), + logger: this.logger.get('usage-collection', 'collector-set'), maximumWaitTimeForAllCollectorsInS: config.maximumWaitTimeForAllCollectorsInS, }); - const globalConfig = this.initializerContext.config.legacy.get(); + this.usageCountersService = new UsageCountersService({ + logger: this.logger.get('usage-collection', 'usage-counters-service'), + retryCount: config.usageCounters.retryCount, + bufferDurationMs: config.usageCounters.bufferDuration.asMilliseconds(), + }); + const { createUsageCounter, getUsageCounterByType } = this.usageCountersService.setup(core); + + const uiCountersUsageCounter = createUsageCounter('uiCounter'); + const globalConfig = this.initializerContext.config.legacy.get(); const router = core.http.createRouter(); setupRoutes({ router, + uiCountersUsageCounter, getSavedObjects: () => this.savedObjects, collectorSet, config: { @@ -52,15 +100,38 @@ export class UsageCollectionPlugin implements Plugin { overallStatus$: core.status.overall$, }); - return collectorSet; + return { + areAllCollectorsReady: collectorSet.areAllCollectorsReady, + bulkFetch: collectorSet.bulkFetch, + getCollectorByType: collectorSet.getCollectorByType, + makeStatsCollector: collectorSet.makeStatsCollector, + makeUsageCollector: collectorSet.makeUsageCollector, + registerCollector: collectorSet.registerCollector, + toApiFieldNames: collectorSet.toApiFieldNames, + toObject: collectorSet.toObject, + createUsageCounter, + getUsageCounterByType, + }; } public start({ savedObjects }: CoreStart) { this.logger.debug('Starting plugin'); + const config = this.initializerContext.config.get(); + if (!this.usageCountersService) { + throw new Error('plugin setup must be called first.'); + } + this.savedObjects = savedObjects.createInternalRepository(); + if (config.usageCounters.enabled) { + this.usageCountersService.start({ savedObjects }); + } else { + // call stop() to complete observers. + this.usageCountersService.stop(); + } } public stop() { this.logger.debug('Stopping plugin'); + this.usageCountersService?.stop(); } } diff --git a/src/plugins/usage_collection/server/report/store_report.test.ts b/src/plugins/usage_collection/server/report/store_report.test.ts index dfcdd1f8e7e4..08fdec4ae804 100644 --- a/src/plugins/usage_collection/server/report/store_report.test.ts +++ b/src/plugins/usage_collection/server/report/store_report.test.ts @@ -12,11 +12,11 @@ import { savedObjectsRepositoryMock } from '../../../../core/server/mocks'; import { storeReport } from './store_report'; import { ReportSchemaType } from './schema'; import { METRIC_TYPE } from '@kbn/analytics'; -import moment from 'moment'; +import { usageCountersServiceMock } from '../usage_counters/usage_counters_service.mock'; describe('store_report', () => { - const momentTimestamp = moment(); - const date = momentTimestamp.format('DDMMYYYY'); + const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); + const uiCountersUsageCounter = usageCountersServiceSetup.createUsageCounter('uiCounter'); let repository: ReturnType; @@ -64,34 +64,56 @@ describe('store_report', () => { }, }, }; - await storeReport(repository, report); + await storeReport(repository, uiCountersUsageCounter, report); - expect(repository.create).toHaveBeenCalledWith( - 'ui-metric', - { count: 1 }, - { - id: 'key-user-agent:test-user-agent', - overwrite: true, - } - ); - expect(repository.incrementCounter).toHaveBeenNthCalledWith( - 1, - 'ui-metric', - 'test-app-name:test-event-name', - [{ fieldName: 'count', incrementBy: 3 }] - ); - expect(repository.incrementCounter).toHaveBeenNthCalledWith( - 2, - 'ui-counter', - `test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`, - [{ fieldName: 'count', incrementBy: 1 }] - ); - expect(repository.incrementCounter).toHaveBeenNthCalledWith( - 3, - 'ui-counter', - `test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`, - [{ fieldName: 'count', incrementBy: 2 }] - ); + expect(repository.create.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "ui-metric", + Object { + "count": 1, + }, + Object { + "id": "key-user-agent:test-user-agent", + "overwrite": true, + }, + ], + ] + `); + + expect(repository.incrementCounter.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "ui-metric", + "test-app-name:test-event-name", + Array [ + Object { + "fieldName": "count", + "incrementBy": 3, + }, + ], + ], + ] + `); + expect((uiCountersUsageCounter.incrementCounter as jest.Mock).mock.calls) + .toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "counterName": "test-app-name:test-event-name", + "counterType": "loaded", + "incrementBy": 1, + }, + ], + Array [ + Object { + "counterName": "test-app-name:test-event-name", + "counterType": "click", + "incrementBy": 2, + }, + ], + ] + `); expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1); expect(storeApplicationUsageMock).toHaveBeenCalledWith( @@ -108,7 +130,7 @@ describe('store_report', () => { uiCounter: void 0, application_usage: void 0, }; - await storeReport(repository, report); + await storeReport(repository, uiCountersUsageCounter, report); expect(repository.bulkCreate).not.toHaveBeenCalled(); expect(repository.incrementCounter).not.toHaveBeenCalled(); diff --git a/src/plugins/usage_collection/server/report/store_report.ts b/src/plugins/usage_collection/server/report/store_report.ts index 0545a54792d4..1647fb8893be 100644 --- a/src/plugins/usage_collection/server/report/store_report.ts +++ b/src/plugins/usage_collection/server/report/store_report.ts @@ -11,9 +11,12 @@ import moment from 'moment'; import { chain, sumBy } from 'lodash'; import { ReportSchemaType } from './schema'; import { storeApplicationUsage } from './store_application_usage'; +import { UsageCounter } from '../usage_counters'; +import { serializeUiCounterName } from '../../common/ui_counters'; export async function storeReport( internalRepository: ISavedObjectsRepository, + uiCountersUsageCounter: UsageCounter, report: ReportSchemaType ) { const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : []; @@ -21,7 +24,6 @@ export async function storeReport( const appUsages = report.application_usage ? Object.values(report.application_usage) : []; const momentTimestamp = moment(); - const date = momentTimestamp.format('DDMMYYYY'); const timestamp = momentTimestamp.toDate(); return Promise.allSettled([ @@ -55,14 +57,14 @@ export async function storeReport( }) .value(), // UI Counters - ...uiCounters.map(async ([key, metric]) => { + ...uiCounters.map(async ([, metric]) => { const { appName, eventName, total, type } = metric; - const savedObjectId = `${appName}:${date}:${type}:${eventName}`; - return [ - await internalRepository.incrementCounter('ui-counter', savedObjectId, [ - { fieldName: 'count', incrementBy: total }, - ]), - ]; + const counterName = serializeUiCounterName({ appName, eventName }); + uiCountersUsageCounter.incrementCounter({ + counterName, + counterType: type, + incrementBy: total, + }); }), // Application Usage storeApplicationUsage(internalRepository, appUsages, timestamp), diff --git a/src/plugins/usage_collection/server/routes/index.ts b/src/plugins/usage_collection/server/routes/index.ts index 0e17ebcbfd69..20949224c0f6 100644 --- a/src/plugins/usage_collection/server/routes/index.ts +++ b/src/plugins/usage_collection/server/routes/index.ts @@ -16,14 +16,16 @@ import { Observable } from 'rxjs'; import { CollectorSet } from '../collector'; import { registerUiCountersRoute } from './ui_counters'; import { registerStatsRoute } from './stats'; - +import type { UsageCounter } from '../usage_counters'; export function setupRoutes({ router, + uiCountersUsageCounter, getSavedObjects, ...rest }: { router: IRouter; getSavedObjects: () => ISavedObjectsRepository | undefined; + uiCountersUsageCounter: UsageCounter; config: { allowAnonymous: boolean; kibanaIndex: string; @@ -39,6 +41,6 @@ export function setupRoutes({ metrics: MetricsServiceSetup; overallStatus$: Observable; }) { - registerUiCountersRoute(router, getSavedObjects); + registerUiCountersRoute(router, getSavedObjects, uiCountersUsageCounter); registerStatsRoute({ router, ...rest }); } diff --git a/src/plugins/usage_collection/server/routes/ui_counters.ts b/src/plugins/usage_collection/server/routes/ui_counters.ts index 07983ba1d65c..c03541b1032b 100644 --- a/src/plugins/usage_collection/server/routes/ui_counters.ts +++ b/src/plugins/usage_collection/server/routes/ui_counters.ts @@ -9,10 +9,12 @@ import { schema } from '@kbn/config-schema'; import { IRouter, ISavedObjectsRepository } from 'src/core/server'; import { storeReport, reportSchema } from '../report'; +import { UsageCounter } from '../usage_counters'; export function registerUiCountersRoute( router: IRouter, - getSavedObjects: () => ISavedObjectsRepository | undefined + getSavedObjects: () => ISavedObjectsRepository | undefined, + uiCountersUsageCounter: UsageCounter ) { router.post( { @@ -30,7 +32,7 @@ export function registerUiCountersRoute( if (!internalRepository) { throw Error(`The saved objects client hasn't been initialised yet`); } - await storeReport(internalRepository, report); + await storeReport(internalRepository, uiCountersUsageCounter, report); return res.ok({ body: { status: 'ok' } }); } catch (error) { return res.ok({ body: { status: 'fail' } }); diff --git a/src/plugins/usage_collection/server/usage_collection.mock.ts b/src/plugins/usage_collection/server/usage_collection.mock.ts deleted file mode 100644 index 7e3f4273bbea..000000000000 --- a/src/plugins/usage_collection/server/usage_collection.mock.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { - elasticsearchServiceMock, - httpServerMock, - loggingSystemMock, - savedObjectsClientMock, -} from '../../../../src/core/server/mocks'; - -import { CollectorOptions, Collector, UsageCollector } from './collector'; -import { UsageCollectionSetup, CollectorFetchContext } from './index'; - -export type { CollectorOptions }; -export { Collector }; - -const logger = loggingSystemMock.createLogger(); - -export const createUsageCollectionSetupMock = () => { - const usageCollectionSetupMock: jest.Mocked = { - areAllCollectorsReady: jest.fn(), - bulkFetch: jest.fn(), - bulkFetchUsage: jest.fn(), - getCollectorByType: jest.fn(), - toApiFieldNames: jest.fn(), - toObject: jest.fn(), - makeStatsCollector: jest.fn().mockImplementation((cfg) => new Collector(logger, cfg)), - makeUsageCollector: jest.fn().mockImplementation((cfg) => new UsageCollector(logger, cfg)), - registerCollector: jest.fn(), - }; - - usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true); - return usageCollectionSetupMock; -}; - -export function createCollectorFetchContextMock(): jest.Mocked> { - const collectorFetchClientsMock: jest.Mocked> = { - esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, - soClient: savedObjectsClientMock.create(), - }; - return collectorFetchClientsMock; -} - -export function createCollectorFetchContextWithKibanaMock(): jest.Mocked< - CollectorFetchContext -> { - const collectorFetchClientsMock: jest.Mocked> = { - esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, - soClient: savedObjectsClientMock.create(), - kibanaRequest: httpServerMock.createKibanaRequest(), - }; - return collectorFetchClientsMock; -} diff --git a/src/plugins/usage_collection/server/usage_counters/index.ts b/src/plugins/usage_collection/server/usage_counters/index.ts new file mode 100644 index 000000000000..dc1d1f5b43ed --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type { UsageCountersServiceSetup } from './usage_counters_service'; +export type { UsageCountersSavedObjectAttributes, UsageCountersSavedObject } from './saved_objects'; +export type { IncrementCounterParams } from './usage_counter'; + +export { UsageCountersService } from './usage_counters_service'; +export { UsageCounter } from './usage_counter'; +export { USAGE_COUNTERS_SAVED_OBJECT_TYPE, serializeCounterKey } from './saved_objects'; diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts new file mode 100644 index 000000000000..f857d449312e --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { serializeCounterKey, storeCounter } from './saved_objects'; +import { savedObjectsRepositoryMock } from '../../../../core/server/mocks'; +import { CounterMetric } from './usage_counter'; +import moment from 'moment'; + +describe('counterKey', () => { + test('#serializeCounterKey returns a serialized string', () => { + const result = serializeCounterKey({ + domainId: 'a', + counterName: 'b', + counterType: 'c', + date: moment('09042021', 'DDMMYYYY'), + }); + + expect(result).toMatchInlineSnapshot(`"a:09042021:c:b"`); + }); +}); + +describe('storeCounter', () => { + const internalRepository = savedObjectsRepositoryMock.create(); + + const mockNow = 1617954426939; + + beforeEach(() => { + jest.spyOn(moment, 'now').mockReturnValue(mockNow); + }); + + afterAll(() => { + jest.resetAllMocks(); + }); + + it('stores counter in a saved object', async () => { + const counterMetric: CounterMetric = { + domainId: 'a', + counterName: 'b', + counterType: 'c', + incrementBy: 13, + }; + + await storeCounter(counterMetric, internalRepository); + + expect(internalRepository.incrementCounter).toBeCalledTimes(1); + expect(internalRepository.incrementCounter.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "usage-counters", + "a:09042021:c:b", + Array [ + Object { + "fieldName": "count", + "incrementBy": 13, + }, + ], + Object { + "upsertAttributes": Object { + "counterName": "b", + "counterType": "c", + "domainId": "a", + }, + }, + ] + `); + }); +}); diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts new file mode 100644 index 000000000000..6c585d756e8c --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + SavedObject, + SavedObjectsRepository, + SavedObjectAttributes, + SavedObjectsServiceSetup, +} from 'kibana/server'; +import moment from 'moment'; +import { CounterMetric } from './usage_counter'; + +export interface UsageCountersSavedObjectAttributes extends SavedObjectAttributes { + domainId: string; + counterName: string; + counterType: string; + count: number; +} + +export type UsageCountersSavedObject = SavedObject; + +export const USAGE_COUNTERS_SAVED_OBJECT_TYPE = 'usage-counters'; + +export const registerUsageCountersSavedObjectType = ( + savedObjectsSetup: SavedObjectsServiceSetup +) => { + savedObjectsSetup.registerType({ + name: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + hidden: false, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + domainId: { type: 'keyword' }, + }, + }, + }); +}; + +export interface SerializeCounterParams { + domainId: string; + counterName: string; + counterType: string; + date: moment.MomentInput; +} + +export const serializeCounterKey = ({ + domainId, + counterName, + counterType, + date, +}: SerializeCounterParams) => { + const dayDate = moment(date).format('DDMMYYYY'); + return `${domainId}:${dayDate}:${counterType}:${counterName}`; +}; + +export const storeCounter = async ( + counterMetric: CounterMetric, + internalRepository: Pick +) => { + const { counterName, counterType, domainId, incrementBy } = counterMetric; + const key = serializeCounterKey({ + date: moment.now(), + domainId, + counterName, + counterType, + }); + + return await internalRepository.incrementCounter( + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + key, + [{ fieldName: 'count', incrementBy }], + { + upsertAttributes: { + domainId, + counterName, + counterType, + }, + } + ); +}; diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts new file mode 100644 index 000000000000..3602ff1a2937 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { UsageCounter, CounterMetric } from './usage_counter'; +import * as Rx from 'rxjs'; +import * as rxOp from 'rxjs/operators'; + +describe('UsageCounter', () => { + const domainId = 'test-domain-id'; + const counter$ = new Rx.Subject(); + const usageCounter = new UsageCounter({ domainId, counter$ }); + + afterAll(() => { + counter$.complete(); + }); + + describe('#incrementCounter', () => { + it('#incrementCounter calls counter$.next', async () => { + const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise(); + usageCounter.incrementCounter({ counterName: 'test', counterType: 'type', incrementBy: 13 }); + await expect(result).resolves.toEqual([ + { counterName: 'test', counterType: 'type', domainId: 'test-domain-id', incrementBy: 13 }, + ]); + }); + + it('passes default configs to counter$', async () => { + const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise(); + usageCounter.incrementCounter({ counterName: 'test' }); + await expect(result).resolves.toEqual([ + { counterName: 'test', counterType: 'count', domainId: 'test-domain-id', incrementBy: 1 }, + ]); + }); + }); +}); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts new file mode 100644 index 000000000000..af00ad04149b --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Rx from 'rxjs'; + +export interface CounterMetric { + domainId: string; + counterName: string; + counterType: string; + incrementBy: number; +} + +export interface UsageCounterDeps { + domainId: string; + counter$: Rx.Subject; +} + +export interface IncrementCounterParams { + counterName: string; + counterType?: string; + incrementBy?: number; +} + +export class UsageCounter { + private domainId: string; + private counter$: Rx.Subject; + + constructor({ domainId, counter$ }: UsageCounterDeps) { + this.domainId = domainId; + this.counter$ = counter$; + } + + public incrementCounter = (params: IncrementCounterParams) => { + const { counterName, counterType = 'count', incrementBy = 1 } = params; + + this.counter$.next({ + counterName, + domainId: this.domainId, + counterType, + incrementBy, + }); + }; +} diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts new file mode 100644 index 000000000000..beb67d1eb260 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { PublicMethodsOf } from '@kbn/utility-types'; +import type { UsageCountersService, UsageCountersServiceSetup } from './usage_counters_service'; +import type { UsageCounter } from './usage_counter'; + +const createSetupContractMock = () => { + const setupContract: jest.Mocked = { + createUsageCounter: jest.fn(), + getUsageCounterByType: jest.fn(), + }; + + setupContract.createUsageCounter.mockReturnValue(({ + incrementCounter: jest.fn(), + } as unknown) as jest.Mocked); + + return setupContract; +}; + +const createUsageCountersServiceMock = () => { + const mocked: jest.Mocked> = { + setup: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + }; + + mocked.setup.mockReturnValue(createSetupContractMock()); + return mocked; +}; + +export const usageCountersServiceMock = { + create: createUsageCountersServiceMock, + createSetupContract: createSetupContractMock, +}; diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts new file mode 100644 index 000000000000..c800bce6390c --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable dot-notation */ +import { UsageCountersService } from './usage_counters_service'; +import { loggingSystemMock, coreMock } from '../../../../core/server/mocks'; +import * as rxOp from 'rxjs/operators'; +import moment from 'moment'; + +const tick = () => { + jest.useRealTimers(); + return new Promise((resolve) => setTimeout(resolve, 1)); +}; + +describe('UsageCountersService', () => { + const retryCount = 1; + const bufferDurationMs = 100; + const mockNow = 1617954426939; + const logger = loggingSystemMock.createLogger(); + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); + + beforeEach(() => { + jest.spyOn(moment, 'now').mockReturnValue(mockNow); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('stores data in cache during setup', async () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + + const usageCounter = createUsageCounter('test-counter'); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + + const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise(); + usageCountersService['flushCache$'].next(); + usageCountersService['source$'].complete(); + await expect(dataInSourcePromise).resolves.toHaveLength(2); + }); + + it('registers savedObject type during setup', () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); + usageCountersService.setup(coreSetup); + expect(coreSetup.savedObjects.registerType).toBeCalledTimes(1); + }); + + it('flushes cached data on start', async () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockIncrementCounter = jest.fn(); + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + + const usageCounter = createUsageCounter('test-counter'); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + + const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise(); + usageCountersService.start(coreStart); + usageCountersService['source$'].complete(); + + await expect(dataInSourcePromise).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + }, + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + }, + ] + `); + }); + + it('buffers data into savedObject', async () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockIncrementCounter = jest.fn().mockResolvedValue('success'); + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + jest.useFakeTimers('modern'); + const usageCounter = createUsageCounter('test-counter'); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + + usageCountersService.start(coreStart); + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterB' }); + jest.runOnlyPendingTimers(); + expect(mockIncrementCounter).toBeCalledTimes(2); + expect(mockIncrementCounter.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "usage-counters", + "test-counter:09042021:count:counterA", + Array [ + Object { + "fieldName": "count", + "incrementBy": 3, + }, + ], + Object { + "upsertAttributes": Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + }, + }, + ], + Array [ + "usage-counters", + "test-counter:09042021:count:counterB", + Array [ + Object { + "fieldName": "count", + "incrementBy": 1, + }, + ], + Object { + "upsertAttributes": Object { + "counterName": "counterB", + "counterType": "count", + "domainId": "test-counter", + }, + }, + ], + ] + `); + }); + + it('retries errors by `retryCount` times before failing to store', async () => { + const usageCountersService = new UsageCountersService({ + logger, + retryCount: 1, + bufferDurationMs, + }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockError = new Error('failed.'); + const mockIncrementCounter = jest.fn().mockImplementation((_, key) => { + switch (key) { + case 'test-counter:09042021:count:counterA': + throw mockError; + case 'test-counter:09042021:count:counterB': + return 'pass'; + default: + throw new Error(`unknown key ${key}`); + } + }); + + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + jest.useFakeTimers('modern'); + const usageCounter = createUsageCounter('test-counter'); + + usageCountersService.start(coreStart); + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterB' }); + jest.runOnlyPendingTimers(); + + // wait for retries to kick in on next scheduler call + await tick(); + // number of incrementCounter calls + number of retries + expect(mockIncrementCounter).toBeCalledTimes(2 + 1); + expect(logger.debug).toHaveBeenNthCalledWith(1, 'Store counters into savedObjects', [ + mockError, + 'pass', + ]); + }); + + it('buffers counters within `bufferDurationMs` time', async () => { + const usageCountersService = new UsageCountersService({ + logger, + retryCount, + bufferDurationMs: 30000, + }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockIncrementCounter = jest.fn().mockImplementation((_data, key, counter) => { + expect(counter).toHaveLength(1); + return { key, incrementBy: counter[0].incrementBy }; + }); + + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + + const { createUsageCounter } = usageCountersService.setup(coreSetup); + jest.useFakeTimers('modern'); + const usageCounter = createUsageCounter('test-counter'); + + usageCountersService.start(coreStart); + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + jest.advanceTimersByTime(30000); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + jest.runOnlyPendingTimers(); + + // wait for debounce to kick in on next scheduler call + await tick(); + expect(mockIncrementCounter).toBeCalledTimes(2); + expect(mockIncrementCounter.mock.results.map(({ value }) => value)).toMatchInlineSnapshot(` + Array [ + Object { + "incrementBy": 2, + "key": "test-counter:09042021:count:counterA", + }, + Object { + "incrementBy": 1, + "key": "test-counter:09042021:count:counterA", + }, + ] + `); + }); +}); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts new file mode 100644 index 000000000000..88ca9f635892 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Rx from 'rxjs'; +import * as rxOp from 'rxjs/operators'; +import { + SavedObjectsRepository, + SavedObjectsServiceSetup, + SavedObjectsServiceStart, +} from 'src/core/server'; +import type { Logger } from 'src/core/server'; + +import moment from 'moment'; +import { CounterMetric, UsageCounter } from './usage_counter'; +import { + registerUsageCountersSavedObjectType, + storeCounter, + serializeCounterKey, +} from './saved_objects'; + +export interface UsageCountersServiceDeps { + logger: Logger; + retryCount: number; + bufferDurationMs: number; +} + +export interface UsageCountersServiceSetup { + createUsageCounter: (type: string) => UsageCounter; + getUsageCounterByType: (type: string) => UsageCounter | undefined; +} + +/* internal */ +export interface UsageCountersServiceSetupDeps { + savedObjects: SavedObjectsServiceSetup; +} + +/* internal */ +export interface UsageCountersServiceStartDeps { + savedObjects: SavedObjectsServiceStart; +} + +export class UsageCountersService { + private readonly stop$ = new Rx.Subject(); + private readonly retryCount: number; + private readonly bufferDurationMs: number; + + private readonly counterSets = new Map(); + private readonly source$ = new Rx.Subject(); + private readonly counter$ = this.source$.pipe(rxOp.multicast(new Rx.Subject()), rxOp.refCount()); + private readonly flushCache$ = new Rx.Subject(); + + private readonly stopCaching$ = new Rx.Subject(); + + private readonly logger: Logger; + + constructor({ logger, retryCount, bufferDurationMs }: UsageCountersServiceDeps) { + this.logger = logger; + this.retryCount = retryCount; + this.bufferDurationMs = bufferDurationMs; + } + + public setup = (core: UsageCountersServiceSetupDeps): UsageCountersServiceSetup => { + const cache$ = new Rx.ReplaySubject(); + const storingCache$ = new Rx.BehaviorSubject(false); + // flush cache data from cache -> source + this.flushCache$ + .pipe( + rxOp.exhaustMap(() => cache$), + rxOp.takeUntil(this.stop$) + ) + .subscribe((data) => { + storingCache$.next(true); + this.source$.next(data); + }); + + // store data into cache when not paused + storingCache$ + .pipe( + rxOp.distinctUntilChanged(), + rxOp.switchMap((isStoring) => (isStoring ? Rx.EMPTY : this.source$)), + rxOp.takeUntil(Rx.merge(this.stopCaching$, this.stop$)) + ) + .subscribe((data) => { + cache$.next(data); + storingCache$.next(false); + }); + + registerUsageCountersSavedObjectType(core.savedObjects); + + return { + createUsageCounter: this.createUsageCounter, + getUsageCounterByType: this.getUsageCounterByType, + }; + }; + + public start = ({ savedObjects }: UsageCountersServiceStartDeps): void => { + this.stopCaching$.next(); + const internalRepository = savedObjects.createInternalRepository(); + this.counter$ + .pipe( + /* buffer source events every ${bufferDurationMs} */ + rxOp.bufferTime(this.bufferDurationMs), + /** + * bufferTime will trigger every ${bufferDurationMs} + * regardless if source emitted anything or not. + * using filter will stop cut the pipe short + */ + rxOp.filter((counters) => Array.isArray(counters) && counters.length > 0), + rxOp.map((counters) => Object.values(this.mergeCounters(counters))), + rxOp.takeUntil(this.stop$), + rxOp.concatMap((counters) => this.storeDate$(counters, internalRepository)) + ) + .subscribe((results) => { + this.logger.debug('Store counters into savedObjects', results); + }); + + this.flushCache$.next(); + }; + + public stop = () => { + this.stop$.next(); + }; + + private storeDate$( + counters: CounterMetric[], + internalRepository: Pick + ) { + return Rx.forkJoin( + counters.map((counter) => + Rx.defer(() => storeCounter(counter, internalRepository)).pipe( + rxOp.retry(this.retryCount), + rxOp.catchError((error) => { + this.logger.warn(error); + return Rx.of(error); + }) + ) + ) + ); + } + + private createUsageCounter = (type: string): UsageCounter => { + if (this.counterSets.get(type)) { + throw new Error(`Usage counter set "${type}" already exists.`); + } + + const counterSet = new UsageCounter({ + domainId: type, + counter$: this.source$, + }); + + this.counterSets.set(type, counterSet); + + return counterSet; + }; + + private getUsageCounterByType = (type: string): UsageCounter | undefined => { + return this.counterSets.get(type); + }; + + private mergeCounters = (counters: CounterMetric[]): Record => { + const date = moment.now(); + return counters.reduce((acc, counter) => { + const { counterName, domainId, counterType } = counter; + const key = serializeCounterKey({ domainId, counterName, counterType, date }); + const existingCounter = acc[key]; + if (!existingCounter) { + acc[key] = counter; + return acc; + } + return { + ...acc, + [key]: { + ...existingCounter, + ...counter, + incrementBy: existingCounter.incrementBy + counter.incrementBy, + }, + }; + }, {} as Record); + }; +} diff --git a/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts b/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts index b87e6d54733a..e045788897b6 100644 --- a/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts +++ b/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts @@ -10,8 +10,10 @@ jest.mock('./get_stats', () => ({ getStats: jest.fn().mockResolvedValue({ somestat: 1 }), })); -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; -import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; +import { + createUsageCollectionSetupMock, + createCollectorFetchContextMock, +} from 'src/plugins/usage_collection/server/mocks'; import { registerVisTypeTableUsageCollector } from './register_usage_collector'; import { getStats } from './get_stats'; diff --git a/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts b/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts index 2612a3882af2..726ad972ab8d 100644 --- a/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts +++ b/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts @@ -8,7 +8,7 @@ import { of } from 'rxjs'; import { mockStats, mockGetStats } from './get_usage_collector.mock'; -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; +import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { registerTimeseriesUsageCollector } from './register_timeseries_collector'; import { ConfigObservable } from '../types'; diff --git a/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts b/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts index 9db1b7657f44..7933da3e675f 100644 --- a/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts +++ b/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts @@ -8,7 +8,7 @@ import { of } from 'rxjs'; import { mockStats, mockGetStats } from './get_usage_collector.mock'; -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; +import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { HomeServerPluginSetup } from '../../../home/server'; import { registerVegaUsageCollector } from './register_vega_collector'; diff --git a/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts b/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts index 743ec29fe9af..a3617631f734 100644 --- a/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts +++ b/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts @@ -8,7 +8,7 @@ import { of } from 'rxjs'; import { mockStats, mockGetStats } from './get_usage_collector.mock'; -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; +import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { registerVisualizationsCollector } from './register_visualizations_collector'; diff --git a/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts b/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts index 762b91791820..07a11f3876d8 100644 --- a/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts +++ b/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts @@ -8,6 +8,14 @@ export const basicUiCounters = { dailyEvents: [ + { + appName: 'myApp', + eventName: 'some_app_event', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + counterType: 'count', + total: 2, + }, { appName: 'myApp', eventName: 'my_event_885082425109579', diff --git a/test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts b/test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts new file mode 100644 index 000000000000..988bc2e77528 --- /dev/null +++ b/test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const basicUsageCounters = { + dailyEvents: [ + { + domainId: 'anotherDomainId', + counterName: 'some_event_name', + counterType: 'count', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + total: 3, + }, + { + domainId: 'anotherDomainId', + counterName: 'some_event_name', + counterType: 'count', + lastUpdatedAt: '2021-04-09T11:43:00.961Z', + fromTimestamp: '2021-04-09T00:00:00Z', + total: 2, + }, + { + domainId: 'anotherDomainId2', + counterName: 'some_event_name', + counterType: 'count', + lastUpdatedAt: '2021-04-20T08:18:03.030Z', + fromTimestamp: '2021-04-20T00:00:00Z', + total: 1, + }, + ], +}; diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index 88bbc5a08c39..36c9660064a4 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -9,6 +9,7 @@ import expect from '@kbn/expect'; import supertestAsPromised from 'supertest-as-promised'; import { basicUiCounters } from './__fixtures__/ui_counters'; +import { basicUsageCounters } from './__fixtures__/usage_counters'; import type { FtrProviderContext } from '../../ftr_provider_context'; import type { SavedObject } from '../../../../src/core/server'; import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json'; @@ -153,6 +154,20 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('Usage Counters telemetry', () => { + before('Add UI Counters saved objects', () => + esArchiver.load('saved_objects/usage_counters') + ); + after('cleanup saved objects changes', () => + esArchiver.unload('saved_objects/usage_counters') + ); + + it('returns usage counters aggregated by day', async () => { + const stats = await retrieveTelemetry(supertest); + expect(stats.stack_stats.kibana.plugins.usage_counters).to.eql(basicUsageCounters); + }); + }); + describe('application usage limits', () => { function createSavedObject(viewId?: string) { return supertest diff --git a/test/api_integration/apis/ui_counters/ui_counters.ts b/test/api_integration/apis/ui_counters/ui_counters.ts index 2d55e224f31c..aa201eb6a96f 100644 --- a/test/api_integration/apis/ui_counters/ui_counters.ts +++ b/test/api_integration/apis/ui_counters/ui_counters.ts @@ -7,11 +7,10 @@ */ import expect from '@kbn/expect'; -import { ReportManager, METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; +import { ReportManager, METRIC_TYPE, UiCounterMetricType, Report } from '@kbn/analytics'; import moment from 'moment'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { SavedObject } from '../../../../src/core/server'; -import { UICounterSavedObjectAttributes } from '../../../../src/plugins/kibana_usage_collection/server/collectors/ui_counters/ui_counter_saved_object_type'; +import { UsageCountersSavedObject } from '../../../../src/plugins/usage_collection/server'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -24,10 +23,22 @@ export default function ({ getService }: FtrProviderContext) { count, }); + const sendReport = async (report: Report) => { + await supertest + .post('/api/ui_counters/_report') + .set('kbn-xsrf', 'kibana') + .set('content-type', 'application/json') + .send({ report }) + .expect(200); + + // wait for SO to index data into ES + await new Promise((res) => setTimeout(res, 5 * 1000)); + }; + const getCounterById = ( - savedObjects: Array>, + savedObjects: UsageCountersSavedObject[], targetId: string - ): SavedObject => { + ): UsageCountersSavedObject => { const savedObject = savedObjects.find(({ id }: { id: string }) => id === targetId); if (!savedObject) { throw new Error(`Unable to find savedObject id ${targetId}`); @@ -40,30 +51,25 @@ export default function ({ getService }: FtrProviderContext) { const dayDate = moment().format('DDMMYYYY'); before(async () => await esArchiver.emptyKibanaIndex()); - it('stores ui counter events in savedObjects', async () => { + it('stores ui counter events in usage counters savedObjects', async () => { const reportManager = new ReportManager(); const { report } = reportManager.assignReports([ createUiCounterEvent('my_event', METRIC_TYPE.COUNT), ]); - await supertest - .post('/api/ui_counters/_report') - .set('kbn-xsrf', 'kibana') - .set('content-type', 'application/json') - .send({ report }) - .expect(200); + await sendReport(report); const { body: { saved_objects: savedObjects }, } = await supertest - .get('/api/saved_objects/_find?type=ui-counter') + .get('/api/saved_objects/_find?type=usage-counters') .set('kbn-xsrf', 'kibana') .expect(200); const countTypeEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.COUNT}:my_event` + `uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:my_event` ); expect(countTypeEvent.attributes.count).to.eql(1); }); @@ -78,35 +84,31 @@ export default function ({ getService }: FtrProviderContext) { createUiCounterEvent(`${uniqueEventName}_2`, METRIC_TYPE.COUNT), createUiCounterEvent(uniqueEventName, METRIC_TYPE.CLICK, 2), ]); - await supertest - .post('/api/ui_counters/_report') - .set('kbn-xsrf', 'kibana') - .set('content-type', 'application/json') - .send({ report }) - .expect(200); + + await sendReport(report); const { body: { saved_objects: savedObjects }, } = await supertest - .get('/api/saved_objects/_find?type=ui-counter&fields=count') + .get('/api/saved_objects/_find?type=usage-counters&fields=count') .set('kbn-xsrf', 'kibana') .expect(200); const countTypeEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.COUNT}:${uniqueEventName}` + `uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:${uniqueEventName}` ); expect(countTypeEvent.attributes.count).to.eql(1); const clickTypeEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.CLICK}:${uniqueEventName}` + `uiCounter:${dayDate}:${METRIC_TYPE.CLICK}:myApp:${uniqueEventName}` ); expect(clickTypeEvent.attributes.count).to.eql(2); const secondEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.COUNT}:${uniqueEventName}_2` + `uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:${uniqueEventName}_2` ); expect(secondEvent.attributes.count).to.eql(1); }); diff --git a/test/api_integration/config.js b/test/api_integration/config.js index 1c19dd24fa96..7bbace4c6057 100644 --- a/test/api_integration/config.js +++ b/test/api_integration/config.js @@ -31,6 +31,8 @@ export default async function ({ readConfigFile }) { '--server.xsrf.disableProtection=true', '--server.compression.referrerWhitelist=["some-host.com"]', `--savedObjects.maxImportExportSize=10001`, + // for testing set buffer duration to 0 to immediately flush counters into saved objects. + '--usageCollection.usageCounters.bufferDuration=0', ], }, }; diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json new file mode 100644 index 000000000000..80071fe42278 --- /dev/null +++ b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json @@ -0,0 +1,111 @@ +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:loaded:my_event_885082425109579", + "source": { + "ui-counter": { + "count": 1 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:count:my_event_885082425109579_2", + "source": { + "ui-counter": { + "count": 1 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:count:my_event_885082425109579_2", + "source": { + "ui-counter": { + "count": 1 + }, + "type": "ui-counter", + "updated_at": "2020-10-28T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:click:my_event_885082425109579", + "source": { + "ui-counter": { + "count": 2 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:click:my_event_885082425109579", + "source": { + "ui-counter": { + "count": 2 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "uiCounter:09042021:count:myApp:some_app_event", + "source": { + "usage-counters": { + "count": 2, + "domainId": "uiCounter", + "counterName": "myApp:some_app_event", + "counterType": "count" + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId:09042021:count:some_event_name", + "source": { + "usage-counters": { + "count": 2, + "domainId": "anotherDomainId", + "counterName": "some_event_name", + "counterType": "count" + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz deleted file mode 100644 index 3f42c777260b3bb8c9892f0b4e7c1ed0f18292ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 236 zcmVQOZ*BnXld%qhFc5}!o`Q6yXaMqJu++`}+TP?Von^e4lhfqX_qj)CCC~=tX558Es+9vX<)Z1mUGT zi(1Sg$EAa&q=hzhr&@j;4o$-&KxDvxS6WCVEzMQ0>Ml>y1X32W1R+cI+0y2wOfof+Hf2BMuN|J3NtDK6!3Uo;Pk8 m%#1(glCys@znBbAmVPmrsw^%W{3W*ei+KQ7tJo%F1ONd3YHSDq diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json index 926fd5d79faa..39902f8a9211 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json @@ -35,6 +35,15 @@ } } }, + "usage-counters": { + "dynamic": false, + "properties": { + "domainId": { + "type": "keyword", + "ignore_above": 256 + } + } + }, "dashboard": { "properties": { "description": { diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json new file mode 100644 index 000000000000..16e0364b24fd --- /dev/null +++ b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json @@ -0,0 +1,89 @@ +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "uiCounter:20112020:count:myApp:some_app_event", + "source": { + "usage-counters": { + "count": 2, + "domainId": "uiCounter", + "counterName": "myApp:some_app_event", + "counterType": "count" + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId:20112020:count:some_event_name", + "source": { + "usage-counters": { + "count": 3, + "domainId": "anotherDomainId", + "counterName": "some_event_name", + "counterType": "count" + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId:09042021:count:some_event_name", + "source": { + "usage-counters": { + "count": 2, + "domainId": "anotherDomainId", + "counterName": "some_event_name", + "counterType": "count" + }, + "type": "usage-counters", + "updated_at": "2021-04-09T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId2:09042021:count:some_event_name", + "source": { + "usage-counters": { + "count": 1, + "domainId": "anotherDomainId2", + "counterName": "some_event_name", + "counterType": "count" + }, + "type": "usage-counters", + "updated_at": "2021-04-20T08:18:03.030Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId3:09042021:custom_type:zero_count", + "source": { + "usage-counters": { + "count": 0, + "domainId": "anotherDomainId3", + "counterName": "zero_count", + "counterType": "custom_type" + }, + "type": "usage-counters", + "updated_at": "2021-04-20T08:18:03.030Z" + } + } +} diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json new file mode 100644 index 000000000000..14ed147b2da8 --- /dev/null +++ b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json @@ -0,0 +1,276 @@ +{ + "type": "index", + "value": { + "index": ".kibana", + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + }, + "mappings": { + "dynamic": "strict", + "properties": { + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + }, + "defaultIndex": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "usage-counters": { + "dynamic": false, + "properties": { + "domainId": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "namespace": { + "type": "keyword" + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 2048 + } + } + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchId": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + } + } +} diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index 1651e213ee82..d21a157975ac 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -21,6 +21,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { return { testFiles: [ + require.resolve('./test_suites/usage_collection'), require.resolve('./test_suites/core'), require.resolve('./test_suites/custom_visualizations'), require.resolve('./test_suites/panel_actions'), @@ -59,6 +60,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--corePluginDeprecations.oldProperty=hello', '--corePluginDeprecations.secret=100', '--corePluginDeprecations.noLongerUsed=still_using', + // for testing set buffer duration to 0 to immediately flush counters into saved objects. + '--usageCollection.usageCounters.bufferDuration=0', ...plugins.map( (pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}` ), diff --git a/test/plugin_functional/plugins/usage_collection/kibana.json b/test/plugin_functional/plugins/usage_collection/kibana.json new file mode 100644 index 000000000000..c98e3b95d389 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/kibana.json @@ -0,0 +1,9 @@ +{ + "id": "usageCollectionTestPlugin", + "version": "0.0.1", + "kibanaVersion": "kibana", + "configPath": ["usageCollectionTestPlugin"], + "requiredPlugins": ["usageCollection"], + "server": true, + "ui": false +} diff --git a/test/plugin_functional/plugins/usage_collection/package.json b/test/plugin_functional/plugins/usage_collection/package.json new file mode 100644 index 000000000000..33289bd8d727 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/package.json @@ -0,0 +1,14 @@ +{ + "name": "usage_collection_test_plugin", + "version": "1.0.0", + "main": "target/test/plugin_functional/plugins/usage_collection", + "kibana": { + "version": "kibana", + "templateVersion": "1.0.0" + }, + "license": "SSPL-1.0 OR Elastic License 2.0", + "scripts": { + "kbn": "node ../../../../scripts/kbn.js", + "build": "rm -rf './target' && ../../../../node_modules/.bin/tsc" + } +} diff --git a/test/plugin_functional/plugins/usage_collection/server/index.ts b/test/plugin_functional/plugins/usage_collection/server/index.ts new file mode 100644 index 000000000000..172f8491a1a4 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { UsageCollectionTestPlugin } from './plugin'; +export const plugin = () => new UsageCollectionTestPlugin(); diff --git a/test/plugin_functional/plugins/usage_collection/server/plugin.ts b/test/plugin_functional/plugins/usage_collection/server/plugin.ts new file mode 100644 index 000000000000..523fbcfe058d --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/plugin.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Plugin, CoreSetup } from 'kibana/server'; +import { + UsageCollectionSetup, + UsageCounter, +} from '../../../../../src/plugins/usage_collection/server'; +import { registerRoutes } from './routes'; + +export interface TestPluginDepsSetup { + usageCollection: UsageCollectionSetup; +} + +export class UsageCollectionTestPlugin implements Plugin { + private usageCounter?: UsageCounter; + + public setup(core: CoreSetup, { usageCollection }: TestPluginDepsSetup) { + const usageCounter = usageCollection.createUsageCounter('usageCollectionTestPlugin'); + + registerRoutes(core.http, usageCounter); + usageCounter.incrementCounter({ + counterName: 'duringSetup', + incrementBy: 10, + }); + usageCounter.incrementCounter({ counterName: 'duringSetup' }); + this.usageCounter = usageCounter; + } + + public start() { + if (!this.usageCounter) { + throw new Error('this.usageCounter is expected to be defined during setup.'); + } + this.usageCounter.incrementCounter({ counterName: 'duringStart' }); + } + + public stop() {} +} diff --git a/test/plugin_functional/plugins/usage_collection/server/routes.ts b/test/plugin_functional/plugins/usage_collection/server/routes.ts new file mode 100644 index 000000000000..e67e45451277 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/routes.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { HttpServiceSetup } from 'kibana/server'; +import { UsageCounter } from '../../../../../src/plugins/usage_collection/server'; + +export function registerRoutes(http: HttpServiceSetup, usageCounter: UsageCounter) { + const router = http.createRouter(); + router.get( + { + path: '/api/usage_collection_test_plugin', + validate: false, + }, + async (context, req, res) => { + usageCounter.incrementCounter({ counterName: 'routeAccessed' }); + return res.ok(); + } + ); +} diff --git a/test/plugin_functional/plugins/usage_collection/tsconfig.json b/test/plugin_functional/plugins/usage_collection/tsconfig.json new file mode 100644 index 000000000000..3d9d8ca9451d --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./target", + "skipLibCheck": true + }, + "include": [ + "index.ts", + "public/**/*.ts", + "public/**/*.tsx", + "server/**/*.ts", + "../../../../typings/**/*", + ], + "exclude": [], + "references": [ + { "path": "../../../../src/core/tsconfig.json" } + ] +} diff --git a/test/plugin_functional/test_suites/usage_collection/index.ts b/test/plugin_functional/test_suites/usage_collection/index.ts new file mode 100644 index 000000000000..201b7b04ff22 --- /dev/null +++ b/test/plugin_functional/test_suites/usage_collection/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PluginFunctionalProviderContext } from '../../services'; + +export default function ({ loadTestFile }: PluginFunctionalProviderContext) { + describe('usage collection', function () { + loadTestFile(require.resolve('./usage_counters')); + }); +} diff --git a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts new file mode 100644 index 000000000000..f1591165b8d6 --- /dev/null +++ b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { PluginFunctionalProviderContext } from '../../services'; +import { + UsageCountersSavedObject, + serializeCounterKey, +} from '../../../../src/plugins/usage_collection/server/usage_counters'; + +export default function ({ getService, getPageObjects }: PluginFunctionalProviderContext) { + const supertest = getService('supertest'); + + async function getSavedObjectCounters() { + // wait until ES indexes the counter SavedObject; + await new Promise((res) => setTimeout(res, 7 * 1000)); + + return await supertest + .get('/api/saved_objects/_find?type=usage-counters') + .set('kbn-xsrf', 'true') + .expect(200) + .then(({ body }) => { + expect(body.total).to.above(1); + return (body.saved_objects as UsageCountersSavedObject[]).reduce((acc, savedObj) => { + const { count, counterName, domainId } = savedObj.attributes; + if (domainId === 'usageCollectionTestPlugin') { + acc[counterName] = count; + } + + return acc; + }, {} as Record); + }); + } + + describe('Usage Counters service', () => { + before(async () => { + const key = serializeCounterKey({ + counterName: 'routeAccessed', + counterType: 'count', + domainId: 'usageCollectionTestPlugin', + date: Date.now(), + }); + + await supertest.delete(`/api/saved_objects/usage-counters/${key}`).set('kbn-xsrf', 'true'); + }); + + it('stores usage counters sent during start and setup', async () => { + const { duringSetup, duringStart, routeAccessed } = await getSavedObjectCounters(); + + expect(duringSetup).to.be(11); + expect(duringStart).to.be(1); + expect(routeAccessed).to.be(undefined); + }); + + it('stores usage counters triggered by runtime activities', async () => { + await supertest.get('/api/usage_collection_test_plugin').set('kbn-xsrf', 'true').expect(200); + + const { routeAccessed } = await getSavedObjectCounters(); + expect(routeAccessed).to.be(1); + }); + }); +}