maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Usage collection] Usage counters (#96696) (#97103)

Browse source 
Co-authored-by: Alejandro Fernández Haro <afharo@gmail.com>

Co-authored-by: Ahmad Bamieh <ahmadbamieh@gmail.com>
Co-authored-by: Alejandro Fernández Haro <afharo@gmail.com>
This commit is contained in:
Kibana Machine 2021-04-14 10:23:45 -04:00 committed by GitHub
parent 3f5e0d7c1b
commit 630d4bbfcf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 3120 additions and 318 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md
View file

@ -8,7 +8,7 @@
```typescript
start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): {
indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "update" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "update" | "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
};
```
@ -22,6 +22,6 @@ start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps):
<b>Returns:</b>
`{
indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "update" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "update" | "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
}`

1
src/plugins/data/server/server.api.md
View file

@ -56,6 +56,7 @@ import { PublicMethodsOf } from '@kbn/utility-types';
import { RecursiveReadonly } from '@kbn/utility-types';
import { RequestAdapter } from 'src/plugins/inspector/common';
import { RequestHandlerContext } from 'src/core/server';
import * as Rx from 'rxjs';
import { SavedObject } from 'kibana/server';
import { SavedObject as SavedObject_2 } from 'src/core/server';
import { SavedObjectsClientContract } from 'src/core/server';

21
src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap generated
View file

@ -1,21 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`kibana_usage_collection Runs the setup method without issues 1`] = `true`;
exports[`kibana_usage_collection Runs the setup method without issues 2`] = `false`;
exports[`kibana_usage_collection Runs the setup method without issues 3`] = `true`;
exports[`kibana_usage_collection Runs the setup method without issues 4`] = `false`;
exports[`kibana_usage_collection Runs the setup method without issues 5`] = `false`;
exports[`kibana_usage_collection Runs the setup method without issues 6`] = `false`;
exports[`kibana_usage_collection Runs the setup method without issues 7`] = `false`;
exports[`kibana_usage_collection Runs the setup method without issues 8`] = `true`;
exports[`kibana_usage_collection Runs the setup method without issues 9`] = `false`;
exports[`kibana_usage_collection Runs the setup method without issues 10`] = `true`;

2
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
View file

@ -10,7 +10,7 @@ import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../co
import {
Collector,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import {

14
src/plugins/kibana_usage_collection/server/collectors/cloud/cloud_provider_collector.test.ts
View file

@ -12,25 +12,23 @@ import {
Collector,
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import { registerCloudProviderUsageCollector } from './cloud_provider_collector';
describe('registerCloudProviderUsageCollector', () => {
let collector: Collector<unknown>;
const logger = loggingSystemMock.createLogger();
const usageCollectionMock = createUsageCollectionSetupMock();
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = new Collector(logger, config);
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
const mockedFetchContext = createCollectorFetchContextMock();
beforeEach(() => {
cloudDetailsMock.mockClear();
detectCloudServiceMock.mockClear();
const usageCollectionMock = createUsageCollectionSetupMock();
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = new Collector(logger, config);
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
registerCloudProviderUsageCollector(usageCollectionMock);
});

2
src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts
View file

@ -9,7 +9,7 @@
import {
Collector,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import { registerCoreUsageCollector } from '.';
import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks';

4
src/plugins/kibana_usage_collection/server/collectors/index.ts
View file

@ -20,3 +20,7 @@ export {
registerUiCounterSavedObjectType,
registerUiCountersRollups,
} from './ui_counters';
export {
registerUsageCountersRollups,
registerUsageCountersUsageCollector,
} from './usage_counters';

2
src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts
View file

@ -15,7 +15,7 @@ import {
Collector,
createCollectorFetchContextMock,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import { registerKibanaUsageCollector } from './';
const logger = loggingSystemMock.createLogger();

2
src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts
View file

@ -11,7 +11,7 @@ import {
Collector,
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import {
registerManagementUsageCollector,

2
src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts
View file

@ -11,7 +11,7 @@ import {
Collector,
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import { registerOpsStatsCollector } from './';
import { OpsMetrics } from '../../../../../core/server';

51
src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { UICounterSavedObject } from '../ui_counter_saved_object_type';
export const rawUiCounters: UICounterSavedObject[] = [
{
type: 'ui-counter',
id: 'Kibana_home:23102020:click:different_type',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-11-24T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:25102020:loaded:intersecting_event',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-10-25T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:23102020:loaded:intersecting_event',
attributes: {
count: 3,
},
references: [],
updated_at: '2020-10-23T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:24112020:click:only_reported_in_ui_counters',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-11-24T11:27:57.067Z',
version: 'WzI5NDRd',
},
];

104
src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts Normal file
View file

@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { UsageCountersSavedObject } from '../../../../../usage_collection/server';
export const rawUsageCounters: UsageCountersSavedObject[] = [
{
type: 'usage-counters',
id: 'uiCounter:09042021:count:myApp:my_event',
attributes: {
count: 1,
counterName: 'myApp:my_event',
counterType: 'count',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:17:57.693Z',
},
{
type: 'usage-counters',
id: 'uiCounter:23102020:loaded:Kibana_home:intersecting_event',
attributes: {
count: 60,
counterName: 'Kibana_home:intersecting_event',
counterType: 'loaded',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2020-10-23T11:27:57.067Z',
},
{
type: 'usage-counters',
id: 'uiCounter:09042021:count:myApp:my_event_4457914848544',
attributes: {
count: 0,
counterName: 'myApp:my_event_4457914848544',
counterType: 'count',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'uiCounter:09042021:count:myApp:my_event_malformed',
attributes: {
// @ts-expect-error
count: 'malformed',
counterName: 'myApp:my_event_malformed',
counterType: 'count',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId:09042021:count:some_event_name',
attributes: {
count: 4,
counterName: 'some_event_name',
counterType: 'count',
domainId: 'anotherDomainId',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'uiCounter:09042021:count:myApp:my_event_4457914848544_2',
attributes: {
count: 8,
counterName: 'myApp:my_event_4457914848544_2',
counterType: 'count',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.031Z',
},
{
type: 'usage-counters',
id: 'uiCounter:09042021:count:myApp:only_reported_in_usage_counters',
attributes: {
count: 1,
counterName: 'myApp:only_reported_in_usage_counters',
counterType: 'count',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.031Z',
},
];

266
src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts
View file

@ -6,70 +6,208 @@
* Side Public License, v 1.
*/
import { transformRawCounter } from './register_ui_counters_collector';
import { UICounterSavedObject } from './ui_counter_saved_object_type';
import {
transformRawUiCounterObject,
transformRawUsageCounterObject,
createFetchUiCounters,
} from './register_ui_counters_collector';
import { BehaviorSubject } from 'rxjs';
import { rawUiCounters } from './__fixtures__/ui_counter_saved_objects';
import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects';
import { savedObjectsClientMock } from '../../../../../core/server/mocks';
import { UI_COUNTER_SAVED_OBJECT_TYPE } from './ui_counter_saved_object_type';
import { USAGE_COUNTERS_SAVED_OBJECT_TYPE } from '../../../../usage_collection/server';
describe('transformRawCounter', () => {
const mockRawUiCounters = [
{
type: 'ui-counter',
id: 'Kibana_home:24112020:click:ingest_data_card_home_tutorial_directory',
attributes: {
count: 3,
},
references: [],
updated_at: '2020-11-24T11:27:57.067Z',
version: 'WzI5LDFd',
},
{
type: 'ui-counter',
id: 'Kibana_home:24112020:click:home_tutorial_directory',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-11-24T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:24112020:loaded:home_tutorial_directory',
attributes: {
count: 3,
},
references: [],
updated_at: '2020-10-23T11:27:57.067Z',
version: 'WzI5NDRd',
},
] as UICounterSavedObject[];
it('transforms saved object raw entries', () => {
const result = mockRawUiCounters.map(transformRawCounter);
expect(result).toEqual([
{
appName: 'Kibana_home',
eventName: 'ingest_data_card_home_tutorial_directory',
lastUpdatedAt: '2020-11-24T11:27:57.067Z',
fromTimestamp: '2020-11-24T00:00:00Z',
counterType: 'click',
total: 3,
},
{
appName: 'Kibana_home',
eventName: 'home_tutorial_directory',
lastUpdatedAt: '2020-11-24T11:27:57.067Z',
fromTimestamp: '2020-11-24T00:00:00Z',
counterType: 'click',
total: 1,
},
{
appName: 'Kibana_home',
eventName: 'home_tutorial_directory',
lastUpdatedAt: '2020-10-23T11:27:57.067Z',
fromTimestamp: '2020-10-23T00:00:00Z',
counterType: 'loaded',
total: 3,
},
]);
describe('transformRawUsageCounterObject', () => {
it('transforms usage counters savedObject raw entries', () => {
const result = rawUsageCounters.map(transformRawUsageCounterObject);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"appName": "myApp",
"counterType": "count",
"eventName": "my_event",
"fromTimestamp": "2021-04-09T00:00:00Z",
"lastUpdatedAt": "2021-04-09T08:17:57.693Z",
"total": 1,
},
Object {
"appName": "Kibana_home",
"counterType": "loaded",
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-23T00:00:00Z",
"lastUpdatedAt": "2020-10-23T11:27:57.067Z",
"total": 60,
},
undefined,
undefined,
undefined,
Object {
"appName": "myApp",
"counterType": "count",
"eventName": "my_event_4457914848544_2",
"fromTimestamp": "2021-04-09T00:00:00Z",
"lastUpdatedAt": "2021-04-09T08:18:03.031Z",
"total": 8,
},
Object {
"appName": "myApp",
"counterType": "count",
"eventName": "only_reported_in_usage_counters",
"fromTimestamp": "2021-04-09T00:00:00Z",
"lastUpdatedAt": "2021-04-09T08:18:03.031Z",
"total": 1,
},
]
`);
});
});
describe('transformRawUiCounterObject', () => {
it('transforms ui counters savedObject raw entries', () => {
const result = rawUiCounters.map(transformRawUiCounterObject);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"appName": "Kibana_home",
"counterType": "click",
"eventName": "different_type",
"fromTimestamp": "2020-11-24T00:00:00Z",
"lastUpdatedAt": "2020-11-24T11:27:57.067Z",
"total": 1,
},
Object {
"appName": "Kibana_home",
"counterType": "loaded",
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-25T00:00:00Z",
"lastUpdatedAt": "2020-10-25T11:27:57.067Z",
"total": 1,
},
Object {
"appName": "Kibana_home",
"counterType": "loaded",
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-23T00:00:00Z",
"lastUpdatedAt": "2020-10-23T11:27:57.067Z",
"total": 3,
},
Object {
"appName": "Kibana_home",
"counterType": "click",
"eventName": "only_reported_in_ui_counters",
"fromTimestamp": "2020-11-24T00:00:00Z",
"lastUpdatedAt": "2020-11-24T11:27:57.067Z",
"total": 1,
},
]
`);
});
});
describe('createFetchUiCounters', () => {
let stopUsingUiCounterIndicies$: BehaviorSubject<boolean>;
const soClientMock = savedObjectsClientMock.create();
beforeEach(() => {
jest.clearAllMocks();
stopUsingUiCounterIndicies$ = new BehaviorSubject<boolean>(false);
});
it('does not query ui_counters saved objects if stopUsingUiCounterIndicies$ is complete', async () => {
// @ts-expect-error incomplete mock implementation
soClientMock.find.mockImplementation(async ({ type }) => {
switch (type) {
case USAGE_COUNTERS_SAVED_OBJECT_TYPE:
return { saved_objects: rawUsageCounters };
default:
throw new Error(`unexpected type ${type}`);
}
});
stopUsingUiCounterIndicies$.complete();
// @ts-expect-error incomplete mock implementation
const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({
soClient: soClientMock,
});
const transforemdUsageCounters = rawUsageCounters.map(transformRawUsageCounterObject);
expect(soClientMock.find).toBeCalledTimes(1);
expect(dailyEvents).toEqual(transforemdUsageCounters.filter(Boolean));
});
it('merges saved objects from both ui_counters and usage_counters saved objects', async () => {
// @ts-expect-error incomplete mock implementation
soClientMock.find.mockImplementation(async ({ type }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: rawUiCounters };
case USAGE_COUNTERS_SAVED_OBJECT_TYPE:
return { saved_objects: rawUsageCounters };
default:
throw new Error(`unexpected type ${type}`);
}
});
// @ts-expect-error incomplete mock implementation
const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({
soClient: soClientMock,
});
expect(dailyEvents).toHaveLength(7);
const intersectingEntry = dailyEvents.find(
({ eventName, fromTimestamp }) =>
eventName === 'intersecting_event' && fromTimestamp === '2020-10-23T00:00:00Z'
);
const onlyFromUICountersEntry = dailyEvents.find(
({ eventName }) => eventName === 'only_reported_in_ui_counters'
);
const onlyFromUsageCountersEntry = dailyEvents.find(
({ eventName }) => eventName === 'only_reported_in_usage_counters'
);
const invalidCountEntry = dailyEvents.find(
({ eventName }) => eventName === 'my_event_malformed'
);
const zeroCountEntry = dailyEvents.find(
({ eventName }) => eventName === 'my_event_4457914848544'
);
const nonUiCountersEntry = dailyEvents.find(({ eventName }) => eventName === 'some_event_name');
expect(invalidCountEntry).toBe(undefined);
expect(nonUiCountersEntry).toBe(undefined);
expect(zeroCountEntry).toBe(undefined);
expect(onlyFromUICountersEntry).toMatchInlineSnapshot(`
Object {
"appName": "Kibana_home",
"counterType": "click",
"eventName": "only_reported_in_ui_counters",
"fromTimestamp": "2020-11-24T00:00:00Z",
"lastUpdatedAt": "2020-11-24T11:27:57.067Z",
"total": 1,
}
`);
expect(onlyFromUsageCountersEntry).toMatchInlineSnapshot(`
Object {
"appName": "myApp",
"counterType": "count",
"eventName": "only_reported_in_usage_counters",
"fromTimestamp": "2021-04-09T00:00:00Z",
"lastUpdatedAt": "2021-04-09T08:18:03.031Z",
"total": 1,
}
`);
expect(intersectingEntry).toMatchInlineSnapshot(`
Object {
"appName": "Kibana_home",
"counterType": "loaded",
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-23T00:00:00Z",
"lastUpdatedAt": "2020-10-23T11:27:57.067Z",
"total": 63,
}
`);
});
});

156
src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts
View file

@ -7,13 +7,28 @@
*/
import moment from 'moment';
import { CollectorFetchContext, UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { mergeWith } from 'lodash';
import type { Subject } from 'rxjs';
import {
UICounterSavedObject,
UICounterSavedObjectAttributes,
UI_COUNTER_SAVED_OBJECT_TYPE,
} from './ui_counter_saved_object_type';
import {
CollectorFetchContext,
UsageCollectionSetup,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
UsageCountersSavedObject,
UsageCountersSavedObjectAttributes,
serializeCounterKey,
} from '../../../../usage_collection/server';
import {
deserializeUiCounterName,
serializeUiCounterName,
} from '../../../../usage_collection/common/ui_counters';
interface UiCounterEvent {
appName: string;
eventName: string;
@ -27,12 +42,20 @@ export interface UiCountersUsage {
dailyEvents: UiCounterEvent[];
}
export function transformRawCounter(rawUiCounter: UICounterSavedObject) {
const { id, attributes, updated_at: lastUpdatedAt } = rawUiCounter;
export function transformRawUiCounterObject(
rawUiCounter: UICounterSavedObject
): UiCounterEvent | undefined {
const {
id,
attributes: { count },
updated_at: lastUpdatedAt,
} = rawUiCounter;
if (typeof count !== 'number' || count < 1) {
return;
}
const [appName, , counterType, ...restId] = id.split(':');
const eventName = restId.join(':');
const counterTotal: unknown = attributes.count;
const total = typeof counterTotal === 'number' ? counterTotal : 0;
const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format();
return {
@ -41,11 +64,110 @@ export function transformRawCounter(rawUiCounter: UICounterSavedObject) {
lastUpdatedAt,
fromTimestamp,
counterType,
total,
total: count,
};
}
export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) {
export function transformRawUsageCounterObject(
rawUsageCounter: UsageCountersSavedObject
): UiCounterEvent | undefined {
const {
attributes: { count, counterName, counterType, domainId },
updated_at: lastUpdatedAt,
} = rawUsageCounter;
if (domainId !== 'uiCounter' || typeof count !== 'number' || count < 1) {
return;
}
const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format();
const { appName, eventName } = deserializeUiCounterName(counterName);
return {
appName,
eventName,
lastUpdatedAt,
fromTimestamp,
counterType,
total: count,
};
}
export const createFetchUiCounters = (stopUsingUiCounterIndicies$: Subject<void>) =>
async function fetchUiCounters({ soClient }: CollectorFetchContext) {
const {
saved_objects: rawUsageCounters,
} = await soClient.find<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 10000,
});
const skipFetchingUiCounters = stopUsingUiCounterIndicies$.isStopped;
const result =
skipFetchingUiCounters ||
(await soClient.find<UICounterSavedObjectAttributes>({
type: UI_COUNTER_SAVED_OBJECT_TYPE,
fields: ['count'],
perPage: 10000,
}));
const rawUiCounters = typeof result === 'object' ? result.saved_objects : [];
const dailyEventsFromUiCounters = rawUiCounters.reduce((acc, raw) => {
try {
const event = transformRawUiCounterObject(raw);
if (event) {
const { appName, eventName, counterType } = event;
const key = serializeCounterKey({
domainId: 'uiCounter',
counterName: serializeUiCounterName({ appName, eventName }),
counterType,
date: event.lastUpdatedAt,
});
acc[key] = event;
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, {} as Record<string, UiCounterEvent>);
const dailyEventsFromUsageCounters = rawUsageCounters.reduce((acc, raw) => {
try {
const event = transformRawUsageCounterObject(raw);
if (event) {
acc[raw.id] = event;
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, {} as Record<string, UiCounterEvent>);
const mergedDailyCounters = mergeWith(
dailyEventsFromUsageCounters,
dailyEventsFromUiCounters,
(value: UiCounterEvent | undefined, srcValue: UiCounterEvent): UiCounterEvent => {
if (!value) {
return srcValue;
}
return {
...srcValue,
total: srcValue.total + value.total,
};
}
);
return { dailyEvents: Object.values(mergedDailyCounters) };
};
export function registerUiCountersUsageCollector(
usageCollection: UsageCollectionSetup,
stopUsingUiCounterIndicies$: Subject<void>
) {
const collector = usageCollection.makeUsageCollector<UiCountersUsage>({
type: 'ui_counters',
schema: {
@ -76,25 +198,7 @@ export function registerUiCountersUsageCollector(usageCollection: UsageCollectio
},
},
},
fetch: async ({ soClient }: CollectorFetchContext) => {
const { saved_objects: rawUiCounters } = await soClient.find<UICounterSavedObjectAttributes>({
type: UI_COUNTER_SAVED_OBJECT_TYPE,
fields: ['count'],
perPage: 10000,
});
return {
dailyEvents: rawUiCounters.reduce((acc, raw) => {
try {
const aggEvent = transformRawCounter(raw);
acc.push(aggEvent);
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, [] as UiCounterEvent[]),
};
},
fetch: createFetchUiCounters(stopUsingUiCounterIndicies$),
isReady: () => true,
});

12
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts
View file

@ -6,16 +6,20 @@
* Side Public License, v 1.
*/
import { timer } from 'rxjs';
import { Subject, timer } from 'rxjs';
import { takeUntil } from 'rxjs/operators';
import { Logger, ISavedObjectsRepository } from 'kibana/server';
import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import { rollUiCounterIndices } from './rollups';
export function registerUiCountersRollups(
logger: Logger,
stopRollingUiCounterIndicies$: Subject<void>,
getSavedObjectsClient: () => ISavedObjectsRepository | undefined
) {
timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL).subscribe(() =>
rollUiCounterIndices(logger, getSavedObjectsClient())
);
timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL)
.pipe(takeUntil(stopRollingUiCounterIndicies$))
.subscribe(() =>
rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, getSavedObjectsClient())
);
}

38
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts
View file

@ -7,9 +7,11 @@
*/
import moment from 'moment';
import * as Rx from 'rxjs';
import { isSavedObjectOlderThan, rollUiCounterIndices } from './rollups';
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
import { SavedObjectsFindResult } from 'kibana/server';
import {
UICounterSavedObjectAttributes,
UI_COUNTER_SAVED_OBJECT_TYPE,
@ -70,14 +72,18 @@ describe('isSavedObjectOlderThan', () => {
describe('rollUiCounterIndices', () => {
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
let savedObjectClient: ReturnType<typeof savedObjectsRepositoryMock.create>;
let stopUsingUiCounterIndicies$: Rx.Subject<void>;
beforeEach(() => {
logger = loggingSystemMock.createLogger();
savedObjectClient = savedObjectsRepositoryMock.create();
stopUsingUiCounterIndicies$ = new Rx.Subject();
});
it('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollUiCounterIndices(logger, undefined)).resolves.toBe(undefined);
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, undefined)
).resolves.toBe(undefined);
expect(logger.warn).toHaveBeenCalledTimes(0);
});
@ -90,11 +96,27 @@ describe('rollUiCounterIndices', () => {
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual([]);
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual([]);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it('calls Subject complete() on empty saved objects', async () => {
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual([]);
expect(stopUsingUiCounterIndicies$.isStopped).toBe(true);
});
it(`deletes documents older than ${UI_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => {
const mockSavedObjects = [
@ -111,7 +133,9 @@ describe('rollUiCounterIndices', () => {
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toHaveLength(2);
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toHaveLength(2);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).toHaveBeenCalledTimes(2);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
@ -131,7 +155,9 @@ describe('rollUiCounterIndices', () => {
savedObjectClient.find.mockImplementation(async () => {
throw new Error(`Expected error!`);
});
await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual(undefined);
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual(undefined);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
expect(logger.warn).toHaveBeenCalledTimes(2);
@ -151,7 +177,9 @@ describe('rollUiCounterIndices', () => {
savedObjectClient.delete.mockImplementation(async () => {
throw new Error(`Expected error!`);
});
await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual(undefined);
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual(undefined);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(

16
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts
View file

@ -8,6 +8,7 @@
import { ISavedObjectsRepository, Logger } from 'kibana/server';
import moment from 'moment';
import type { Subject } from 'rxjs';
import { UI_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants';
import {
@ -38,6 +39,7 @@ export function isSavedObjectOlderThan({
export async function rollUiCounterIndices(
logger: Logger,
stopUsingUiCounterIndicies$: Subject<void>,
savedObjectsClient?: ISavedObjectsRepository
) {
if (!savedObjectsClient) {
@ -54,6 +56,20 @@ export async function rollUiCounterIndices(
}
);
if (rawUiCounterDocs.length === 0) {
/**
* @deprecated 7.13 to be removed in 8.0.0
* Stop triggering rollups when we've rolled up all documents.
*
* This Saved Object registry is no longer used.
* Migration from one SO registry to another is not yet supported.
* In a future release we can remove this piece of code and
* migrate any docs to the Usage Counters Saved object.
*/
stopUsingUiCounterIndicies$.complete();
}
const docsToDelete = rawUiCounterDocs.filter((doc) =>
isSavedObjectOlderThan({
numberOfDays: UI_COUNTERS_KEEP_DOCS_FOR_DAYS,

2
src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts
View file

@ -11,7 +11,7 @@ import {
Collector,
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from '../../../../usage_collection/server/usage_collection.mock';
} from '../../../../usage_collection/server/mocks';
import { registerUiMetricUsageCollector } from './';

104
src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts Normal file
View file

@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { UsageCountersSavedObject } from '../../../../../usage_collection/server';
export const rawUsageCounters: UsageCountersSavedObject[] = [
{
type: 'usage-counters',
id: 'uiCounter:09042021:count:myApp:my_event',
attributes: {
count: 13,
counterName: 'my_event',
counterType: 'count',
domainId: 'uiCounter',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId:09042021:count:some_event_name',
attributes: {
count: 4,
counterName: 'some_event_name',
counterType: 'count',
domainId: 'anotherDomainId',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-09T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId:09042021:count:some_event_name',
attributes: {
count: 4,
counterName: 'some_event_name',
counterType: 'count',
domainId: 'anotherDomainId',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-11T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId2:09042021:count:some_event_name',
attributes: {
count: 1,
counterName: 'some_event_name',
counterType: 'count',
domainId: 'anotherDomainId2',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-20T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId2:09042021:count:malformed_event',
attributes: {
// @ts-expect-error
count: 'malformed',
counterName: 'malformed_event',
counterType: 'count',
domainId: 'anotherDomainId2',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-20T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId2:09042021:custom_type:some_event_name',
attributes: {
count: 3,
counterName: 'some_event_name',
counterType: 'custom_type',
domainId: 'anotherDomainId2',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-20T08:18:03.030Z',
},
{
type: 'usage-counters',
id: 'anotherDomainId3:09042021:custom_type:zero_count',
attributes: {
count: 0,
counterName: 'zero_count',
counterType: 'custom_type',
domainId: 'anotherDomainId3',
},
references: [],
coreMigrationVersion: '8.0.0',
updated_at: '2021-04-20T08:18:03.030Z',
},
];

10
src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export { registerUsageCountersUsageCollector } from './register_usage_counters_collector';
export { registerUsageCountersRollups } from './rollups';

55
src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { transformRawCounter } from './register_usage_counters_collector';
import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects';
describe('transformRawCounter', () => {
it('transforms saved object raw entries', () => {
const result = rawUsageCounters.map(transformRawCounter);
expect(result).toMatchInlineSnapshot(`
Array [
undefined,
Object {
"counterName": "some_event_name",
"counterType": "count",
"domainId": "anotherDomainId",
"fromTimestamp": "2021-04-09T00:00:00Z",
"lastUpdatedAt": "2021-04-09T08:18:03.030Z",
"total": 4,
},
Object {
"counterName": "some_event_name",
"counterType": "count",
"domainId": "anotherDomainId",
"fromTimestamp": "2021-04-11T00:00:00Z",
"lastUpdatedAt": "2021-04-11T08:18:03.030Z",
"total": 4,
},
Object {
"counterName": "some_event_name",
"counterType": "count",
"domainId": "anotherDomainId2",
"fromTimestamp": "2021-04-20T00:00:00Z",
"lastUpdatedAt": "2021-04-20T08:18:03.030Z",
"total": 1,
},
undefined,
Object {
"counterName": "some_event_name",
"counterType": "custom_type",
"domainId": "anotherDomainId2",
"fromTimestamp": "2021-04-20T00:00:00Z",
"lastUpdatedAt": "2021-04-20T08:18:03.030Z",
"total": 3,
},
undefined,
]
`);
});
});

116
src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts Normal file
View file

@ -0,0 +1,116 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import {
CollectorFetchContext,
UsageCollectionSetup,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
UsageCountersSavedObject,
UsageCountersSavedObjectAttributes,
} from '../../../../usage_collection/server';
interface UsageCounterEvent {
domainId: string;
counterName: string;
counterType: string;
lastUpdatedAt?: string;
fromTimestamp?: string;
total: number;
}
export interface UiCountersUsage {
dailyEvents: UsageCounterEvent[];
}
export function transformRawCounter(
rawUsageCounter: UsageCountersSavedObject
): UsageCounterEvent | undefined {
const {
attributes: { count, counterName, counterType, domainId },
updated_at: lastUpdatedAt,
} = rawUsageCounter;
const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format();
if (domainId === 'uiCounter' || typeof count !== 'number' || count < 1) {
return;
}
return {
domainId,
counterName,
counterType,
lastUpdatedAt,
fromTimestamp,
total: count,
};
}
export function registerUsageCountersUsageCollector(usageCollection: UsageCollectionSetup) {
const collector = usageCollection.makeUsageCollector<UiCountersUsage>({
type: 'usage_counters',
schema: {
dailyEvents: {
type: 'array',
items: {
domainId: {
type: 'keyword',
_meta: { description: 'Domain name of the metric (ie plugin name).' },
},
counterName: {
type: 'keyword',
_meta: { description: 'Name of the counter that happened.' },
},
lastUpdatedAt: {
type: 'date',
_meta: { description: 'Time at which the metric was last updated.' },
},
fromTimestamp: {
type: 'date',
_meta: { description: 'Time at which the metric was captured.' },
},
counterType: {
type: 'keyword',
_meta: { description: 'The type of counter used.' },
},
total: {
type: 'integer',
_meta: { description: 'The total number of times the event happened.' },
},
},
},
},
fetch: async ({ soClient }: CollectorFetchContext) => {
const {
saved_objects: rawUsageCounters,
} = await soClient.find<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `NOT ${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 10000,
});
return {
dailyEvents: rawUsageCounters.reduce((acc, rawUsageCounter) => {
try {
const event = transformRawCounter(rawUsageCounter);
if (event) {
acc.push(event);
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, [] as UsageCounterEvent[]),
};
},
isReady: () => true,
});
usageCollection.registerCollector(collector);
}

22
src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/**
* Roll indices every 24h
*/
export const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up
*/
export const ROLL_INDICES_START = 5 * 60 * 1000;
/**
* Number of days to keep the Usage counters saved object documents
*/
export const USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS = 5;

9
src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export { registerUsageCountersRollups } from './register_rollups';

21
src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { timer } from 'rxjs';
import { Logger, ISavedObjectsRepository } from 'kibana/server';
import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import { rollUsageCountersIndices } from './rollups';
export function registerUsageCountersRollups(
logger: Logger,
getSavedObjectsClient: () => ISavedObjectsRepository | undefined
) {
timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL).subscribe(() =>
rollUsageCountersIndices(logger, getSavedObjectsClient())
);
}

170
src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts Normal file
View file

@ -0,0 +1,170 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import { isSavedObjectOlderThan, rollUsageCountersIndices } from './rollups';
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
import { SavedObjectsFindResult } from '../../../../../../core/server';
import {
UsageCountersSavedObjectAttributes,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
} from '../../../../../usage_collection/server';
import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants';
const createMockSavedObjectDoc = (updatedAt: moment.Moment, id: string) =>
({
id,
type: 'usage-counter',
attributes: {
count: 3,
counterName: 'testName',
counterType: 'count',
domainId: 'testDomain',
},
references: [],
updated_at: updatedAt.format(),
version: 'WzI5LDFd',
score: 0,
} as SavedObjectsFindResult<UsageCountersSavedObjectAttributes>);
describe('isSavedObjectOlderThan', () => {
it(`returns true if doc is older than x days`, () => {
const numberOfDays = 1;
const startDate = moment().format();
const doc = createMockSavedObjectDoc(moment().subtract(2, 'days'), 'some-id');
const result = isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
});
expect(result).toBe(true);
});
it(`returns false if doc is exactly x days old`, () => {
const numberOfDays = 1;
const startDate = moment().format();
const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id');
const result = isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
});
expect(result).toBe(false);
});
it(`returns false if doc is younger than x days`, () => {
const numberOfDays = 2;
const startDate = moment().format();
const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id');
const result = isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
});
expect(result).toBe(false);
});
});
describe('rollUsageCountersIndices', () => {
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
let savedObjectClient: ReturnType<typeof savedObjectsRepositoryMock.create>;
beforeEach(() => {
logger = loggingSystemMock.createLogger();
savedObjectClient = savedObjectsRepositoryMock.create();
});
it('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollUsageCountersIndices(logger, undefined)).resolves.toBe(undefined);
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it('does not delete any documents on empty saved objects', async () => {
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case USAGE_COUNTERS_SAVED_OBJECT_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual([]);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it(`deletes documents older than ${USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => {
const mockSavedObjects = [
createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1'),
createMockSavedObjectDoc(moment().subtract(9, 'days'), 'doc-id-1'),
createMockSavedObjectDoc(moment().subtract(1, 'days'), 'doc-id-2'),
createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3'),
];
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case USAGE_COUNTERS_SAVED_OBJECT_TYPE:
return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toHaveLength(2);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).toHaveBeenCalledTimes(2);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
1,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
'doc-id-1'
);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
2,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
'doc-id-3'
);
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it(`logs warnings on savedObject.find failure`, async () => {
savedObjectClient.find.mockImplementation(async () => {
throw new Error(`Expected error!`);
});
await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual(undefined);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
expect(logger.warn).toHaveBeenCalledTimes(2);
});
it(`logs warnings on savedObject.delete failure`, async () => {
const mockSavedObjects = [createMockSavedObjectDoc(moment().subtract(7, 'days'), 'doc-id-1')];
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case USAGE_COUNTERS_SAVED_OBJECT_TYPE:
return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
savedObjectClient.delete.mockImplementation(async () => {
throw new Error(`Expected error!`);
});
await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual(undefined);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
1,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
'doc-id-1'
);
expect(logger.warn).toHaveBeenCalledTimes(2);
});
});

73
src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts Normal file
View file

@ -0,0 +1,73 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ISavedObjectsRepository, Logger } from 'kibana/server';
import moment from 'moment';
import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants';
import {
UsageCountersSavedObject,
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
} from '../../../../../usage_collection/server';
export function isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
}: {
numberOfDays: number;
startDate: moment.Moment | string | number;
doc: Pick<UsageCountersSavedObject, 'updated_at'>;
}): boolean {
const { updated_at: updatedAt } = doc;
const today = moment(startDate).startOf('day');
const updateDay = moment(updatedAt).startOf('day');
const diffInDays = today.diff(updateDay, 'days');
if (diffInDays > numberOfDays) {
return true;
}
return false;
}
export async function rollUsageCountersIndices(
logger: Logger,
savedObjectsClient?: ISavedObjectsRepository
) {
if (!savedObjectsClient) {
return;
}
const now = moment();
try {
const {
saved_objects: rawUiCounterDocs,
} = await savedObjectsClient.find<UsageCountersSavedObject>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
perPage: 1000, // Process 1000 at a time as a compromise of speed and overload
});
const docsToDelete = rawUiCounterDocs.filter((doc) =>
isSavedObjectOlderThan({
numberOfDays: USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS,
startDate: now,
doc,
})
);
return await Promise.all(
docsToDelete.map(({ id }) => savedObjectsClient.delete(USAGE_COUNTERS_SAVED_OBJECT_TYPE, id))
);
} catch (err) {
logger.warn(`Failed to rollup Usage Counters saved objects.`);
logger.warn(err);
}
}

0
src/plugins/kibana_usage_collection/server/index.test.mocks.ts → src/plugins/kibana_usage_collection/server/mocks.ts
View file

66
src/plugins/kibana_usage_collection/server/index.test.ts → src/plugins/kibana_usage_collection/server/plugin.test.ts
View file

@ -14,8 +14,8 @@ import {
import {
CollectorOptions,
createUsageCollectionSetupMock,
} from '../../usage_collection/server/usage_collection.mock';
import { cloudDetailsMock } from './index.test.mocks';
} from '../../usage_collection/server/mocks';
import { cloudDetailsMock } from './mocks';
import { plugin } from './';
@ -38,13 +38,67 @@ describe('kibana_usage_collection', () => {
cloudDetailsMock.mockClear();
});
test('Runs the setup method without issues', () => {
test('Runs the setup method without issues', async () => {
const coreSetup = coreMock.createSetup();
expect(pluginInstance.setup(coreSetup, { usageCollection })).toBe(undefined);
usageCollectors.forEach(({ isReady }) => {
expect(isReady()).toMatchSnapshot(); // Some should return false at this stage
});
await expect(
Promise.all(
usageCollectors.map(async (usageCollector) => {
const isReady = await usageCollector.isReady();
const type = usageCollector.type;
return { type, isReady };
})
)
).resolves.toMatchInlineSnapshot(`
Array [
Object {
"isReady": true,
"type": "ui_counters",
},
Object {
"isReady": true,
"type": "usage_counters",
},
Object {
"isReady": false,
"type": "kibana_stats",
},
Object {
"isReady": true,
"type": "kibana",
},
Object {
"isReady": false,
"type": "stack_management",
},
Object {
"isReady": false,
"type": "ui_metric",
},
Object {
"isReady": false,
"type": "application_usage",
},
Object {
"isReady": false,
"type": "cloud_provider",
},
Object {
"isReady": true,
"type": "csp",
},
Object {
"isReady": false,
"type": "core",
},
Object {
"isReady": true,
"type": "localization",
},
]
`);
});
test('Runs the start method without issues', () => {

20
src/plugins/kibana_usage_collection/server/plugin.ts
View file

@ -35,6 +35,8 @@ import {
registerUiCountersUsageCollector,
registerUiCounterSavedObjectType,
registerUiCountersRollups,
registerUsageCountersRollups,
registerUsageCountersUsageCollector,
} from './collectors';
interface KibanaUsageCollectionPluginsDepsSetup {
@ -50,18 +52,23 @@ export class KibanaUsageCollectionPlugin implements Plugin {
private uiSettingsClient?: IUiSettingsClient;
private metric$: Subject<OpsMetrics>;
private coreUsageData?: CoreUsageDataStart;
private stopUsingUiCounterIndicies$: Subject<void>;
constructor(initializerContext: PluginInitializerContext) {
this.logger = initializerContext.logger.get();
this.legacyConfig$ = initializerContext.config.legacy.globalConfig$;
this.metric$ = new Subject<OpsMetrics>();
this.stopUsingUiCounterIndicies$ = new Subject();
}
public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) {
usageCollection.createUsageCounter('uiCounters');
this.registerUsageCollectors(
usageCollection,
coreSetup,
this.metric$,
this.stopUsingUiCounterIndicies$,
coreSetup.savedObjects.registerType.bind(coreSetup.savedObjects)
);
}
@ -77,12 +84,14 @@ export class KibanaUsageCollectionPlugin implements Plugin {
public stop() {
this.metric$.complete();
this.stopUsingUiCounterIndicies$.complete();
}
private registerUsageCollectors(
usageCollection: UsageCollectionSetup,
coreSetup: CoreSetup,
metric$: Subject<OpsMetrics>,
stopUsingUiCounterIndicies$: Subject<void>,
registerType: SavedObjectsRegisterType
) {
const getSavedObjectsClient = () => this.savedObjectsClient;
@ -90,8 +99,15 @@ export class KibanaUsageCollectionPlugin implements Plugin {
const getCoreUsageDataService = () => this.coreUsageData!;
registerUiCounterSavedObjectType(coreSetup.savedObjects);
registerUiCountersRollups(this.logger.get('ui-counters'), getSavedObjectsClient);
registerUiCountersUsageCollector(usageCollection);
registerUiCountersRollups(
this.logger.get('ui-counters'),
stopUsingUiCounterIndicies$,
getSavedObjectsClient
);
registerUiCountersUsageCollector(usageCollection, stopUsingUiCounterIndicies$);
registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient);
registerUsageCountersUsageCollector(usageCollection);
registerOpsStatsCollector(usageCollection, metric$);
registerKibanaUsageCollector(usageCollection, this.legacyConfig$);

47
src/plugins/telemetry/schema/oss_plugins.json
View file

@ -9314,6 +9314,53 @@
}
}
},
"usage_counters": {
"properties": {
"dailyEvents": {
"type": "array",
"items": {
"properties": {
"domainId": {
"type": "keyword",
"_meta": {
"description": "Domain name of the metric (ie plugin name)."
}
},
"counterName": {
"type": "keyword",
"_meta": {
"description": "Name of the counter that happened."
}
},
"lastUpdatedAt": {
"type": "date",
"_meta": {
"description": "Time at which the metric was last updated."
}
},
"fromTimestamp": {
"type": "date",
"_meta": {
"description": "Time at which the metric was captured."
}
},
"counterType": {
"type": "keyword",
"_meta": {
"description": "The type of counter used."
}
},
"total": {
"type": "integer",
"_meta": {
"description": "The total number of times the event happened."
}
}
}
}
}
}
},
"telemetry": {
"properties": {
"opt_in_status": {

95
src/plugins/usage_collection/README.mdx
View file

@ -20,6 +20,7 @@ The way to report the usage of any feature depends on whether the actions to tra
In any case, to use any of these APIs, the plugin must optionally require the plugin `usageCollection`:
```json
// plugin/kibana.json
{
@ -112,6 +113,100 @@ Not an API as such. However, Data Telemetry collects the usage of known patterns
This collector does not report the name of the indices nor any content. It only provides stats about usage of known shippers/ingest tools.
#### Usage Counters
Usage counters allows plugins to report user triggered events from the server. This api has feature parity with UI Counters on the `public` plugin side of usage_collection.
Usage counters provide instrumentation on the server to count triggered events such as "api called", "threshold reached", and miscellaneous events count.
It is useful for gathering _semi-aggregated_ events with a per day granularity.
This allows tracking trends in usage and provides enough granularity for this type of telemetry to provide insights such as
- "How many times this threshold has been reached?"
- "What is the trend in usage of this api?"
- "How frequent are users hitting this error per day?"
- "What is the success rate of this operation?"
- "Which option is being selected the most/least?"
##### How to use it
To create a usage counter for your plugin, use the API `usageCollection.createUsageCounter` as follows:
```ts
// server/plugin.ts
import type { Plugin, CoreStart } from '../../../core/server';
import type { UsageCollectionSetup, UsageCounter } from '../../../plugins/usage_collection/server';
export class MyPlugin implements Plugin {
private usageCounter?: UsageCounter;
public setup(
core: CoreStart,
{ usageCollection }: { usageCollection?: UsageCollectionSetup }
) {
/**
* Create a usage counter for this plugin. Domain ID must be unique.
* It is advised to use the plugin name as the domain ID for most cases.
*/
this.usageCounter = usageCollection?.createUsageCounter('<Domain ID>');
try {
doSomeOperation();
this.usageCounter?.incrementCounter({
counterName: 'doSomeOperation_success',
incrementBy: 1,
});
} catch (err) {
this.usageCounter?.incrementCounter({
counterName: 'doSomeOperation_error',
counterType: 'error',
incrementBy: 1,
});
logger.error(err);
}
}
}
```
Pass the created `usageCounter` around in your service to instrument usage.
That's all you need to do! The Usage counters service will handle piping these counters all the way to the telemetry service.
##### Telemetry reported usage
Usage counters are reported inside the telemetry usage payload under `stack_stats.kibana.plugins.usage_counters`.
```ts
{
usage_counters: {
dailyEvents: [
{
domainId: '<Domain ID>',
counterName: 'doSomeOperation_success',
counterType: 'count',
lastUpdatedAt: '2021-11-20T11:43:00.961Z',
fromTimestamp: '2021-11-20T00:00:00Z',
total: 3,
},
{
domainId: '<Domain ID>',
counterName: 'doSomeOperation_success',
counterType: 'count',
lastUpdatedAt: '2021-11-21T10:30:00.961Z',
fromTimestamp: '2021-11-21T00:00:00Z',
total: 5,
},
{
domainId: '<Domain ID>',
counterName: 'doSomeOperation_error',
counterType: 'error',
lastUpdatedAt: '2021-11-20T11:43:00.961Z',
fromTimestamp: '2021-11-20T00:00:00Z',
total: 1,
},
],
},
}
```
#### Custom collector
In many cases, plugins need to report the custom usage of a feature. In this cases, the plugins must complete the following 2 steps in the `setup` lifecycle step:

23
src/plugins/usage_collection/common/ui_counters.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export const serializeUiCounterName = ({
appName,
eventName,
}: {
appName: string;
eventName: string;
}) => {
return `${appName}:${eventName}`;
};
export const deserializeUiCounterName = (key: string) => {
const [appName, ...restKey] = key.split(':');
const eventName = restKey.join(':');
return { appName, eventName };
};

32
src/plugins/usage_collection/server/collector/collector_set.ts
View file

@ -25,22 +25,6 @@ interface CollectorSetConfig {
collectors?: AnyCollector[];
}
/**
* Public interface of the CollectorSet (makes it easier to mock only the public methods)
*/
export type CollectorSetPublic = Pick<
CollectorSet,
| 'makeStatsCollector'
| 'makeUsageCollector'
| 'registerCollector'
| 'getCollectorByType'
| 'areAllCollectorsReady'
| 'bulkFetch'
| 'bulkFetchUsage'
| 'toObject'
| 'toApiFieldNames'
>;
export class CollectorSet {
private _waitingForAllCollectorsTimestamp?: number;
private readonly logger: Logger;
@ -215,19 +199,19 @@ export class CollectorSet {
* Convert an array of fetched stats results into key/object
* @param statsData Array of fetched stats results
*/
public toObject<Result extends Record<string, unknown>, T = unknown>(
public toObject = <Result extends Record<string, unknown>, T = unknown>(
statsData: Array<{ type: string; result: T }> = []
): Result {
): Result => {
return Object.fromEntries(statsData.map(({ type, result }) => [type, result])) as Result;
}
};
/**
* Rename fields to use API conventions
* @param apiData Data to be normalized
*/
public toApiFieldNames(
public toApiFieldNames = (
apiData: Record<string, unknown> | unknown[]
): Record<string, unknown> | unknown[] {
): Record<string, unknown> | unknown[] => {
// handle array and return early, or return a reduced object
if (Array.isArray(apiData)) {
return apiData.map((value) => this.getValueOrRecurse(value));
@ -244,14 +228,14 @@ export class CollectorSet {
return [newName, this.getValueOrRecurse(value)];
})
);
}
};
private getValueOrRecurse(value: unknown) {
private getValueOrRecurse = (value: unknown) => {
if (Array.isArray(value) || (typeof value === 'object' && value !== null)) {
return this.toApiFieldNames(value as Record<string, unknown> | unknown[]); // recurse
}
return value;
}
};
private makeCollectorSetFromArray = (collectors: AnyCollector[]) => {
return new CollectorSet({

1
src/plugins/usage_collection/server/collector/index.ts
View file

@ -7,7 +7,6 @@
*/
export { CollectorSet } from './collector_set';
export type { CollectorSetPublic } from './collector_set';
export { Collector } from './collector';
export type {
AllowedSchemaTypes,

5
src/plugins/usage_collection/server/config.ts
View file

@ -11,6 +11,11 @@ import { PluginConfigDescriptor } from 'src/core/server';
import { DEFAULT_MAXIMUM_WAIT_TIME_FOR_ALL_COLLECTORS_IN_S } from '../common/constants';
export const configSchema = schema.object({
usageCounters: schema.object({
enabled: schema.boolean({ defaultValue: true }),
retryCount: schema.number({ defaultValue: 1 }),
bufferDuration: schema.duration({ defaultValue: '5s' }),
}),
uiCounters: schema.object({
enabled: schema.boolean({ defaultValue: true }),
debug: schema.boolean({ defaultValue: schema.contextRef('dev') }),

13
src/plugins/usage_collection/server/index.ts
View file

@ -18,6 +18,19 @@ export type {
UsageCollectorOptions,
CollectorFetchContext,
} from './collector';
export type {
UsageCountersSavedObject,
UsageCountersSavedObjectAttributes,
IncrementCounterParams,
} from './usage_counters';
export {
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
serializeCounterKey,
UsageCounter,
} from './usage_counters';
export type { UsageCollectionSetup } from './plugin';
export { config } from './config';
export const plugin = (initializerContext: PluginInitializerContext) =>

65
src/plugins/usage_collection/server/mocks.ts
View file

@ -6,20 +6,61 @@
* Side Public License, v 1.
*/
import { loggingSystemMock } from '../../../core/server/mocks';
import { UsageCollectionSetup } from './plugin';
import { CollectorSet } from './collector';
export { Collector, createCollectorFetchContextMock } from './usage_collection.mock';
import {
elasticsearchServiceMock,
httpServerMock,
loggingSystemMock,
savedObjectsClientMock,
} from '../../../../src/core/server/mocks';
const createSetupContract = () => {
return {
...new CollectorSet({
logger: loggingSystemMock.createLogger(),
maximumWaitTimeForAllCollectorsInS: 1,
}),
} as UsageCollectionSetup;
import { CollectorOptions, Collector, CollectorSet } from './collector';
import { UsageCollectionSetup, CollectorFetchContext } from './index';
export type { CollectorOptions };
export { Collector };
export const createUsageCollectionSetupMock = () => {
const collectorSet = new CollectorSet({
logger: loggingSystemMock.createLogger(),
maximumWaitTimeForAllCollectorsInS: 1,
});
const usageCollectionSetupMock: jest.Mocked<UsageCollectionSetup> = {
createUsageCounter: jest.fn(),
getUsageCounterByType: jest.fn(),
areAllCollectorsReady: jest.fn().mockImplementation(collectorSet.areAllCollectorsReady),
bulkFetch: jest.fn().mockImplementation(collectorSet.bulkFetch),
getCollectorByType: jest.fn().mockImplementation(collectorSet.getCollectorByType),
toApiFieldNames: jest.fn().mockImplementation(collectorSet.toApiFieldNames),
toObject: jest.fn().mockImplementation(collectorSet.toObject),
makeStatsCollector: jest.fn().mockImplementation(collectorSet.makeStatsCollector),
makeUsageCollector: jest.fn().mockImplementation(collectorSet.makeUsageCollector),
registerCollector: jest.fn().mockImplementation(collectorSet.registerCollector),
};
usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true);
return usageCollectionSetupMock;
};
export function createCollectorFetchContextMock(): jest.Mocked<CollectorFetchContext<false>> {
const collectorFetchClientsMock: jest.Mocked<CollectorFetchContext<false>> = {
esClient: elasticsearchServiceMock.createClusterClient().asInternalUser,
soClient: savedObjectsClientMock.create(),
};
return collectorFetchClientsMock;
}
export function createCollectorFetchContextWithKibanaMock(): jest.Mocked<
CollectorFetchContext<true>
> {
const collectorFetchClientsMock: jest.Mocked<CollectorFetchContext<true>> = {
esClient: elasticsearchServiceMock.createClusterClient().asInternalUser,
soClient: savedObjectsClientMock.create(),
kibanaRequest: httpServerMock.createKibanaRequest(),
};
return collectorFetchClientsMock;
}
export const usageCollectionPluginMock = {
createSetupContract,
createSetupContract: createUsageCollectionSetupMock,
};

85
src/plugins/usage_collection/server/plugin.ts
View file

@ -15,30 +15,78 @@ import {
Plugin,
} from 'src/core/server';
import { ConfigType } from './config';
import { CollectorSet, CollectorSetPublic } from './collector';
import { CollectorSet } from './collector';
import { setupRoutes } from './routes';
export type UsageCollectionSetup = CollectorSetPublic;
export class UsageCollectionPlugin implements Plugin<CollectorSet> {
import { UsageCountersService } from './usage_counters';
import type { UsageCountersServiceSetup } from './usage_counters';
export interface UsageCollectionSetup {
/**
* Creates and registers a usage counter to collect daily aggregated plugin counter events
*/
createUsageCounter: UsageCountersServiceSetup['createUsageCounter'];
/**
* Returns a usage counter by type
*/
getUsageCounterByType: UsageCountersServiceSetup['getUsageCounterByType'];
/**
* Creates a usage collector to collect plugin telemetry data.
* registerCollector must be called to connect the created collecter with the service.
*/
makeUsageCollector: CollectorSet['makeUsageCollector'];
/**
* Register a usage collector or a stats collector.
* Used to connect the created collector to telemetry.
*/
registerCollector: CollectorSet['registerCollector'];
/**
* Returns a usage collector by type
*/
getCollectorByType: CollectorSet['getCollectorByType'];
/* internal: telemetry use */
areAllCollectorsReady: CollectorSet['areAllCollectorsReady'];
/* internal: telemetry use */
bulkFetch: CollectorSet['bulkFetch'];
/* internal: telemetry use */
toObject: CollectorSet['toObject'];
/* internal: monitoring use */
toApiFieldNames: CollectorSet['toApiFieldNames'];
/* internal: telemtery and monitoring use */
makeStatsCollector: CollectorSet['makeStatsCollector'];
}
export class UsageCollectionPlugin implements Plugin<UsageCollectionSetup> {
private readonly logger: Logger;
private savedObjects?: ISavedObjectsRepository;
private usageCountersService?: UsageCountersService;
constructor(private readonly initializerContext: PluginInitializerContext) {
this.logger = this.initializerContext.logger.get();
}
public setup(core: CoreSetup) {
public setup(core: CoreSetup): UsageCollectionSetup {
const config = this.initializerContext.config.get<ConfigType>();
const collectorSet = new CollectorSet({
logger: this.logger.get('collector-set'),
logger: this.logger.get('usage-collection', 'collector-set'),
maximumWaitTimeForAllCollectorsInS: config.maximumWaitTimeForAllCollectorsInS,
});
const globalConfig = this.initializerContext.config.legacy.get();
this.usageCountersService = new UsageCountersService({
logger: this.logger.get('usage-collection', 'usage-counters-service'),
retryCount: config.usageCounters.retryCount,
bufferDurationMs: config.usageCounters.bufferDuration.asMilliseconds(),
});
const { createUsageCounter, getUsageCounterByType } = this.usageCountersService.setup(core);
const uiCountersUsageCounter = createUsageCounter('uiCounter');
const globalConfig = this.initializerContext.config.legacy.get();
const router = core.http.createRouter();
setupRoutes({
router,
uiCountersUsageCounter,
getSavedObjects: () => this.savedObjects,
collectorSet,
config: {
@ -52,15 +100,38 @@ export class UsageCollectionPlugin implements Plugin<CollectorSet> {
overallStatus$: core.status.overall$,
});
return collectorSet;
return {
areAllCollectorsReady: collectorSet.areAllCollectorsReady,
bulkFetch: collectorSet.bulkFetch,
getCollectorByType: collectorSet.getCollectorByType,
makeStatsCollector: collectorSet.makeStatsCollector,
makeUsageCollector: collectorSet.makeUsageCollector,
registerCollector: collectorSet.registerCollector,
toApiFieldNames: collectorSet.toApiFieldNames,
toObject: collectorSet.toObject,
createUsageCounter,
getUsageCounterByType,
};
}
public start({ savedObjects }: CoreStart) {
this.logger.debug('Starting plugin');
const config = this.initializerContext.config.get<ConfigType>();
if (!this.usageCountersService) {
throw new Error('plugin setup must be called first.');
}
this.savedObjects = savedObjects.createInternalRepository();
if (config.usageCounters.enabled) {
this.usageCountersService.start({ savedObjects });
} else {
// call stop() to complete observers.
this.usageCountersService.stop();
}
}
public stop() {
this.logger.debug('Stopping plugin');
this.usageCountersService?.stop();
}
}

84
src/plugins/usage_collection/server/report/store_report.test.ts
View file

@ -12,11 +12,11 @@ import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { storeReport } from './store_report';
import { ReportSchemaType } from './schema';
import { METRIC_TYPE } from '@kbn/analytics';
import moment from 'moment';
import { usageCountersServiceMock } from '../usage_counters/usage_counters_service.mock';
describe('store_report', () => {
const momentTimestamp = moment();
const date = momentTimestamp.format('DDMMYYYY');
const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract();
const uiCountersUsageCounter = usageCountersServiceSetup.createUsageCounter('uiCounter');
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
@ -64,34 +64,56 @@ describe('store_report', () => {
},
},
};
await storeReport(repository, report);
await storeReport(repository, uiCountersUsageCounter, report);
expect(repository.create).toHaveBeenCalledWith(
'ui-metric',
{ count: 1 },
{
id: 'key-user-agent:test-user-agent',
overwrite: true,
}
);
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
1,
'ui-metric',
'test-app-name:test-event-name',
[{ fieldName: 'count', incrementBy: 3 }]
);
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
2,
'ui-counter',
`test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`,
[{ fieldName: 'count', incrementBy: 1 }]
);
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
3,
'ui-counter',
`test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`,
[{ fieldName: 'count', incrementBy: 2 }]
);
expect(repository.create.mock.calls).toMatchInlineSnapshot(`
Array [
Array [
"ui-metric",
Object {
"count": 1,
},
Object {
"id": "key-user-agent:test-user-agent",
"overwrite": true,
},
],
]
`);
expect(repository.incrementCounter.mock.calls).toMatchInlineSnapshot(`
Array [
Array [
"ui-metric",
"test-app-name:test-event-name",
Array [
Object {
"fieldName": "count",
"incrementBy": 3,
},
],
],
]
`);
expect((uiCountersUsageCounter.incrementCounter as jest.Mock).mock.calls)
.toMatchInlineSnapshot(`
Array [
Array [
Object {
"counterName": "test-app-name:test-event-name",
"counterType": "loaded",
"incrementBy": 1,
},
],
Array [
Object {
"counterName": "test-app-name:test-event-name",
"counterType": "click",
"incrementBy": 2,
},
],
]
`);
expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1);
expect(storeApplicationUsageMock).toHaveBeenCalledWith(
@ -108,7 +130,7 @@ describe('store_report', () => {
uiCounter: void 0,
application_usage: void 0,
};
await storeReport(repository, report);
await storeReport(repository, uiCountersUsageCounter, report);
expect(repository.bulkCreate).not.toHaveBeenCalled();
expect(repository.incrementCounter).not.toHaveBeenCalled();

18
src/plugins/usage_collection/server/report/store_report.ts
View file

@ -11,9 +11,12 @@ import moment from 'moment';
import { chain, sumBy } from 'lodash';
import { ReportSchemaType } from './schema';
import { storeApplicationUsage } from './store_application_usage';
import { UsageCounter } from '../usage_counters';
import { serializeUiCounterName } from '../../common/ui_counters';
export async function storeReport(
internalRepository: ISavedObjectsRepository,
uiCountersUsageCounter: UsageCounter,
report: ReportSchemaType
) {
const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : [];
@ -21,7 +24,6 @@ export async function storeReport(
const appUsages = report.application_usage ? Object.values(report.application_usage) : [];
const momentTimestamp = moment();
const date = momentTimestamp.format('DDMMYYYY');
const timestamp = momentTimestamp.toDate();
return Promise.allSettled([
@ -55,14 +57,14 @@ export async function storeReport(
})
.value(),
// UI Counters
...uiCounters.map(async ([key, metric]) => {
...uiCounters.map(async ([, metric]) => {
const { appName, eventName, total, type } = metric;
const savedObjectId = `${appName}:${date}:${type}:${eventName}`;
return [
await internalRepository.incrementCounter('ui-counter', savedObjectId, [
{ fieldName: 'count', incrementBy: total },
]),
];
const counterName = serializeUiCounterName({ appName, eventName });
uiCountersUsageCounter.incrementCounter({
counterName,
counterType: type,
incrementBy: total,
});
}),
// Application Usage
storeApplicationUsage(internalRepository, appUsages, timestamp),

6
src/plugins/usage_collection/server/routes/index.ts
View file

@ -16,14 +16,16 @@ import { Observable } from 'rxjs';
import { CollectorSet } from '../collector';
import { registerUiCountersRoute } from './ui_counters';
import { registerStatsRoute } from './stats';
import type { UsageCounter } from '../usage_counters';
export function setupRoutes({
router,
uiCountersUsageCounter,
getSavedObjects,
...rest
}: {
router: IRouter;
getSavedObjects: () => ISavedObjectsRepository | undefined;
uiCountersUsageCounter: UsageCounter;
config: {
allowAnonymous: boolean;
kibanaIndex: string;
@ -39,6 +41,6 @@ export function setupRoutes({
metrics: MetricsServiceSetup;
overallStatus$: Observable<ServiceStatus>;
}) {
registerUiCountersRoute(router, getSavedObjects);
registerUiCountersRoute(router, getSavedObjects, uiCountersUsageCounter);
registerStatsRoute({ router, ...rest });
}

6
src/plugins/usage_collection/server/routes/ui_counters.ts
View file

@ -9,10 +9,12 @@
import { schema } from '@kbn/config-schema';
import { IRouter, ISavedObjectsRepository } from 'src/core/server';
import { storeReport, reportSchema } from '../report';
import { UsageCounter } from '../usage_counters';
export function registerUiCountersRoute(
router: IRouter,
getSavedObjects: () => ISavedObjectsRepository | undefined
getSavedObjects: () => ISavedObjectsRepository | undefined,
uiCountersUsageCounter: UsageCounter
) {
router.post(
{
@ -30,7 +32,7 @@ export function registerUiCountersRoute(
if (!internalRepository) {
throw Error(`The saved objects client hasn't been initialised yet`);
}
await storeReport(internalRepository, report);
await storeReport(internalRepository, uiCountersUsageCounter, report);
return res.ok({ body: { status: 'ok' } });
} catch (error) {
return res.ok({ body: { status: 'fail' } });

58
src/plugins/usage_collection/server/usage_collection.mock.ts
View file

@ -1,58 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import {
elasticsearchServiceMock,
httpServerMock,
loggingSystemMock,
savedObjectsClientMock,
} from '../../../../src/core/server/mocks';
import { CollectorOptions, Collector, UsageCollector } from './collector';
import { UsageCollectionSetup, CollectorFetchContext } from './index';
export type { CollectorOptions };
export { Collector };
const logger = loggingSystemMock.createLogger();
export const createUsageCollectionSetupMock = () => {
const usageCollectionSetupMock: jest.Mocked<UsageCollectionSetup> = {
areAllCollectorsReady: jest.fn(),
bulkFetch: jest.fn(),
bulkFetchUsage: jest.fn(),
getCollectorByType: jest.fn(),
toApiFieldNames: jest.fn(),
toObject: jest.fn(),
makeStatsCollector: jest.fn().mockImplementation((cfg) => new Collector(logger, cfg)),
makeUsageCollector: jest.fn().mockImplementation((cfg) => new UsageCollector(logger, cfg)),
registerCollector: jest.fn(),
};
usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true);
return usageCollectionSetupMock;
};
export function createCollectorFetchContextMock(): jest.Mocked<CollectorFetchContext<false>> {
const collectorFetchClientsMock: jest.Mocked<CollectorFetchContext<false>> = {
esClient: elasticsearchServiceMock.createClusterClient().asInternalUser,
soClient: savedObjectsClientMock.create(),
};
return collectorFetchClientsMock;
}
export function createCollectorFetchContextWithKibanaMock(): jest.Mocked<
CollectorFetchContext<true>
> {
const collectorFetchClientsMock: jest.Mocked<CollectorFetchContext<true>> = {
esClient: elasticsearchServiceMock.createClusterClient().asInternalUser,
soClient: savedObjectsClientMock.create(),
kibanaRequest: httpServerMock.createKibanaRequest(),
};
return collectorFetchClientsMock;
}

15
src/plugins/usage_collection/server/usage_counters/index.ts Normal file
View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export type { UsageCountersServiceSetup } from './usage_counters_service';
export type { UsageCountersSavedObjectAttributes, UsageCountersSavedObject } from './saved_objects';
export type { IncrementCounterParams } from './usage_counter';
export { UsageCountersService } from './usage_counters_service';
export { UsageCounter } from './usage_counter';
export { USAGE_COUNTERS_SAVED_OBJECT_TYPE, serializeCounterKey } from './saved_objects';

71
src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts Normal file
View file

@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { serializeCounterKey, storeCounter } from './saved_objects';
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { CounterMetric } from './usage_counter';
import moment from 'moment';
describe('counterKey', () => {
test('#serializeCounterKey returns a serialized string', () => {
const result = serializeCounterKey({
domainId: 'a',
counterName: 'b',
counterType: 'c',
date: moment('09042021', 'DDMMYYYY'),
});
expect(result).toMatchInlineSnapshot(`"a:09042021:c:b"`);
});
});
describe('storeCounter', () => {
const internalRepository = savedObjectsRepositoryMock.create();
const mockNow = 1617954426939;
beforeEach(() => {
jest.spyOn(moment, 'now').mockReturnValue(mockNow);
});
afterAll(() => {
jest.resetAllMocks();
});
it('stores counter in a saved object', async () => {
const counterMetric: CounterMetric = {
domainId: 'a',
counterName: 'b',
counterType: 'c',
incrementBy: 13,
};
await storeCounter(counterMetric, internalRepository);
expect(internalRepository.incrementCounter).toBeCalledTimes(1);
expect(internalRepository.incrementCounter.mock.calls[0]).toMatchInlineSnapshot(`
Array [
"usage-counters",
"a:09042021:c:b",
Array [
Object {
"fieldName": "count",
"incrementBy": 13,
},
],
Object {
"upsertAttributes": Object {
"counterName": "b",
"counterType": "c",
"domainId": "a",
},
},
]
`);
});
});

86
src/plugins/usage_collection/server/usage_counters/saved_objects.ts Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import {
SavedObject,
SavedObjectsRepository,
SavedObjectAttributes,
SavedObjectsServiceSetup,
} from 'kibana/server';
import moment from 'moment';
import { CounterMetric } from './usage_counter';
export interface UsageCountersSavedObjectAttributes extends SavedObjectAttributes {
domainId: string;
counterName: string;
counterType: string;
count: number;
}
export type UsageCountersSavedObject = SavedObject<UsageCountersSavedObjectAttributes>;
export const USAGE_COUNTERS_SAVED_OBJECT_TYPE = 'usage-counters';
export const registerUsageCountersSavedObjectType = (
savedObjectsSetup: SavedObjectsServiceSetup
) => {
savedObjectsSetup.registerType({
name: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
hidden: false,
namespaceType: 'agnostic',
mappings: {
dynamic: false,
properties: {
domainId: { type: 'keyword' },
},
},
});
};
export interface SerializeCounterParams {
domainId: string;
counterName: string;
counterType: string;
date: moment.MomentInput;
}
export const serializeCounterKey = ({
domainId,
counterName,
counterType,
date,
}: SerializeCounterParams) => {
const dayDate = moment(date).format('DDMMYYYY');
return `${domainId}:${dayDate}:${counterType}:${counterName}`;
};
export const storeCounter = async (
counterMetric: CounterMetric,
internalRepository: Pick<SavedObjectsRepository, 'incrementCounter'>
) => {
const { counterName, counterType, domainId, incrementBy } = counterMetric;
const key = serializeCounterKey({
date: moment.now(),
domainId,
counterName,
counterType,
});
return await internalRepository.incrementCounter(
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
key,
[{ fieldName: 'count', incrementBy }],
{
upsertAttributes: {
domainId,
counterName,
counterType,
},
}
);
};

38
src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { UsageCounter, CounterMetric } from './usage_counter';
import * as Rx from 'rxjs';
import * as rxOp from 'rxjs/operators';
describe('UsageCounter', () => {
const domainId = 'test-domain-id';
const counter$ = new Rx.Subject<CounterMetric>();
const usageCounter = new UsageCounter({ domainId, counter$ });
afterAll(() => {
counter$.complete();
});
describe('#incrementCounter', () => {
it('#incrementCounter calls counter$.next', async () => {
const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise();
usageCounter.incrementCounter({ counterName: 'test', counterType: 'type', incrementBy: 13 });
await expect(result).resolves.toEqual([
{ counterName: 'test', counterType: 'type', domainId: 'test-domain-id', incrementBy: 13 },
]);
});
it('passes default configs to counter$', async () => {
const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise();
usageCounter.incrementCounter({ counterName: 'test' });
await expect(result).resolves.toEqual([
{ counterName: 'test', counterType: 'count', domainId: 'test-domain-id', incrementBy: 1 },
]);
});
});
});

48
src/plugins/usage_collection/server/usage_counters/usage_counter.ts Normal file
View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import * as Rx from 'rxjs';
export interface CounterMetric {
domainId: string;
counterName: string;
counterType: string;
incrementBy: number;
}
export interface UsageCounterDeps {
domainId: string;
counter$: Rx.Subject<CounterMetric>;
}
export interface IncrementCounterParams {
counterName: string;
counterType?: string;
incrementBy?: number;
}
export class UsageCounter {
private domainId: string;
private counter$: Rx.Subject<CounterMetric>;
constructor({ domainId, counter$ }: UsageCounterDeps) {
this.domainId = domainId;
this.counter$ = counter$;
}
public incrementCounter = (params: IncrementCounterParams) => {
const { counterName, counterType = 'count', incrementBy = 1 } = params;
this.counter$.next({
counterName,
domainId: this.domainId,
counterType,
incrementBy,
});
};
}

40
src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { PublicMethodsOf } from '@kbn/utility-types';
import type { UsageCountersService, UsageCountersServiceSetup } from './usage_counters_service';
import type { UsageCounter } from './usage_counter';
const createSetupContractMock = () => {
const setupContract: jest.Mocked<UsageCountersServiceSetup> = {
createUsageCounter: jest.fn(),
getUsageCounterByType: jest.fn(),
};
setupContract.createUsageCounter.mockReturnValue(({
incrementCounter: jest.fn(),
} as unknown) as jest.Mocked<UsageCounter>);
return setupContract;
};
const createUsageCountersServiceMock = () => {
const mocked: jest.Mocked<PublicMethodsOf<UsageCountersService>> = {
setup: jest.fn(),
start: jest.fn(),
stop: jest.fn(),
};
mocked.setup.mockReturnValue(createSetupContractMock());
return mocked;
};
export const usageCountersServiceMock = {
create: createUsageCountersServiceMock,
createSetupContract: createSetupContractMock,
};

241
src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts Normal file
View file

@ -0,0 +1,241 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/* eslint-disable dot-notation */
import { UsageCountersService } from './usage_counters_service';
import { loggingSystemMock, coreMock } from '../../../../core/server/mocks';
import * as rxOp from 'rxjs/operators';
import moment from 'moment';
const tick = () => {
jest.useRealTimers();
return new Promise((resolve) => setTimeout(resolve, 1));
};
describe('UsageCountersService', () => {
const retryCount = 1;
const bufferDurationMs = 100;
const mockNow = 1617954426939;
const logger = loggingSystemMock.createLogger();
const coreSetup = coreMock.createSetup();
const coreStart = coreMock.createStart();
beforeEach(() => {
jest.spyOn(moment, 'now').mockReturnValue(mockNow);
});
afterEach(() => {
jest.clearAllMocks();
});
it('stores data in cache during setup', async () => {
const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs });
const { createUsageCounter } = usageCountersService.setup(coreSetup);
const usageCounter = createUsageCounter('test-counter');
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCounter.incrementCounter({ counterName: 'counterA' });
const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise();
usageCountersService['flushCache$'].next();
usageCountersService['source$'].complete();
await expect(dataInSourcePromise).resolves.toHaveLength(2);
});
it('registers savedObject type during setup', () => {
const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs });
usageCountersService.setup(coreSetup);
expect(coreSetup.savedObjects.registerType).toBeCalledTimes(1);
});
it('flushes cached data on start', async () => {
const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs });
const mockRepository = coreStart.savedObjects.createInternalRepository();
const mockIncrementCounter = jest.fn();
mockRepository.incrementCounter = mockIncrementCounter;
coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository);
const { createUsageCounter } = usageCountersService.setup(coreSetup);
const usageCounter = createUsageCounter('test-counter');
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCounter.incrementCounter({ counterName: 'counterA' });
const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise();
usageCountersService.start(coreStart);
usageCountersService['source$'].complete();
await expect(dataInSourcePromise).resolves.toMatchInlineSnapshot(`
Array [
Object {
"counterName": "counterA",
"counterType": "count",
"domainId": "test-counter",
"incrementBy": 1,
},
Object {
"counterName": "counterA",
"counterType": "count",
"domainId": "test-counter",
"incrementBy": 1,
},
]
`);
});
it('buffers data into savedObject', async () => {
const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs });
const mockRepository = coreStart.savedObjects.createInternalRepository();
const mockIncrementCounter = jest.fn().mockResolvedValue('success');
mockRepository.incrementCounter = mockIncrementCounter;
coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository);
const { createUsageCounter } = usageCountersService.setup(coreSetup);
jest.useFakeTimers('modern');
const usageCounter = createUsageCounter('test-counter');
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCountersService.start(coreStart);
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCounter.incrementCounter({ counterName: 'counterB' });
jest.runOnlyPendingTimers();
expect(mockIncrementCounter).toBeCalledTimes(2);
expect(mockIncrementCounter.mock.calls).toMatchInlineSnapshot(`
Array [
Array [
"usage-counters",
"test-counter:09042021:count:counterA",
Array [
Object {
"fieldName": "count",
"incrementBy": 3,
},
],
Object {
"upsertAttributes": Object {
"counterName": "counterA",
"counterType": "count",
"domainId": "test-counter",
},
},
],
Array [
"usage-counters",
"test-counter:09042021:count:counterB",
Array [
Object {
"fieldName": "count",
"incrementBy": 1,
},
],
Object {
"upsertAttributes": Object {
"counterName": "counterB",
"counterType": "count",
"domainId": "test-counter",
},
},
],
]
`);
});
it('retries errors by `retryCount` times before failing to store', async () => {
const usageCountersService = new UsageCountersService({
logger,
retryCount: 1,
bufferDurationMs,
});
const mockRepository = coreStart.savedObjects.createInternalRepository();
const mockError = new Error('failed.');
const mockIncrementCounter = jest.fn().mockImplementation((_, key) => {
switch (key) {
case 'test-counter:09042021:count:counterA':
throw mockError;
case 'test-counter:09042021:count:counterB':
return 'pass';
default:
throw new Error(`unknown key ${key}`);
}
});
mockRepository.incrementCounter = mockIncrementCounter;
coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository);
const { createUsageCounter } = usageCountersService.setup(coreSetup);
jest.useFakeTimers('modern');
const usageCounter = createUsageCounter('test-counter');
usageCountersService.start(coreStart);
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCounter.incrementCounter({ counterName: 'counterB' });
jest.runOnlyPendingTimers();
// wait for retries to kick in on next scheduler call
await tick();
// number of incrementCounter calls + number of retries
expect(mockIncrementCounter).toBeCalledTimes(2 + 1);
expect(logger.debug).toHaveBeenNthCalledWith(1, 'Store counters into savedObjects', [
mockError,
'pass',
]);
});
it('buffers counters within `bufferDurationMs` time', async () => {
const usageCountersService = new UsageCountersService({
logger,
retryCount,
bufferDurationMs: 30000,
});
const mockRepository = coreStart.savedObjects.createInternalRepository();
const mockIncrementCounter = jest.fn().mockImplementation((_data, key, counter) => {
expect(counter).toHaveLength(1);
return { key, incrementBy: counter[0].incrementBy };
});
mockRepository.incrementCounter = mockIncrementCounter;
coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository);
const { createUsageCounter } = usageCountersService.setup(coreSetup);
jest.useFakeTimers('modern');
const usageCounter = createUsageCounter('test-counter');
usageCountersService.start(coreStart);
usageCounter.incrementCounter({ counterName: 'counterA' });
usageCounter.incrementCounter({ counterName: 'counterA' });
jest.advanceTimersByTime(30000);
usageCounter.incrementCounter({ counterName: 'counterA' });
jest.runOnlyPendingTimers();
// wait for debounce to kick in on next scheduler call
await tick();
expect(mockIncrementCounter).toBeCalledTimes(2);
expect(mockIncrementCounter.mock.results.map(({ value }) => value)).toMatchInlineSnapshot(`
Array [
Object {
"incrementBy": 2,
"key": "test-counter:09042021:count:counterA",
},
Object {
"incrementBy": 1,
"key": "test-counter:09042021:count:counterA",
},
]
`);
});
});

185
src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts Normal file
View file

@ -0,0 +1,185 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import * as Rx from 'rxjs';
import * as rxOp from 'rxjs/operators';
import {
SavedObjectsRepository,
SavedObjectsServiceSetup,
SavedObjectsServiceStart,
} from 'src/core/server';
import type { Logger } from 'src/core/server';
import moment from 'moment';
import { CounterMetric, UsageCounter } from './usage_counter';
import {
registerUsageCountersSavedObjectType,
storeCounter,
serializeCounterKey,
} from './saved_objects';
export interface UsageCountersServiceDeps {
logger: Logger;
retryCount: number;
bufferDurationMs: number;
}
export interface UsageCountersServiceSetup {
createUsageCounter: (type: string) => UsageCounter;
getUsageCounterByType: (type: string) => UsageCounter | undefined;
}
/* internal */
export interface UsageCountersServiceSetupDeps {
savedObjects: SavedObjectsServiceSetup;
}
/* internal */
export interface UsageCountersServiceStartDeps {
savedObjects: SavedObjectsServiceStart;
}
export class UsageCountersService {
private readonly stop$ = new Rx.Subject();
private readonly retryCount: number;
private readonly bufferDurationMs: number;
private readonly counterSets = new Map<string, UsageCounter>();
private readonly source$ = new Rx.Subject<CounterMetric>();
private readonly counter$ = this.source$.pipe(rxOp.multicast(new Rx.Subject()), rxOp.refCount());
private readonly flushCache$ = new Rx.Subject<CounterMetric>();
private readonly stopCaching$ = new Rx.Subject();
private readonly logger: Logger;
constructor({ logger, retryCount, bufferDurationMs }: UsageCountersServiceDeps) {
this.logger = logger;
this.retryCount = retryCount;
this.bufferDurationMs = bufferDurationMs;
}
public setup = (core: UsageCountersServiceSetupDeps): UsageCountersServiceSetup => {
const cache$ = new Rx.ReplaySubject<CounterMetric>();
const storingCache$ = new Rx.BehaviorSubject<boolean>(false);
// flush cache data from cache -> source
this.flushCache$
.pipe(
rxOp.exhaustMap(() => cache$),
rxOp.takeUntil(this.stop$)
)
.subscribe((data) => {
storingCache$.next(true);
this.source$.next(data);
});
// store data into cache when not paused
storingCache$
.pipe(
rxOp.distinctUntilChanged(),
rxOp.switchMap((isStoring) => (isStoring ? Rx.EMPTY : this.source$)),
rxOp.takeUntil(Rx.merge(this.stopCaching$, this.stop$))
)
.subscribe((data) => {
cache$.next(data);
storingCache$.next(false);
});
registerUsageCountersSavedObjectType(core.savedObjects);
return {
createUsageCounter: this.createUsageCounter,
getUsageCounterByType: this.getUsageCounterByType,
};
};
public start = ({ savedObjects }: UsageCountersServiceStartDeps): void => {
this.stopCaching$.next();
const internalRepository = savedObjects.createInternalRepository();
this.counter$
.pipe(
/* buffer source events every ${bufferDurationMs} */
rxOp.bufferTime(this.bufferDurationMs),
/**
* bufferTime will trigger every ${bufferDurationMs}
* regardless if source emitted anything or not.
* using filter will stop cut the pipe short
*/
rxOp.filter((counters) => Array.isArray(counters) && counters.length > 0),
rxOp.map((counters) => Object.values(this.mergeCounters(counters))),
rxOp.takeUntil(this.stop$),
rxOp.concatMap((counters) => this.storeDate$(counters, internalRepository))
)
.subscribe((results) => {
this.logger.debug('Store counters into savedObjects', results);
});
this.flushCache$.next();
};
public stop = () => {
this.stop$.next();
};
private storeDate$(
counters: CounterMetric[],
internalRepository: Pick<SavedObjectsRepository, 'incrementCounter'>
) {
return Rx.forkJoin(
counters.map((counter) =>
Rx.defer(() => storeCounter(counter, internalRepository)).pipe(
rxOp.retry(this.retryCount),
rxOp.catchError((error) => {
this.logger.warn(error);
return Rx.of(error);
})
)
)
);
}
private createUsageCounter = (type: string): UsageCounter => {
if (this.counterSets.get(type)) {
throw new Error(`Usage counter set "${type}" already exists.`);
}
const counterSet = new UsageCounter({
domainId: type,
counter$: this.source$,
});
this.counterSets.set(type, counterSet);
return counterSet;
};
private getUsageCounterByType = (type: string): UsageCounter | undefined => {
return this.counterSets.get(type);
};
private mergeCounters = (counters: CounterMetric[]): Record<string, CounterMetric> => {
const date = moment.now();
return counters.reduce((acc, counter) => {
const { counterName, domainId, counterType } = counter;
const key = serializeCounterKey({ domainId, counterName, counterType, date });
const existingCounter = acc[key];
if (!existingCounter) {
acc[key] = counter;
return acc;
}
return {
...acc,
[key]: {
...existingCounter,
...counter,
incrementBy: existingCounter.incrementBy + counter.incrementBy,
},
};
}, {} as Record<string, CounterMetric>);
};
}

6
src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts
View file

@ -10,8 +10,10 @@ jest.mock('./get_stats', () => ({
getStats: jest.fn().mockResolvedValue({ somestat: 1 }),
}));
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import {
createUsageCollectionSetupMock,
createCollectorFetchContextMock,
} from 'src/plugins/usage_collection/server/mocks';
import { registerVisTypeTableUsageCollector } from './register_usage_collector';
import { getStats } from './get_stats';

2
src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts
View file

@ -8,7 +8,7 @@
import { of } from 'rxjs';
import { mockStats, mockGetStats } from './get_usage_collector.mock';
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock';
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import { registerTimeseriesUsageCollector } from './register_timeseries_collector';
import { ConfigObservable } from '../types';

2
src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts
View file

@ -8,7 +8,7 @@
import { of } from 'rxjs';
import { mockStats, mockGetStats } from './get_usage_collector.mock';
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock';
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import { HomeServerPluginSetup } from '../../../home/server';
import { registerVegaUsageCollector } from './register_vega_collector';

2
src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts
View file

@ -8,7 +8,7 @@
import { of } from 'rxjs';
import { mockStats, mockGetStats } from './get_usage_collector.mock';
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock';
import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import { registerVisualizationsCollector } from './register_visualizations_collector';

8
test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts
View file

@ -8,6 +8,14 @@
export const basicUiCounters = {
dailyEvents: [
{
appName: 'myApp',
eventName: 'some_app_event',
lastUpdatedAt: '2021-11-20T11:43:00.961Z',
fromTimestamp: '2021-11-20T00:00:00Z',
counterType: 'count',
total: 2,
},
{
appName: 'myApp',
eventName: 'my_event_885082425109579',

36
test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export const basicUsageCounters = {
dailyEvents: [
{
domainId: 'anotherDomainId',
counterName: 'some_event_name',
counterType: 'count',
lastUpdatedAt: '2021-11-20T11:43:00.961Z',
fromTimestamp: '2021-11-20T00:00:00Z',
total: 3,
},
{
domainId: 'anotherDomainId',
counterName: 'some_event_name',
counterType: 'count',
lastUpdatedAt: '2021-04-09T11:43:00.961Z',
fromTimestamp: '2021-04-09T00:00:00Z',
total: 2,
},
{
domainId: 'anotherDomainId2',
counterName: 'some_event_name',
counterType: 'count',
lastUpdatedAt: '2021-04-20T08:18:03.030Z',
fromTimestamp: '2021-04-20T00:00:00Z',
total: 1,
},
],
};

15
test/api_integration/apis/telemetry/telemetry_local.ts
View file

@ -9,6 +9,7 @@
import expect from '@kbn/expect';
import supertestAsPromised from 'supertest-as-promised';
import { basicUiCounters } from './__fixtures__/ui_counters';
import { basicUsageCounters } from './__fixtures__/usage_counters';
import type { FtrProviderContext } from '../../ftr_provider_context';
import type { SavedObject } from '../../../../src/core/server';
import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json';
@ -153,6 +154,20 @@ export default function ({ getService }: FtrProviderContext) {
});
});
describe('Usage Counters telemetry', () => {
before('Add UI Counters saved objects', () =>
esArchiver.load('saved_objects/usage_counters')
);
after('cleanup saved objects changes', () =>
esArchiver.unload('saved_objects/usage_counters')
);
it('returns usage counters aggregated by day', async () => {
const stats = await retrieveTelemetry(supertest);
expect(stats.stack_stats.kibana.plugins.usage_counters).to.eql(basicUsageCounters);
});
});
describe('application usage limits', () => {
function createSavedObject(viewId?: string) {
return supertest

50
test/api_integration/apis/ui_counters/ui_counters.ts
View file

@ -7,11 +7,10 @@
*/
import expect from '@kbn/expect';
import { ReportManager, METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics';
import { ReportManager, METRIC_TYPE, UiCounterMetricType, Report } from '@kbn/analytics';
import moment from 'moment';
import { FtrProviderContext } from '../../ftr_provider_context';
import { SavedObject } from '../../../../src/core/server';
import { UICounterSavedObjectAttributes } from '../../../../src/plugins/kibana_usage_collection/server/collectors/ui_counters/ui_counter_saved_object_type';
import { UsageCountersSavedObject } from '../../../../src/plugins/usage_collection/server';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
@ -24,10 +23,22 @@ export default function ({ getService }: FtrProviderContext) {
count,
});
const sendReport = async (report: Report) => {
await supertest
.post('/api/ui_counters/_report')
.set('kbn-xsrf', 'kibana')
.set('content-type', 'application/json')
.send({ report })
.expect(200);
// wait for SO to index data into ES
await new Promise((res) => setTimeout(res, 5 * 1000));
};
const getCounterById = (
savedObjects: Array<SavedObject<UICounterSavedObjectAttributes>>,
savedObjects: UsageCountersSavedObject[],
targetId: string
): SavedObject<UICounterSavedObjectAttributes> => {
): UsageCountersSavedObject => {
const savedObject = savedObjects.find(({ id }: { id: string }) => id === targetId);
if (!savedObject) {
throw new Error(`Unable to find savedObject id ${targetId}`);
@ -40,30 +51,25 @@ export default function ({ getService }: FtrProviderContext) {
const dayDate = moment().format('DDMMYYYY');
before(async () => await esArchiver.emptyKibanaIndex());
it('stores ui counter events in savedObjects', async () => {
it('stores ui counter events in usage counters savedObjects', async () => {
const reportManager = new ReportManager();
const { report } = reportManager.assignReports([
createUiCounterEvent('my_event', METRIC_TYPE.COUNT),
]);
await supertest
.post('/api/ui_counters/_report')
.set('kbn-xsrf', 'kibana')
.set('content-type', 'application/json')
.send({ report })
.expect(200);
await sendReport(report);
const {
body: { saved_objects: savedObjects },
} = await supertest
.get('/api/saved_objects/_find?type=ui-counter')
.get('/api/saved_objects/_find?type=usage-counters')
.set('kbn-xsrf', 'kibana')
.expect(200);
const countTypeEvent = getCounterById(
savedObjects,
`myApp:${dayDate}:${METRIC_TYPE.COUNT}:my_event`
`uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:my_event`
);
expect(countTypeEvent.attributes.count).to.eql(1);
});
@ -78,35 +84,31 @@ export default function ({ getService }: FtrProviderContext) {
createUiCounterEvent(`${uniqueEventName}_2`, METRIC_TYPE.COUNT),
createUiCounterEvent(uniqueEventName, METRIC_TYPE.CLICK, 2),
]);
await supertest
.post('/api/ui_counters/_report')
.set('kbn-xsrf', 'kibana')
.set('content-type', 'application/json')
.send({ report })
.expect(200);
await sendReport(report);
const {
body: { saved_objects: savedObjects },
} = await supertest
.get('/api/saved_objects/_find?type=ui-counter&fields=count')
.get('/api/saved_objects/_find?type=usage-counters&fields=count')
.set('kbn-xsrf', 'kibana')
.expect(200);
const countTypeEvent = getCounterById(
savedObjects,
`myApp:${dayDate}:${METRIC_TYPE.COUNT}:${uniqueEventName}`
`uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:${uniqueEventName}`
);
expect(countTypeEvent.attributes.count).to.eql(1);
const clickTypeEvent = getCounterById(
savedObjects,
`myApp:${dayDate}:${METRIC_TYPE.CLICK}:${uniqueEventName}`
`uiCounter:${dayDate}:${METRIC_TYPE.CLICK}:myApp:${uniqueEventName}`
);
expect(clickTypeEvent.attributes.count).to.eql(2);
const secondEvent = getCounterById(
savedObjects,
`myApp:${dayDate}:${METRIC_TYPE.COUNT}:${uniqueEventName}_2`
`uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:${uniqueEventName}_2`
);
expect(secondEvent.attributes.count).to.eql(1);
});

2
test/api_integration/config.js
View file

@ -31,6 +31,8 @@ export default async function ({ readConfigFile }) {
'--server.xsrf.disableProtection=true',
'--server.compression.referrerWhitelist=["some-host.com"]',
`--savedObjects.maxImportExportSize=10001`,
// for testing set buffer duration to 0 to immediately flush counters into saved objects.
'--usageCollection.usageCounters.bufferDuration=0',
],
},
};

111
test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json Normal file
View file

@ -0,0 +1,111 @@
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "ui-counter:myApp:30112020:loaded:my_event_885082425109579",
"source": {
"ui-counter": {
"count": 1
},
"type": "ui-counter",
"updated_at": "2020-11-30T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "ui-counter:myApp:30112020:count:my_event_885082425109579_2",
"source": {
"ui-counter": {
"count": 1
},
"type": "ui-counter",
"updated_at": "2020-11-30T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "ui-counter:myApp:30112020:count:my_event_885082425109579_2",
"source": {
"ui-counter": {
"count": 1
},
"type": "ui-counter",
"updated_at": "2020-10-28T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "ui-counter:myApp:30112020:click:my_event_885082425109579",
"source": {
"ui-counter": {
"count": 2
},
"type": "ui-counter",
"updated_at": "2020-11-30T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "ui-counter:myApp:30112020:click:my_event_885082425109579",
"source": {
"ui-counter": {
"count": 2
},
"type": "ui-counter",
"updated_at": "2020-11-30T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "uiCounter:09042021:count:myApp:some_app_event",
"source": {
"usage-counters": {
"count": 2,
"domainId": "uiCounter",
"counterName": "myApp:some_app_event",
"counterType": "count"
},
"type": "usage-counters",
"updated_at": "2021-11-20T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "anotherDomainId:09042021:count:some_event_name",
"source": {
"usage-counters": {
"count": 2,
"domainId": "anotherDomainId",
"counterName": "some_event_name",
"counterType": "count"
},
"type": "usage-counters",
"updated_at": "2021-11-20T11:43:00.961Z"
}
}
}

BIN
test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz
View file

Binary file not shown.

9
test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json
View file

@ -35,6 +35,15 @@
}
}
},
"usage-counters": {
"dynamic": false,
"properties": {
"domainId": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dashboard": {
"properties": {
"description": {

89
test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json Normal file
View file

@ -0,0 +1,89 @@
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "uiCounter:20112020:count:myApp:some_app_event",
"source": {
"usage-counters": {
"count": 2,
"domainId": "uiCounter",
"counterName": "myApp:some_app_event",
"counterType": "count"
},
"type": "usage-counters",
"updated_at": "2021-11-20T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "anotherDomainId:20112020:count:some_event_name",
"source": {
"usage-counters": {
"count": 3,
"domainId": "anotherDomainId",
"counterName": "some_event_name",
"counterType": "count"
},
"type": "usage-counters",
"updated_at": "2021-11-20T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "anotherDomainId:09042021:count:some_event_name",
"source": {
"usage-counters": {
"count": 2,
"domainId": "anotherDomainId",
"counterName": "some_event_name",
"counterType": "count"
},
"type": "usage-counters",
"updated_at": "2021-04-09T11:43:00.961Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "anotherDomainId2:09042021:count:some_event_name",
"source": {
"usage-counters": {
"count": 1,
"domainId": "anotherDomainId2",
"counterName": "some_event_name",
"counterType": "count"
},
"type": "usage-counters",
"updated_at": "2021-04-20T08:18:03.030Z"
}
}
}
{
"type": "doc",
"value": {
"index": ".kibana",
"id": "anotherDomainId3:09042021:custom_type:zero_count",
"source": {
"usage-counters": {
"count": 0,
"domainId": "anotherDomainId3",
"counterName": "zero_count",
"counterType": "custom_type"
},
"type": "usage-counters",
"updated_at": "2021-04-20T08:18:03.030Z"
}
}
}

276
test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json Normal file
View file

@ -0,0 +1,276 @@
{
"type": "index",
"value": {
"index": ".kibana",
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
},
"mappings": {
"dynamic": "strict",
"properties": {
"config": {
"dynamic": "true",
"properties": {
"buildNum": {
"type": "keyword"
},
"defaultIndex": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"usage-counters": {
"dynamic": false,
"properties": {
"domainId": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"optionsJSON": {
"type": "text"
},
"panelsJSON": {
"type": "text"
},
"refreshInterval": {
"properties": {
"display": {
"type": "keyword"
},
"pause": {
"type": "boolean"
},
"section": {
"type": "integer"
},
"value": {
"type": "integer"
}
}
},
"timeFrom": {
"type": "keyword"
},
"timeRestore": {
"type": "boolean"
},
"timeTo": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"index-pattern": {
"properties": {
"fieldFormatMap": {
"type": "text"
},
"fields": {
"type": "text"
},
"intervalName": {
"type": "keyword"
},
"notExpandable": {
"type": "boolean"
},
"sourceFilters": {
"type": "text"
},
"timeFieldName": {
"type": "keyword"
},
"title": {
"type": "text"
}
}
},
"search": {
"properties": {
"columns": {
"type": "keyword"
},
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"sort": {
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"server": {
"properties": {
"uuid": {
"type": "keyword"
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"namespace": {
"type": "keyword"
},
"references": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"type": {
"type": "keyword"
}
},
"type": "nested"
},
"type": {
"type": "keyword"
},
"updated_at": {
"type": "date"
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 2048
}
}
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"savedSearchId": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
},
"visState": {
"type": "text"
}
}
}
}
}
}
}

3
test/plugin_functional/config.ts
View file

@ -21,6 +21,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
return {
testFiles: [
require.resolve('./test_suites/usage_collection'),
require.resolve('./test_suites/core'),
require.resolve('./test_suites/custom_visualizations'),
require.resolve('./test_suites/panel_actions'),
@ -59,6 +60,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
'--corePluginDeprecations.oldProperty=hello',
'--corePluginDeprecations.secret=100',
'--corePluginDeprecations.noLongerUsed=still_using',
// for testing set buffer duration to 0 to immediately flush counters into saved objects.
'--usageCollection.usageCounters.bufferDuration=0',
...plugins.map(
(pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}`
),

9
test/plugin_functional/plugins/usage_collection/kibana.json Normal file
View file

@ -0,0 +1,9 @@
{
"id": "usageCollectionTestPlugin",
"version": "0.0.1",
"kibanaVersion": "kibana",
"configPath": ["usageCollectionTestPlugin"],
"requiredPlugins": ["usageCollection"],
"server": true,
"ui": false
}

14
test/plugin_functional/plugins/usage_collection/package.json Normal file
View file

@ -0,0 +1,14 @@
{
"name": "usage_collection_test_plugin",
"version": "1.0.0",
"main": "target/test/plugin_functional/plugins/usage_collection",
"kibana": {
"version": "kibana",
"templateVersion": "1.0.0"
},
"license": "SSPL-1.0 OR Elastic License 2.0",
"scripts": {
"kbn": "node ../../../../scripts/kbn.js",
"build": "rm -rf './target' && ../../../../node_modules/.bin/tsc"
}
}

10
test/plugin_functional/plugins/usage_collection/server/index.ts Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { UsageCollectionTestPlugin } from './plugin';
export const plugin = () => new UsageCollectionTestPlugin();

43
test/plugin_functional/plugins/usage_collection/server/plugin.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { Plugin, CoreSetup } from 'kibana/server';
import {
UsageCollectionSetup,
UsageCounter,
} from '../../../../../src/plugins/usage_collection/server';
import { registerRoutes } from './routes';
export interface TestPluginDepsSetup {
usageCollection: UsageCollectionSetup;
}
export class UsageCollectionTestPlugin implements Plugin {
private usageCounter?: UsageCounter;
public setup(core: CoreSetup, { usageCollection }: TestPluginDepsSetup) {
const usageCounter = usageCollection.createUsageCounter('usageCollectionTestPlugin');
registerRoutes(core.http, usageCounter);
usageCounter.incrementCounter({
counterName: 'duringSetup',
incrementBy: 10,
});
usageCounter.incrementCounter({ counterName: 'duringSetup' });
this.usageCounter = usageCounter;
}
public start() {
if (!this.usageCounter) {
throw new Error('this.usageCounter is expected to be defined during setup.');
}
this.usageCounter.incrementCounter({ counterName: 'duringStart' });
}
public stop() {}
}

24
test/plugin_functional/plugins/usage_collection/server/routes.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { HttpServiceSetup } from 'kibana/server';
import { UsageCounter } from '../../../../../src/plugins/usage_collection/server';
export function registerRoutes(http: HttpServiceSetup, usageCounter: UsageCounter) {
const router = http.createRouter();
router.get(
{
path: '/api/usage_collection_test_plugin',
validate: false,
},
async (context, req, res) => {
usageCounter.incrementCounter({ counterName: 'routeAccessed' });
return res.ok();
}
);
}

18
test/plugin_functional/plugins/usage_collection/tsconfig.json Normal file
View file

@ -0,0 +1,18 @@
{
"extends": "../../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "./target",
"skipLibCheck": true
},
"include": [
"index.ts",
"public/**/*.ts",
"public/**/*.tsx",
"server/**/*.ts",
"../../../../typings/**/*",
],
"exclude": [],
"references": [
{ "path": "../../../../src/core/tsconfig.json" }
]
}

15
test/plugin_functional/test_suites/usage_collection/index.ts Normal file
View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { PluginFunctionalProviderContext } from '../../services';
export default function ({ loadTestFile }: PluginFunctionalProviderContext) {
describe('usage collection', function () {
loadTestFile(require.resolve('./usage_counters'));
});
}

67
test/plugin_functional/test_suites/usage_collection/usage_counters.ts Normal file
View file

@ -0,0 +1,67 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import expect from '@kbn/expect';
import { PluginFunctionalProviderContext } from '../../services';
import {
UsageCountersSavedObject,
serializeCounterKey,
} from '../../../../src/plugins/usage_collection/server/usage_counters';
export default function ({ getService, getPageObjects }: PluginFunctionalProviderContext) {
const supertest = getService('supertest');
async function getSavedObjectCounters() {
// wait until ES indexes the counter SavedObject;
await new Promise((res) => setTimeout(res, 7 * 1000));
return await supertest
.get('/api/saved_objects/_find?type=usage-counters')
.set('kbn-xsrf', 'true')
.expect(200)
.then(({ body }) => {
expect(body.total).to.above(1);
return (body.saved_objects as UsageCountersSavedObject[]).reduce((acc, savedObj) => {
const { count, counterName, domainId } = savedObj.attributes;
if (domainId === 'usageCollectionTestPlugin') {
acc[counterName] = count;
}
return acc;
}, {} as Record<string, number>);
});
}
describe('Usage Counters service', () => {
before(async () => {
const key = serializeCounterKey({
counterName: 'routeAccessed',
counterType: 'count',
domainId: 'usageCollectionTestPlugin',
date: Date.now(),
});
await supertest.delete(`/api/saved_objects/usage-counters/${key}`).set('kbn-xsrf', 'true');
});
it('stores usage counters sent during start and setup', async () => {
const { duringSetup, duringStart, routeAccessed } = await getSavedObjectCounters();
expect(duringSetup).to.be(11);
expect(duringStart).to.be(1);
expect(routeAccessed).to.be(undefined);
});
it('stores usage counters triggered by runtime activities', async () => {
await supertest.get('/api/usage_collection_test_plugin').set('kbn-xsrf', 'true').expect(200);
const { routeAccessed } = await getSavedObjectCounters();
expect(routeAccessed).to.be(1);
});
});
}