[Application Usage] use incrementCounter on daily reports instead of creating transactional documents (#94923)
* initial implementation * add test for upsertAttributes * update generated doc * remove comment * removal transactional documents from the collector aggregation logic * rename generic type * add missing doc file * split rollups into distinct files * for api integ test * extract types to their own file * only roll transactional documents during startup * proper fix is better fix * perform daily rolling until success * unskip flaky test * fix unit tests
This commit is contained in:
parent
5d59c2c8e1
commit
32212644da
|
@ -8,7 +8,7 @@
|
||||||
<b>Signature:</b>
|
<b>Signature:</b>
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions
|
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown> extends SavedObjectsBaseOptions
|
||||||
```
|
```
|
||||||
|
|
||||||
## Properties
|
## Properties
|
||||||
|
@ -18,4 +18,5 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
|
||||||
| [initialize](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.initialize.md) | <code>boolean</code> | (default=false) If true, sets all the counter fields to 0 if they don't already exist. Existing fields will be left as-is and won't be incremented. |
|
| [initialize](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.initialize.md) | <code>boolean</code> | (default=false) If true, sets all the counter fields to 0 if they don't already exist. Existing fields will be left as-is and won't be incremented. |
|
||||||
| [migrationVersion](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.migrationversion.md) | <code>SavedObjectsMigrationVersion</code> | [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) |
|
| [migrationVersion](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.migrationversion.md) | <code>SavedObjectsMigrationVersion</code> | [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) |
|
||||||
| [refresh](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.refresh.md) | <code>MutatingOperationRefreshSetting</code> | (default='wait\_for') The Elasticsearch refresh setting for this operation. See [MutatingOperationRefreshSetting](./kibana-plugin-core-server.mutatingoperationrefreshsetting.md) |
|
| [refresh](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.refresh.md) | <code>MutatingOperationRefreshSetting</code> | (default='wait\_for') The Elasticsearch refresh setting for this operation. See [MutatingOperationRefreshSetting](./kibana-plugin-core-server.mutatingoperationrefreshsetting.md) |
|
||||||
|
| [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md) | <code>Attributes</code> | Attributes to use when upserting the document if it doesn't exist. |
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
|
||||||
|
|
||||||
|
[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) > [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md)
|
||||||
|
|
||||||
|
## SavedObjectsIncrementCounterOptions.upsertAttributes property
|
||||||
|
|
||||||
|
Attributes to use when upserting the document if it doesn't exist.
|
||||||
|
|
||||||
|
<b>Signature:</b>
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
upsertAttributes?: Attributes;
|
||||||
|
```
|
|
@ -9,7 +9,7 @@ Increments all the specified counter fields (by one by default). Creates the doc
|
||||||
<b>Signature:</b>
|
<b>Signature:</b>
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions): Promise<SavedObject<T>>;
|
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions<T>): Promise<SavedObject<T>>;
|
||||||
```
|
```
|
||||||
|
|
||||||
## Parameters
|
## Parameters
|
||||||
|
@ -19,7 +19,7 @@ incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<str
|
||||||
| type | <code>string</code> | The type of saved object whose fields should be incremented |
|
| type | <code>string</code> | The type of saved object whose fields should be incremented |
|
||||||
| id | <code>string</code> | The id of the document whose fields should be incremented |
|
| id | <code>string</code> | The id of the document whose fields should be incremented |
|
||||||
| counterFields | <code>Array<string | SavedObjectsIncrementCounterField></code> | An array of field names to increment or an array of [SavedObjectsIncrementCounterField](./kibana-plugin-core-server.savedobjectsincrementcounterfield.md) |
|
| counterFields | <code>Array<string | SavedObjectsIncrementCounterField></code> | An array of field names to increment or an array of [SavedObjectsIncrementCounterField](./kibana-plugin-core-server.savedobjectsincrementcounterfield.md) |
|
||||||
| options | <code>SavedObjectsIncrementCounterOptions</code> | [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
|
| options | <code>SavedObjectsIncrementCounterOptions<T></code> | [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
|
||||||
|
|
||||||
<b>Returns:</b>
|
<b>Returns:</b>
|
||||||
|
|
||||||
|
@ -52,5 +52,19 @@ repository
|
||||||
'stats.apiCalls',
|
'stats.apiCalls',
|
||||||
])
|
])
|
||||||
|
|
||||||
|
// Increment the apiCalls field counter by 4
|
||||||
|
repository
|
||||||
|
.incrementCounter('dashboard_counter_type', 'counter_id', [
|
||||||
|
{ fieldName: 'stats.apiCalls' incrementBy: 4 },
|
||||||
|
])
|
||||||
|
|
||||||
|
// Initialize the document with arbitrary fields if not present
|
||||||
|
repository.incrementCounter<{ appId: string }>(
|
||||||
|
'dashboard_counter_type',
|
||||||
|
'counter_id',
|
||||||
|
[ 'stats.apiCalls'],
|
||||||
|
{ upsertAttributes: { appId: 'myId' } }
|
||||||
|
)
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
@ -23,6 +23,7 @@ import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock
|
||||||
import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks';
|
import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks';
|
||||||
import { esKuery } from '../../es_query';
|
import { esKuery } from '../../es_query';
|
||||||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||||
|
|
||||||
const { nodeTypes } = esKuery;
|
const { nodeTypes } = esKuery;
|
||||||
|
|
||||||
jest.mock('./search_dsl/search_dsl', () => ({ getSearchDsl: jest.fn() }));
|
jest.mock('./search_dsl/search_dsl', () => ({ getSearchDsl: jest.fn() }));
|
||||||
|
@ -3654,6 +3655,33 @@ describe('SavedObjectsRepository', () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it(`uses the 'upsertAttributes' option when specified`, async () => {
|
||||||
|
const upsertAttributes = {
|
||||||
|
foo: 'bar',
|
||||||
|
hello: 'dolly',
|
||||||
|
};
|
||||||
|
await incrementCounterSuccess(type, id, counterFields, { namespace, upsertAttributes });
|
||||||
|
expect(client.update).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
body: expect.objectContaining({
|
||||||
|
upsert: expect.objectContaining({
|
||||||
|
[type]: {
|
||||||
|
foo: 'bar',
|
||||||
|
hello: 'dolly',
|
||||||
|
...counterFields.reduce((aggs, field) => {
|
||||||
|
return {
|
||||||
|
...aggs,
|
||||||
|
[field]: 1,
|
||||||
|
};
|
||||||
|
}, {}),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
expect.anything()
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => {
|
it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => {
|
||||||
await incrementCounterSuccess(type, id, counterFields, { namespace });
|
await incrementCounterSuccess(type, id, counterFields, { namespace });
|
||||||
expect(client.update).toHaveBeenCalledWith(
|
expect(client.update).toHaveBeenCalledWith(
|
||||||
|
|
|
@ -76,10 +76,16 @@ import {
|
||||||
// BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository
|
// BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository
|
||||||
// so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient.
|
// so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient.
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
|
interface Left {
|
||||||
type Left = { tag: 'Left'; error: Record<string, any> };
|
tag: 'Left';
|
||||||
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
|
error: Record<string, any>;
|
||||||
type Right = { tag: 'Right'; value: Record<string, any> };
|
}
|
||||||
|
|
||||||
|
interface Right {
|
||||||
|
tag: 'Right';
|
||||||
|
value: Record<string, any>;
|
||||||
|
}
|
||||||
|
|
||||||
type Either = Left | Right;
|
type Either = Left | Right;
|
||||||
const isLeft = (either: Either): either is Left => either.tag === 'Left';
|
const isLeft = (either: Either): either is Left => either.tag === 'Left';
|
||||||
const isRight = (either: Either): either is Right => either.tag === 'Right';
|
const isRight = (either: Either): either is Right => either.tag === 'Right';
|
||||||
|
@ -98,7 +104,8 @@ export interface SavedObjectsRepositoryOptions {
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
*/
|
*/
|
||||||
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
|
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown>
|
||||||
|
extends SavedObjectsBaseOptions {
|
||||||
/**
|
/**
|
||||||
* (default=false) If true, sets all the counter fields to 0 if they don't
|
* (default=false) If true, sets all the counter fields to 0 if they don't
|
||||||
* already exist. Existing fields will be left as-is and won't be incremented.
|
* already exist. Existing fields will be left as-is and won't be incremented.
|
||||||
|
@ -111,6 +118,10 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
|
||||||
* operation. See {@link MutatingOperationRefreshSetting}
|
* operation. See {@link MutatingOperationRefreshSetting}
|
||||||
*/
|
*/
|
||||||
refresh?: MutatingOperationRefreshSetting;
|
refresh?: MutatingOperationRefreshSetting;
|
||||||
|
/**
|
||||||
|
* Attributes to use when upserting the document if it doesn't exist.
|
||||||
|
*/
|
||||||
|
upsertAttributes?: Attributes;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1694,6 +1705,20 @@ export class SavedObjectsRepository {
|
||||||
* .incrementCounter('dashboard_counter_type', 'counter_id', [
|
* .incrementCounter('dashboard_counter_type', 'counter_id', [
|
||||||
* 'stats.apiCalls',
|
* 'stats.apiCalls',
|
||||||
* ])
|
* ])
|
||||||
|
*
|
||||||
|
* // Increment the apiCalls field counter by 4
|
||||||
|
* repository
|
||||||
|
* .incrementCounter('dashboard_counter_type', 'counter_id', [
|
||||||
|
* { fieldName: 'stats.apiCalls' incrementBy: 4 },
|
||||||
|
* ])
|
||||||
|
*
|
||||||
|
* // Initialize the document with arbitrary fields if not present
|
||||||
|
* repository.incrementCounter<{ appId: string }>(
|
||||||
|
* 'dashboard_counter_type',
|
||||||
|
* 'counter_id',
|
||||||
|
* [ 'stats.apiCalls'],
|
||||||
|
* { upsertAttributes: { appId: 'myId' } }
|
||||||
|
* )
|
||||||
* ```
|
* ```
|
||||||
*
|
*
|
||||||
* @param type - The type of saved object whose fields should be incremented
|
* @param type - The type of saved object whose fields should be incremented
|
||||||
|
@ -1706,7 +1731,7 @@ export class SavedObjectsRepository {
|
||||||
type: string,
|
type: string,
|
||||||
id: string,
|
id: string,
|
||||||
counterFields: Array<string | SavedObjectsIncrementCounterField>,
|
counterFields: Array<string | SavedObjectsIncrementCounterField>,
|
||||||
options: SavedObjectsIncrementCounterOptions = {}
|
options: SavedObjectsIncrementCounterOptions<T> = {}
|
||||||
): Promise<SavedObject<T>> {
|
): Promise<SavedObject<T>> {
|
||||||
if (typeof type !== 'string') {
|
if (typeof type !== 'string') {
|
||||||
throw new Error('"type" argument must be a string');
|
throw new Error('"type" argument must be a string');
|
||||||
|
@ -1728,12 +1753,16 @@ export class SavedObjectsRepository {
|
||||||
throw SavedObjectsErrorHelpers.createUnsupportedTypeError(type);
|
throw SavedObjectsErrorHelpers.createUnsupportedTypeError(type);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { migrationVersion, refresh = DEFAULT_REFRESH_SETTING, initialize = false } = options;
|
const {
|
||||||
|
migrationVersion,
|
||||||
|
refresh = DEFAULT_REFRESH_SETTING,
|
||||||
|
initialize = false,
|
||||||
|
upsertAttributes,
|
||||||
|
} = options;
|
||||||
|
|
||||||
const normalizedCounterFields = counterFields.map((counterField) => {
|
const normalizedCounterFields = counterFields.map((counterField) => {
|
||||||
const fieldName = typeof counterField === 'string' ? counterField : counterField.fieldName;
|
const fieldName = typeof counterField === 'string' ? counterField : counterField.fieldName;
|
||||||
const incrementBy = typeof counterField === 'string' ? 1 : counterField.incrementBy || 1;
|
const incrementBy = typeof counterField === 'string' ? 1 : counterField.incrementBy || 1;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
fieldName,
|
fieldName,
|
||||||
incrementBy: initialize ? 0 : incrementBy,
|
incrementBy: initialize ? 0 : incrementBy,
|
||||||
|
@ -1757,11 +1786,14 @@ export class SavedObjectsRepository {
|
||||||
type,
|
type,
|
||||||
...(savedObjectNamespace && { namespace: savedObjectNamespace }),
|
...(savedObjectNamespace && { namespace: savedObjectNamespace }),
|
||||||
...(savedObjectNamespaces && { namespaces: savedObjectNamespaces }),
|
...(savedObjectNamespaces && { namespaces: savedObjectNamespaces }),
|
||||||
attributes: normalizedCounterFields.reduce((acc, counterField) => {
|
attributes: {
|
||||||
const { fieldName, incrementBy } = counterField;
|
...(upsertAttributes ?? {}),
|
||||||
acc[fieldName] = incrementBy;
|
...normalizedCounterFields.reduce((acc, counterField) => {
|
||||||
return acc;
|
const { fieldName, incrementBy } = counterField;
|
||||||
}, {} as Record<string, number>),
|
acc[fieldName] = incrementBy;
|
||||||
|
return acc;
|
||||||
|
}, {} as Record<string, number>),
|
||||||
|
},
|
||||||
migrationVersion,
|
migrationVersion,
|
||||||
updated_at: time,
|
updated_at: time,
|
||||||
});
|
});
|
||||||
|
|
|
@ -2735,11 +2735,12 @@ export interface SavedObjectsIncrementCounterField {
|
||||||
}
|
}
|
||||||
|
|
||||||
// @public (undocumented)
|
// @public (undocumented)
|
||||||
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
|
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown> extends SavedObjectsBaseOptions {
|
||||||
initialize?: boolean;
|
initialize?: boolean;
|
||||||
// (undocumented)
|
// (undocumented)
|
||||||
migrationVersion?: SavedObjectsMigrationVersion;
|
migrationVersion?: SavedObjectsMigrationVersion;
|
||||||
refresh?: MutatingOperationRefreshSetting;
|
refresh?: MutatingOperationRefreshSetting;
|
||||||
|
upsertAttributes?: Attributes;
|
||||||
}
|
}
|
||||||
|
|
||||||
// @public
|
// @public
|
||||||
|
@ -2839,7 +2840,7 @@ export class SavedObjectsRepository {
|
||||||
// (undocumented)
|
// (undocumented)
|
||||||
find<T = unknown>(options: SavedObjectsFindOptions): Promise<SavedObjectsFindResponse<T>>;
|
find<T = unknown>(options: SavedObjectsFindOptions): Promise<SavedObjectsFindResponse<T>>;
|
||||||
get<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObject<T>>;
|
get<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObject<T>>;
|
||||||
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions): Promise<SavedObject<T>>;
|
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions<T>): Promise<SavedObject<T>>;
|
||||||
openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise<SavedObjectsOpenPointInTimeResponse>;
|
openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise<SavedObjectsOpenPointInTimeResponse>;
|
||||||
removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise<SavedObjectsRemoveReferencesToResponse>;
|
removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise<SavedObjectsRemoveReferencesToResponse>;
|
||||||
resolve<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObjectsResolveResponse<T>>;
|
resolve<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObjectsResolveResponse<T>>;
|
||||||
|
|
|
@ -12,15 +12,9 @@
|
||||||
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
|
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Roll daily indices every 30 minutes.
|
* Roll daily indices every 24h
|
||||||
* This means that, assuming a user can visit all the 44 apps we can possibly report
|
|
||||||
* in the 3 minutes interval the browser reports to the server, up to 22 users can have the same
|
|
||||||
* behaviour and we wouldn't need to paginate in the transactional documents (less than 10k docs).
|
|
||||||
*
|
|
||||||
* Based on a more normal expected use case, the users could visit up to 5 apps in those 3 minutes,
|
|
||||||
* allowing up to 200 users before reaching the limit.
|
|
||||||
*/
|
*/
|
||||||
export const ROLL_DAILY_INDICES_INTERVAL = 30 * 60 * 1000;
|
export const ROLL_DAILY_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start rolling indices after 5 minutes up
|
* Start rolling indices after 5 minutes up
|
||||||
|
|
|
@ -7,3 +7,4 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';
|
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';
|
||||||
|
export { rollDailyData as migrateTransactionalDocs } from './rollups';
|
||||||
|
|
|
@ -6,21 +6,16 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { rollDailyData, rollTotals } from './rollups';
|
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
|
||||||
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
|
import { SavedObjectsErrorHelpers } from '../../../../../../core/server';
|
||||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TRANSACTIONAL_TYPE } from '../saved_objects_types';
|
||||||
import { SavedObjectsErrorHelpers } from '../../../../../core/server';
|
import { rollDailyData } from './daily';
|
||||||
import {
|
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
|
||||||
SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
|
||||||
} from './saved_objects_types';
|
|
||||||
|
|
||||||
describe('rollDailyData', () => {
|
describe('rollDailyData', () => {
|
||||||
const logger = loggingSystemMock.createLogger();
|
const logger = loggingSystemMock.createLogger();
|
||||||
|
|
||||||
test('returns undefined if no savedObjectsClient initialised yet', async () => {
|
test('returns false if no savedObjectsClient initialised yet', async () => {
|
||||||
await expect(rollDailyData(logger, undefined)).resolves.toBe(undefined);
|
await expect(rollDailyData(logger, undefined)).resolves.toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('handle empty results', async () => {
|
test('handle empty results', async () => {
|
||||||
|
@ -33,7 +28,7 @@ describe('rollDailyData', () => {
|
||||||
throw new Error(`Unexpected type [${type}]`);
|
throw new Error(`Unexpected type [${type}]`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
|
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
|
||||||
expect(savedObjectClient.get).not.toBeCalled();
|
expect(savedObjectClient.get).not.toBeCalled();
|
||||||
expect(savedObjectClient.bulkCreate).not.toBeCalled();
|
expect(savedObjectClient.bulkCreate).not.toBeCalled();
|
||||||
expect(savedObjectClient.delete).not.toBeCalled();
|
expect(savedObjectClient.delete).not.toBeCalled();
|
||||||
|
@ -101,7 +96,7 @@ describe('rollDailyData', () => {
|
||||||
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
|
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
|
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
|
||||||
expect(savedObjectClient.get).toHaveBeenCalledTimes(2);
|
expect(savedObjectClient.get).toHaveBeenCalledTimes(2);
|
||||||
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
|
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
|
||||||
1,
|
1,
|
||||||
|
@ -196,7 +191,7 @@ describe('rollDailyData', () => {
|
||||||
throw new Error('Something went terribly wrong');
|
throw new Error('Something went terribly wrong');
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
|
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(false);
|
||||||
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
|
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
|
||||||
expect(savedObjectClient.get).toHaveBeenCalledWith(
|
expect(savedObjectClient.get).toHaveBeenCalledWith(
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
@ -206,185 +201,3 @@ describe('rollDailyData', () => {
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('rollTotals', () => {
|
|
||||||
const logger = loggingSystemMock.createLogger();
|
|
||||||
|
|
||||||
test('returns undefined if no savedObjectsClient initialised yet', async () => {
|
|
||||||
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('handle empty results', async () => {
|
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
|
||||||
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
|
||||||
switch (type) {
|
|
||||||
case SAVED_OBJECTS_DAILY_TYPE:
|
|
||||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
|
||||||
return { saved_objects: [], total: 0, page, per_page: perPage };
|
|
||||||
default:
|
|
||||||
throw new Error(`Unexpected type [${type}]`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
|
||||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('migrate some documents', async () => {
|
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
|
||||||
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
|
||||||
switch (type) {
|
|
||||||
case SAVED_OBJECTS_DAILY_TYPE:
|
|
||||||
return {
|
|
||||||
saved_objects: [
|
|
||||||
{
|
|
||||||
id: 'appId-2:2020-01-01',
|
|
||||||
type,
|
|
||||||
score: 0,
|
|
||||||
references: [],
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-2',
|
|
||||||
timestamp: '2020-01-01T10:31:00.000Z',
|
|
||||||
minutesOnScreen: 0.5,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'appId-1:2020-01-01',
|
|
||||||
type,
|
|
||||||
score: 0,
|
|
||||||
references: [],
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-1',
|
|
||||||
timestamp: '2020-01-01T11:31:00.000Z',
|
|
||||||
minutesOnScreen: 2.5,
|
|
||||||
numberOfClicks: 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'appId-1:2020-01-01:viewId-1',
|
|
||||||
type,
|
|
||||||
score: 0,
|
|
||||||
references: [],
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-1',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
timestamp: '2020-01-01T11:31:00.000Z',
|
|
||||||
minutesOnScreen: 1,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
total: 3,
|
|
||||||
page,
|
|
||||||
per_page: perPage,
|
|
||||||
};
|
|
||||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
|
||||||
return {
|
|
||||||
saved_objects: [
|
|
||||||
{
|
|
||||||
id: 'appId-1',
|
|
||||||
type,
|
|
||||||
score: 0,
|
|
||||||
references: [],
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-1',
|
|
||||||
minutesOnScreen: 0.5,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'appId-1___viewId-1',
|
|
||||||
type,
|
|
||||||
score: 0,
|
|
||||||
references: [],
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-1',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
minutesOnScreen: 4,
|
|
||||||
numberOfClicks: 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'appId-2___viewId-1',
|
|
||||||
type,
|
|
||||||
score: 0,
|
|
||||||
references: [],
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-2',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
minutesOnScreen: 1,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
total: 3,
|
|
||||||
page,
|
|
||||||
per_page: perPage,
|
|
||||||
};
|
|
||||||
default:
|
|
||||||
throw new Error(`Unexpected type [${type}]`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
|
||||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
|
|
||||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
id: 'appId-1',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-1',
|
|
||||||
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
|
||||||
minutesOnScreen: 3.0,
|
|
||||||
numberOfClicks: 3,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
id: 'appId-1___viewId-1',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-1',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
minutesOnScreen: 5.0,
|
|
||||||
numberOfClicks: 3,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
id: 'appId-2___viewId-1',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-2',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
minutesOnScreen: 1.0,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
id: 'appId-2',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId-2',
|
|
||||||
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
|
||||||
minutesOnScreen: 0.5,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
{ overwrite: true }
|
|
||||||
);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
|
||||||
'appId-2:2020-01-01'
|
|
||||||
);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
|
||||||
'appId-1:2020-01-01'
|
|
||||||
);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
|
||||||
'appId-1:2020-01-01:viewId-1'
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
|
@ -6,18 +6,20 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ISavedObjectsRepository, SavedObject, Logger } from 'kibana/server';
|
|
||||||
import moment from 'moment';
|
import moment from 'moment';
|
||||||
|
import type { Logger } from '@kbn/logging';
|
||||||
|
import {
|
||||||
|
ISavedObjectsRepository,
|
||||||
|
SavedObject,
|
||||||
|
SavedObjectsErrorHelpers,
|
||||||
|
} from '../../../../../../core/server';
|
||||||
|
import { getDailyId } from '../../../../../usage_collection/common/application_usage';
|
||||||
import {
|
import {
|
||||||
ApplicationUsageDaily,
|
ApplicationUsageDaily,
|
||||||
ApplicationUsageTotal,
|
|
||||||
ApplicationUsageTransactional,
|
ApplicationUsageTransactional,
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||||
} from './saved_objects_types';
|
} from '../saved_objects_types';
|
||||||
import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
|
|
||||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
|
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
|
||||||
|
@ -27,18 +29,17 @@ type ApplicationUsageDailyWithVersion = Pick<
|
||||||
'version' | 'attributes'
|
'version' | 'attributes'
|
||||||
>;
|
>;
|
||||||
|
|
||||||
export function serializeKey(appId: string, viewId: string) {
|
|
||||||
return `${appId}___${viewId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Aggregates all the transactional events into daily aggregates
|
* Aggregates all the transactional events into daily aggregates
|
||||||
* @param logger
|
* @param logger
|
||||||
* @param savedObjectsClient
|
* @param savedObjectsClient
|
||||||
*/
|
*/
|
||||||
export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
|
export async function rollDailyData(
|
||||||
|
logger: Logger,
|
||||||
|
savedObjectsClient?: ISavedObjectsRepository
|
||||||
|
): Promise<boolean> {
|
||||||
if (!savedObjectsClient) {
|
if (!savedObjectsClient) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -58,10 +59,7 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
|
||||||
} = doc;
|
} = doc;
|
||||||
const dayId = moment(timestamp).format('YYYY-MM-DD');
|
const dayId = moment(timestamp).format('YYYY-MM-DD');
|
||||||
|
|
||||||
const dailyId =
|
const dailyId = getDailyId({ dayId, appId, viewId });
|
||||||
!viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
|
|
||||||
? `${appId}:${dayId}`
|
|
||||||
: `${appId}:${dayId}:${viewId}`;
|
|
||||||
|
|
||||||
const existingDoc =
|
const existingDoc =
|
||||||
toCreate.get(dailyId) ||
|
toCreate.get(dailyId) ||
|
||||||
|
@ -103,9 +101,11 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} while (toCreate.size > 0);
|
} while (toCreate.size > 0);
|
||||||
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.debug(`Failed to rollup transactional to daily entries`);
|
logger.debug(`Failed to rollup transactional to daily entries`);
|
||||||
logger.debug(err);
|
logger.debug(err);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,7 +125,11 @@ async function getDailyDoc(
|
||||||
dayId: string
|
dayId: string
|
||||||
): Promise<ApplicationUsageDailyWithVersion> {
|
): Promise<ApplicationUsageDailyWithVersion> {
|
||||||
try {
|
try {
|
||||||
return await savedObjectsClient.get<ApplicationUsageDaily>(SAVED_OBJECTS_DAILY_TYPE, id);
|
const { attributes, version } = await savedObjectsClient.get<ApplicationUsageDaily>(
|
||||||
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
id
|
||||||
|
);
|
||||||
|
return { attributes, version };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
|
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
|
||||||
return {
|
return {
|
||||||
|
@ -142,91 +146,3 @@ async function getDailyDoc(
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
|
|
||||||
* @param logger
|
|
||||||
* @param savedObjectsClient
|
|
||||||
*/
|
|
||||||
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
|
|
||||||
if (!savedObjectsClient) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const [
|
|
||||||
{ saved_objects: rawApplicationUsageTotals },
|
|
||||||
{ saved_objects: rawApplicationUsageDaily },
|
|
||||||
] = await Promise.all([
|
|
||||||
savedObjectsClient.find<ApplicationUsageTotal>({
|
|
||||||
perPage: 10000,
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
}),
|
|
||||||
savedObjectsClient.find<ApplicationUsageDaily>({
|
|
||||||
perPage: 10000,
|
|
||||||
type: SAVED_OBJECTS_DAILY_TYPE,
|
|
||||||
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const existingTotals = rawApplicationUsageTotals.reduce(
|
|
||||||
(
|
|
||||||
acc,
|
|
||||||
{
|
|
||||||
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
|
|
||||||
}
|
|
||||||
) => {
|
|
||||||
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
|
||||||
|
|
||||||
return {
|
|
||||||
...acc,
|
|
||||||
// No need to sum because there should be 1 document per appId only
|
|
||||||
[key]: { appId, viewId, numberOfClicks, minutesOnScreen },
|
|
||||||
};
|
|
||||||
},
|
|
||||||
{} as Record<
|
|
||||||
string,
|
|
||||||
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
|
|
||||||
>
|
|
||||||
);
|
|
||||||
|
|
||||||
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
|
|
||||||
const {
|
|
||||||
appId,
|
|
||||||
viewId = MAIN_APP_DEFAULT_VIEW_ID,
|
|
||||||
numberOfClicks,
|
|
||||||
minutesOnScreen,
|
|
||||||
} = attributes;
|
|
||||||
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
|
||||||
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
|
|
||||||
|
|
||||||
return {
|
|
||||||
...acc,
|
|
||||||
[key]: {
|
|
||||||
appId,
|
|
||||||
viewId,
|
|
||||||
numberOfClicks: numberOfClicks + existing.numberOfClicks,
|
|
||||||
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}, existingTotals);
|
|
||||||
|
|
||||||
await Promise.all([
|
|
||||||
Object.entries(totals).length &&
|
|
||||||
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
|
|
||||||
Object.entries(totals).map(([id, entry]) => ({
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
id,
|
|
||||||
attributes: entry,
|
|
||||||
})),
|
|
||||||
{ overwrite: true }
|
|
||||||
),
|
|
||||||
...rawApplicationUsageDaily.map(
|
|
||||||
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
} catch (err) {
|
|
||||||
logger.debug(`Failed to rollup daily entries to totals`);
|
|
||||||
logger.debug(err);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export { rollDailyData } from './daily';
|
||||||
|
export { rollTotals } from './total';
|
||||||
|
export { serializeKey } from './utils';
|
|
@ -0,0 +1,194 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
|
||||||
|
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
|
||||||
|
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from '../saved_objects_types';
|
||||||
|
import { rollTotals } from './total';
|
||||||
|
|
||||||
|
describe('rollTotals', () => {
|
||||||
|
const logger = loggingSystemMock.createLogger();
|
||||||
|
|
||||||
|
test('returns undefined if no savedObjectsClient initialised yet', async () => {
|
||||||
|
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('handle empty results', async () => {
|
||||||
|
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||||
|
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
||||||
|
switch (type) {
|
||||||
|
case SAVED_OBJECTS_DAILY_TYPE:
|
||||||
|
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||||
|
return { saved_objects: [], total: 0, page, per_page: perPage };
|
||||||
|
default:
|
||||||
|
throw new Error(`Unexpected type [${type}]`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||||
|
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
|
||||||
|
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('migrate some documents', async () => {
|
||||||
|
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||||
|
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
||||||
|
switch (type) {
|
||||||
|
case SAVED_OBJECTS_DAILY_TYPE:
|
||||||
|
return {
|
||||||
|
saved_objects: [
|
||||||
|
{
|
||||||
|
id: 'appId-2:2020-01-01',
|
||||||
|
type,
|
||||||
|
score: 0,
|
||||||
|
references: [],
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-2',
|
||||||
|
timestamp: '2020-01-01T10:31:00.000Z',
|
||||||
|
minutesOnScreen: 0.5,
|
||||||
|
numberOfClicks: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'appId-1:2020-01-01',
|
||||||
|
type,
|
||||||
|
score: 0,
|
||||||
|
references: [],
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-1',
|
||||||
|
timestamp: '2020-01-01T11:31:00.000Z',
|
||||||
|
minutesOnScreen: 2.5,
|
||||||
|
numberOfClicks: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'appId-1:2020-01-01:viewId-1',
|
||||||
|
type,
|
||||||
|
score: 0,
|
||||||
|
references: [],
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-1',
|
||||||
|
viewId: 'viewId-1',
|
||||||
|
timestamp: '2020-01-01T11:31:00.000Z',
|
||||||
|
minutesOnScreen: 1,
|
||||||
|
numberOfClicks: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 3,
|
||||||
|
page,
|
||||||
|
per_page: perPage,
|
||||||
|
};
|
||||||
|
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||||
|
return {
|
||||||
|
saved_objects: [
|
||||||
|
{
|
||||||
|
id: 'appId-1',
|
||||||
|
type,
|
||||||
|
score: 0,
|
||||||
|
references: [],
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-1',
|
||||||
|
minutesOnScreen: 0.5,
|
||||||
|
numberOfClicks: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'appId-1___viewId-1',
|
||||||
|
type,
|
||||||
|
score: 0,
|
||||||
|
references: [],
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-1',
|
||||||
|
viewId: 'viewId-1',
|
||||||
|
minutesOnScreen: 4,
|
||||||
|
numberOfClicks: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'appId-2___viewId-1',
|
||||||
|
type,
|
||||||
|
score: 0,
|
||||||
|
references: [],
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-2',
|
||||||
|
viewId: 'viewId-1',
|
||||||
|
minutesOnScreen: 1,
|
||||||
|
numberOfClicks: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 3,
|
||||||
|
page,
|
||||||
|
per_page: perPage,
|
||||||
|
};
|
||||||
|
default:
|
||||||
|
throw new Error(`Unexpected type [${type}]`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||||
|
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
|
||||||
|
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
id: 'appId-1',
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-1',
|
||||||
|
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
||||||
|
minutesOnScreen: 3.0,
|
||||||
|
numberOfClicks: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
id: 'appId-1___viewId-1',
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-1',
|
||||||
|
viewId: 'viewId-1',
|
||||||
|
minutesOnScreen: 5.0,
|
||||||
|
numberOfClicks: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
id: 'appId-2___viewId-1',
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-2',
|
||||||
|
viewId: 'viewId-1',
|
||||||
|
minutesOnScreen: 1.0,
|
||||||
|
numberOfClicks: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
id: 'appId-2',
|
||||||
|
attributes: {
|
||||||
|
appId: 'appId-2',
|
||||||
|
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
||||||
|
minutesOnScreen: 0.5,
|
||||||
|
numberOfClicks: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{ overwrite: true }
|
||||||
|
);
|
||||||
|
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
|
||||||
|
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||||
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
'appId-2:2020-01-01'
|
||||||
|
);
|
||||||
|
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||||
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
'appId-1:2020-01-01'
|
||||||
|
);
|
||||||
|
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||||
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
'appId-1:2020-01-01:viewId-1'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
|
@ -0,0 +1,106 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { Logger } from '@kbn/logging';
|
||||||
|
import type { ISavedObjectsRepository } from 'kibana/server';
|
||||||
|
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
|
||||||
|
import {
|
||||||
|
ApplicationUsageDaily,
|
||||||
|
ApplicationUsageTotal,
|
||||||
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
} from '../saved_objects_types';
|
||||||
|
import { serializeKey } from './utils';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
|
||||||
|
* @param logger
|
||||||
|
* @param savedObjectsClient
|
||||||
|
*/
|
||||||
|
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
|
||||||
|
if (!savedObjectsClient) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const [
|
||||||
|
{ saved_objects: rawApplicationUsageTotals },
|
||||||
|
{ saved_objects: rawApplicationUsageDaily },
|
||||||
|
] = await Promise.all([
|
||||||
|
savedObjectsClient.find<ApplicationUsageTotal>({
|
||||||
|
perPage: 10000,
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
}),
|
||||||
|
savedObjectsClient.find<ApplicationUsageDaily>({
|
||||||
|
perPage: 10000,
|
||||||
|
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const existingTotals = rawApplicationUsageTotals.reduce(
|
||||||
|
(
|
||||||
|
acc,
|
||||||
|
{
|
||||||
|
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
|
||||||
|
}
|
||||||
|
) => {
|
||||||
|
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
||||||
|
|
||||||
|
return {
|
||||||
|
...acc,
|
||||||
|
// No need to sum because there should be 1 document per appId only
|
||||||
|
[key]: { appId, viewId, numberOfClicks, minutesOnScreen },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
{} as Record<
|
||||||
|
string,
|
||||||
|
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
|
||||||
|
>
|
||||||
|
);
|
||||||
|
|
||||||
|
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
|
||||||
|
const {
|
||||||
|
appId,
|
||||||
|
viewId = MAIN_APP_DEFAULT_VIEW_ID,
|
||||||
|
numberOfClicks,
|
||||||
|
minutesOnScreen,
|
||||||
|
} = attributes;
|
||||||
|
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
||||||
|
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
|
||||||
|
|
||||||
|
return {
|
||||||
|
...acc,
|
||||||
|
[key]: {
|
||||||
|
appId,
|
||||||
|
viewId,
|
||||||
|
numberOfClicks: numberOfClicks + existing.numberOfClicks,
|
||||||
|
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}, existingTotals);
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
Object.entries(totals).length &&
|
||||||
|
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
|
||||||
|
Object.entries(totals).map(([id, entry]) => ({
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
id,
|
||||||
|
attributes: entry,
|
||||||
|
})),
|
||||||
|
{ overwrite: true }
|
||||||
|
),
|
||||||
|
...rawApplicationUsageDaily.map(
|
||||||
|
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
} catch (err) {
|
||||||
|
logger.debug(`Failed to rollup daily entries to totals`);
|
||||||
|
logger.debug(err);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function serializeKey(appId: string, viewId: string) {
|
||||||
|
return `${appId}___${viewId}`;
|
||||||
|
}
|
|
@ -6,7 +6,7 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
|
import type { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Used for accumulating the totals of all the stats older than 90d
|
* Used for accumulating the totals of all the stats older than 90d
|
||||||
|
@ -17,6 +17,7 @@ export interface ApplicationUsageTotal extends SavedObjectAttributes {
|
||||||
minutesOnScreen: number;
|
minutesOnScreen: number;
|
||||||
numberOfClicks: number;
|
numberOfClicks: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
|
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -25,6 +26,8 @@ export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
|
||||||
export interface ApplicationUsageTransactional extends ApplicationUsageTotal {
|
export interface ApplicationUsageTransactional extends ApplicationUsageTotal {
|
||||||
timestamp: string;
|
timestamp: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @deprecated transactional type is no longer used, and only preserved for backward compatibility */
|
||||||
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
|
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -62,6 +65,7 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
|
||||||
});
|
});
|
||||||
|
|
||||||
// Type for storing ApplicationUsageTransactional (declaring empty mappings because we don't use the internal fields for query/aggregations)
|
// Type for storing ApplicationUsageTransactional (declaring empty mappings because we don't use the internal fields for query/aggregations)
|
||||||
|
// Remark: this type is deprecated and only here for BWC reasons.
|
||||||
registerType({
|
registerType({
|
||||||
name: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
name: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||||
hidden: false,
|
hidden: false,
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
|
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
|
||||||
import { ApplicationUsageTelemetryReport } from './telemetry_application_usage_collector';
|
import { ApplicationUsageTelemetryReport } from './types';
|
||||||
|
|
||||||
const commonSchema: MakeSchemaFrom<ApplicationUsageTelemetryReport[string]> = {
|
const commonSchema: MakeSchemaFrom<ApplicationUsageTelemetryReport[string]> = {
|
||||||
appId: { type: 'keyword', _meta: { description: 'The application being tracked' } },
|
appId: { type: 'keyword', _meta: { description: 'The application being tracked' } },
|
||||||
|
|
|
@ -11,74 +11,99 @@ import {
|
||||||
Collector,
|
Collector,
|
||||||
createUsageCollectionSetupMock,
|
createUsageCollectionSetupMock,
|
||||||
} from '../../../../usage_collection/server/usage_collection.mock';
|
} from '../../../../usage_collection/server/usage_collection.mock';
|
||||||
|
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||||
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
|
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
|
||||||
import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
|
|
||||||
import {
|
import {
|
||||||
registerApplicationUsageCollector,
|
registerApplicationUsageCollector,
|
||||||
transformByApplicationViews,
|
transformByApplicationViews,
|
||||||
ApplicationUsageViews,
|
|
||||||
} from './telemetry_application_usage_collector';
|
} from './telemetry_application_usage_collector';
|
||||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
import { ApplicationUsageViews } from './types';
|
||||||
import {
|
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from './saved_objects_types';
|
||||||
SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
// use fake timers to avoid triggering rollups during tests
|
||||||
} from './saved_objects_types';
|
jest.useFakeTimers();
|
||||||
|
|
||||||
describe('telemetry_application_usage', () => {
|
describe('telemetry_application_usage', () => {
|
||||||
jest.useFakeTimers();
|
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
|
||||||
|
|
||||||
const logger = loggingSystemMock.createLogger();
|
|
||||||
|
|
||||||
let collector: Collector<unknown>;
|
let collector: Collector<unknown>;
|
||||||
|
let usageCollectionMock: ReturnType<typeof createUsageCollectionSetupMock>;
|
||||||
|
let savedObjectClient: ReturnType<typeof savedObjectsRepositoryMock.create>;
|
||||||
|
let getSavedObjectClient: jest.MockedFunction<() => undefined | typeof savedObjectClient>;
|
||||||
|
|
||||||
const usageCollectionMock = createUsageCollectionSetupMock();
|
|
||||||
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
|
|
||||||
collector = new Collector(logger, config);
|
|
||||||
return createUsageCollectionSetupMock().makeUsageCollector(config);
|
|
||||||
});
|
|
||||||
|
|
||||||
const getUsageCollector = jest.fn();
|
|
||||||
const registerType = jest.fn();
|
const registerType = jest.fn();
|
||||||
const mockedFetchContext = createCollectorFetchContextMock();
|
const mockedFetchContext = createCollectorFetchContextMock();
|
||||||
|
|
||||||
beforeAll(() =>
|
beforeEach(() => {
|
||||||
registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector)
|
logger = loggingSystemMock.createLogger();
|
||||||
);
|
usageCollectionMock = createUsageCollectionSetupMock();
|
||||||
afterAll(() => jest.clearAllTimers());
|
savedObjectClient = savedObjectsRepositoryMock.create();
|
||||||
|
getSavedObjectClient = jest.fn().mockReturnValue(savedObjectClient);
|
||||||
|
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
|
||||||
|
collector = new Collector(logger, config);
|
||||||
|
return createUsageCollectionSetupMock().makeUsageCollector(config);
|
||||||
|
});
|
||||||
|
registerApplicationUsageCollector(
|
||||||
|
logger,
|
||||||
|
usageCollectionMock,
|
||||||
|
registerType,
|
||||||
|
getSavedObjectClient
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.clearAllTimers();
|
||||||
|
});
|
||||||
|
|
||||||
test('registered collector is set', () => {
|
test('registered collector is set', () => {
|
||||||
expect(collector).not.toBeUndefined();
|
expect(collector).not.toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('if no savedObjectClient initialised, return undefined', async () => {
|
test('if no savedObjectClient initialised, return undefined', async () => {
|
||||||
|
getSavedObjectClient.mockReturnValue(undefined);
|
||||||
|
|
||||||
expect(collector.isReady()).toBe(false);
|
expect(collector.isReady()).toBe(false);
|
||||||
expect(await collector.fetch(mockedFetchContext)).toBeUndefined();
|
expect(await collector.fetch(mockedFetchContext)).toBeUndefined();
|
||||||
jest.runTimersToTime(ROLL_INDICES_START);
|
});
|
||||||
|
|
||||||
|
test('calls `savedObjectsClient.find` with the correct parameters', async () => {
|
||||||
|
savedObjectClient.find.mockResolvedValue({
|
||||||
|
saved_objects: [],
|
||||||
|
total: 0,
|
||||||
|
per_page: 20,
|
||||||
|
page: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
await collector.fetch(mockedFetchContext);
|
||||||
|
|
||||||
|
expect(savedObjectClient.find).toHaveBeenCalledTimes(2);
|
||||||
|
|
||||||
|
expect(savedObjectClient.find).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(savedObjectClient.find).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||||
|
})
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('when savedObjectClient is initialised, return something', async () => {
|
test('when savedObjectClient is initialised, return something', async () => {
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
savedObjectClient.find.mockResolvedValue({
|
||||||
savedObjectClient.find.mockImplementation(
|
saved_objects: [],
|
||||||
async () =>
|
total: 0,
|
||||||
({
|
per_page: 20,
|
||||||
saved_objects: [],
|
page: 0,
|
||||||
total: 0,
|
});
|
||||||
} as any)
|
|
||||||
);
|
|
||||||
getUsageCollector.mockImplementation(() => savedObjectClient);
|
|
||||||
|
|
||||||
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
|
|
||||||
|
|
||||||
expect(collector.isReady()).toBe(true);
|
expect(collector.isReady()).toBe(true);
|
||||||
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({});
|
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({});
|
||||||
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
|
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('it only gets 10k even when there are more documents (ES limitation)', async () => {
|
test('it aggregates total and daily data', async () => {
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
|
||||||
const total = 10000;
|
|
||||||
savedObjectClient.find.mockImplementation(async (opts) => {
|
savedObjectClient.find.mockImplementation(async (opts) => {
|
||||||
switch (opts.type) {
|
switch (opts.type) {
|
||||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||||
|
@ -95,18 +120,6 @@ describe('telemetry_application_usage', () => {
|
||||||
],
|
],
|
||||||
total: 1,
|
total: 1,
|
||||||
} as any;
|
} as any;
|
||||||
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
|
|
||||||
const doc = {
|
|
||||||
id: 'test-id',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
minutesOnScreen: 0.5,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const savedObjects = new Array(total).fill(doc);
|
|
||||||
return { saved_objects: savedObjects, total: total + 1 };
|
|
||||||
case SAVED_OBJECTS_DAILY_TYPE:
|
case SAVED_OBJECTS_DAILY_TYPE:
|
||||||
return {
|
return {
|
||||||
saved_objects: [
|
saved_objects: [
|
||||||
|
@ -125,122 +138,21 @@ describe('telemetry_application_usage', () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
getUsageCollector.mockImplementation(() => savedObjectClient);
|
|
||||||
|
|
||||||
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
|
|
||||||
|
|
||||||
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
|
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
|
||||||
appId: {
|
appId: {
|
||||||
appId: 'appId',
|
appId: 'appId',
|
||||||
viewId: 'main',
|
viewId: 'main',
|
||||||
clicks_total: total + 1 + 10,
|
clicks_total: 1 + 10,
|
||||||
clicks_7_days: total + 1,
|
clicks_7_days: 1,
|
||||||
clicks_30_days: total + 1,
|
clicks_30_days: 1,
|
||||||
clicks_90_days: total + 1,
|
clicks_90_days: 1,
|
||||||
minutes_on_screen_total: (total + 1) * 0.5 + 10,
|
minutes_on_screen_total: 0.5 + 10,
|
||||||
minutes_on_screen_7_days: (total + 1) * 0.5,
|
minutes_on_screen_7_days: 0.5,
|
||||||
minutes_on_screen_30_days: (total + 1) * 0.5,
|
minutes_on_screen_30_days: 0.5,
|
||||||
minutes_on_screen_90_days: (total + 1) * 0.5,
|
minutes_on_screen_90_days: 0.5,
|
||||||
views: [],
|
views: [],
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
id: 'appId',
|
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId',
|
|
||||||
viewId: 'main',
|
|
||||||
minutesOnScreen: 10.5,
|
|
||||||
numberOfClicks: 11,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
{ overwrite: true }
|
|
||||||
);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
|
|
||||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
|
||||||
'appId:YYYY-MM-DD'
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('old transactional data not migrated yet', async () => {
|
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
|
||||||
savedObjectClient.find.mockImplementation(async (opts) => {
|
|
||||||
switch (opts.type) {
|
|
||||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
|
||||||
case SAVED_OBJECTS_DAILY_TYPE:
|
|
||||||
return { saved_objects: [], total: 0 } as any;
|
|
||||||
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
|
|
||||||
return {
|
|
||||||
saved_objects: [
|
|
||||||
{
|
|
||||||
id: 'test-id',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId',
|
|
||||||
timestamp: new Date(0).toISOString(),
|
|
||||||
minutesOnScreen: 0.5,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'test-id-2',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId',
|
|
||||||
viewId: 'main',
|
|
||||||
timestamp: new Date(0).toISOString(),
|
|
||||||
minutesOnScreen: 2,
|
|
||||||
numberOfClicks: 2,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'test-id-3',
|
|
||||||
attributes: {
|
|
||||||
appId: 'appId',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
timestamp: new Date(0).toISOString(),
|
|
||||||
minutesOnScreen: 1,
|
|
||||||
numberOfClicks: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
total: 1,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
getUsageCollector.mockImplementation(() => savedObjectClient);
|
|
||||||
|
|
||||||
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
|
|
||||||
appId: {
|
|
||||||
appId: 'appId',
|
|
||||||
viewId: 'main',
|
|
||||||
clicks_total: 3,
|
|
||||||
clicks_7_days: 0,
|
|
||||||
clicks_30_days: 0,
|
|
||||||
clicks_90_days: 0,
|
|
||||||
minutes_on_screen_total: 2.5,
|
|
||||||
minutes_on_screen_7_days: 0,
|
|
||||||
minutes_on_screen_30_days: 0,
|
|
||||||
minutes_on_screen_90_days: 0,
|
|
||||||
views: [
|
|
||||||
{
|
|
||||||
appId: 'appId',
|
|
||||||
viewId: 'viewId-1',
|
|
||||||
clicks_total: 1,
|
|
||||||
clicks_7_days: 0,
|
|
||||||
clicks_30_days: 0,
|
|
||||||
clicks_90_days: 0,
|
|
||||||
minutes_on_screen_total: 1,
|
|
||||||
minutes_on_screen_7_days: 0,
|
|
||||||
minutes_on_screen_30_days: 0,
|
|
||||||
minutes_on_screen_90_days: 0,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -11,57 +11,21 @@ import { timer } from 'rxjs';
|
||||||
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from 'kibana/server';
|
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from 'kibana/server';
|
||||||
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
|
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
|
||||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||||
import { serializeKey } from './rollups';
|
|
||||||
|
|
||||||
import {
|
import {
|
||||||
ApplicationUsageDaily,
|
ApplicationUsageDaily,
|
||||||
ApplicationUsageTotal,
|
ApplicationUsageTotal,
|
||||||
ApplicationUsageTransactional,
|
|
||||||
registerMappings,
|
registerMappings,
|
||||||
SAVED_OBJECTS_DAILY_TYPE,
|
SAVED_OBJECTS_DAILY_TYPE,
|
||||||
SAVED_OBJECTS_TOTAL_TYPE,
|
SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
|
||||||
} from './saved_objects_types';
|
} from './saved_objects_types';
|
||||||
import { applicationUsageSchema } from './schema';
|
import { applicationUsageSchema } from './schema';
|
||||||
import { rollDailyData, rollTotals } from './rollups';
|
import { rollTotals, rollDailyData, serializeKey } from './rollups';
|
||||||
import {
|
import {
|
||||||
ROLL_TOTAL_INDICES_INTERVAL,
|
ROLL_TOTAL_INDICES_INTERVAL,
|
||||||
ROLL_DAILY_INDICES_INTERVAL,
|
ROLL_DAILY_INDICES_INTERVAL,
|
||||||
ROLL_INDICES_START,
|
ROLL_INDICES_START,
|
||||||
} from './constants';
|
} from './constants';
|
||||||
|
import { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
|
||||||
export interface ApplicationViewUsage {
|
|
||||||
appId: string;
|
|
||||||
viewId: string;
|
|
||||||
clicks_total: number;
|
|
||||||
clicks_7_days: number;
|
|
||||||
clicks_30_days: number;
|
|
||||||
clicks_90_days: number;
|
|
||||||
minutes_on_screen_total: number;
|
|
||||||
minutes_on_screen_7_days: number;
|
|
||||||
minutes_on_screen_30_days: number;
|
|
||||||
minutes_on_screen_90_days: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ApplicationUsageViews {
|
|
||||||
[serializedKey: string]: ApplicationViewUsage;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ApplicationUsageTelemetryReport {
|
|
||||||
[appId: string]: {
|
|
||||||
appId: string;
|
|
||||||
viewId: string;
|
|
||||||
clicks_total: number;
|
|
||||||
clicks_7_days: number;
|
|
||||||
clicks_30_days: number;
|
|
||||||
clicks_90_days: number;
|
|
||||||
minutes_on_screen_total: number;
|
|
||||||
minutes_on_screen_7_days: number;
|
|
||||||
minutes_on_screen_30_days: number;
|
|
||||||
minutes_on_screen_90_days: number;
|
|
||||||
views?: ApplicationViewUsage[];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export const transformByApplicationViews = (
|
export const transformByApplicationViews = (
|
||||||
report: ApplicationUsageViews
|
report: ApplicationUsageViews
|
||||||
|
@ -92,6 +56,21 @@ export function registerApplicationUsageCollector(
|
||||||
) {
|
) {
|
||||||
registerMappings(registerType);
|
registerMappings(registerType);
|
||||||
|
|
||||||
|
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
|
||||||
|
rollTotals(logger, getSavedObjectsClient())
|
||||||
|
);
|
||||||
|
|
||||||
|
const dailyRollingSub = timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(
|
||||||
|
async () => {
|
||||||
|
const success = await rollDailyData(logger, getSavedObjectsClient());
|
||||||
|
// we only need to roll the transactional documents once to assure BWC
|
||||||
|
// once we rolling succeeds, we can stop.
|
||||||
|
if (success) {
|
||||||
|
dailyRollingSub.unsubscribe();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
const collector = usageCollection.makeUsageCollector<ApplicationUsageTelemetryReport | undefined>(
|
const collector = usageCollection.makeUsageCollector<ApplicationUsageTelemetryReport | undefined>(
|
||||||
{
|
{
|
||||||
type: 'application_usage',
|
type: 'application_usage',
|
||||||
|
@ -105,7 +84,6 @@ export function registerApplicationUsageCollector(
|
||||||
const [
|
const [
|
||||||
{ saved_objects: rawApplicationUsageTotals },
|
{ saved_objects: rawApplicationUsageTotals },
|
||||||
{ saved_objects: rawApplicationUsageDaily },
|
{ saved_objects: rawApplicationUsageDaily },
|
||||||
{ saved_objects: rawApplicationUsageTransactional },
|
|
||||||
] = await Promise.all([
|
] = await Promise.all([
|
||||||
savedObjectsClient.find<ApplicationUsageTotal>({
|
savedObjectsClient.find<ApplicationUsageTotal>({
|
||||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||||
|
@ -115,10 +93,6 @@ export function registerApplicationUsageCollector(
|
||||||
type: SAVED_OBJECTS_DAILY_TYPE,
|
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||||
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
|
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
|
||||||
}),
|
}),
|
||||||
savedObjectsClient.find<ApplicationUsageTransactional>({
|
|
||||||
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
|
||||||
perPage: 10000, // If we have more than those, we won't report the rest (they'll be rolled up to the daily soon enough to become a problem)
|
|
||||||
}),
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
|
const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
|
||||||
|
@ -156,10 +130,7 @@ export function registerApplicationUsageCollector(
|
||||||
const nowMinus30 = moment().subtract(30, 'days');
|
const nowMinus30 = moment().subtract(30, 'days');
|
||||||
const nowMinus90 = moment().subtract(90, 'days');
|
const nowMinus90 = moment().subtract(90, 'days');
|
||||||
|
|
||||||
const applicationUsage = [
|
const applicationUsage = rawApplicationUsageDaily.reduce(
|
||||||
...rawApplicationUsageDaily,
|
|
||||||
...rawApplicationUsageTransactional,
|
|
||||||
].reduce(
|
|
||||||
(
|
(
|
||||||
acc,
|
acc,
|
||||||
{
|
{
|
||||||
|
@ -224,11 +195,4 @@ export function registerApplicationUsageCollector(
|
||||||
);
|
);
|
||||||
|
|
||||||
usageCollection.registerCollector(collector);
|
usageCollection.registerCollector(collector);
|
||||||
|
|
||||||
timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(() =>
|
|
||||||
rollDailyData(logger, getSavedObjectsClient())
|
|
||||||
);
|
|
||||||
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
|
|
||||||
rollTotals(logger, getSavedObjectsClient())
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,40 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface ApplicationViewUsage {
|
||||||
|
appId: string;
|
||||||
|
viewId: string;
|
||||||
|
clicks_total: number;
|
||||||
|
clicks_7_days: number;
|
||||||
|
clicks_30_days: number;
|
||||||
|
clicks_90_days: number;
|
||||||
|
minutes_on_screen_total: number;
|
||||||
|
minutes_on_screen_7_days: number;
|
||||||
|
minutes_on_screen_30_days: number;
|
||||||
|
minutes_on_screen_90_days: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ApplicationUsageViews {
|
||||||
|
[serializedKey: string]: ApplicationViewUsage;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ApplicationUsageTelemetryReport {
|
||||||
|
[appId: string]: {
|
||||||
|
appId: string;
|
||||||
|
viewId: string;
|
||||||
|
clicks_total: number;
|
||||||
|
clicks_7_days: number;
|
||||||
|
clicks_30_days: number;
|
||||||
|
clicks_90_days: number;
|
||||||
|
minutes_on_screen_total: number;
|
||||||
|
minutes_on_screen_7_days: number;
|
||||||
|
minutes_on_screen_30_days: number;
|
||||||
|
minutes_on_screen_90_days: number;
|
||||||
|
views?: ApplicationViewUsage[];
|
||||||
|
};
|
||||||
|
}
|
23
src/plugins/usage_collection/common/application_usage.ts
Normal file
23
src/plugins/usage_collection/common/application_usage.ts
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { MAIN_APP_DEFAULT_VIEW_ID } from './constants';
|
||||||
|
|
||||||
|
export const getDailyId = ({
|
||||||
|
appId,
|
||||||
|
dayId,
|
||||||
|
viewId,
|
||||||
|
}: {
|
||||||
|
viewId: string;
|
||||||
|
appId: string;
|
||||||
|
dayId: string;
|
||||||
|
}) => {
|
||||||
|
return !viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
|
||||||
|
? `${appId}:${dayId}`
|
||||||
|
: `${appId}:${dayId}:${viewId}`;
|
||||||
|
};
|
|
@ -9,6 +9,13 @@
|
||||||
import { schema, TypeOf } from '@kbn/config-schema';
|
import { schema, TypeOf } from '@kbn/config-schema';
|
||||||
import { METRIC_TYPE } from '@kbn/analytics';
|
import { METRIC_TYPE } from '@kbn/analytics';
|
||||||
|
|
||||||
|
const applicationUsageReportSchema = schema.object({
|
||||||
|
minutesOnScreen: schema.number(),
|
||||||
|
numberOfClicks: schema.number(),
|
||||||
|
appId: schema.string(),
|
||||||
|
viewId: schema.string(),
|
||||||
|
});
|
||||||
|
|
||||||
export const reportSchema = schema.object({
|
export const reportSchema = schema.object({
|
||||||
reportVersion: schema.maybe(schema.oneOf([schema.literal(3)])),
|
reportVersion: schema.maybe(schema.oneOf([schema.literal(3)])),
|
||||||
userAgent: schema.maybe(
|
userAgent: schema.maybe(
|
||||||
|
@ -38,17 +45,8 @@ export const reportSchema = schema.object({
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
application_usage: schema.maybe(
|
application_usage: schema.maybe(schema.recordOf(schema.string(), applicationUsageReportSchema)),
|
||||||
schema.recordOf(
|
|
||||||
schema.string(),
|
|
||||||
schema.object({
|
|
||||||
minutesOnScreen: schema.number(),
|
|
||||||
numberOfClicks: schema.number(),
|
|
||||||
appId: schema.string(),
|
|
||||||
viewId: schema.string(),
|
|
||||||
})
|
|
||||||
)
|
|
||||||
),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
export type ReportSchemaType = TypeOf<typeof reportSchema>;
|
export type ReportSchemaType = TypeOf<typeof reportSchema>;
|
||||||
|
export type ApplicationUsageReport = TypeOf<typeof applicationUsageReportSchema>;
|
||||||
|
|
|
@ -0,0 +1,115 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import moment from 'moment';
|
||||||
|
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
|
||||||
|
import { getDailyId } from '../../common/application_usage';
|
||||||
|
import { storeApplicationUsage } from './store_application_usage';
|
||||||
|
import { ApplicationUsageReport } from './schema';
|
||||||
|
|
||||||
|
const createReport = (parts: Partial<ApplicationUsageReport>): ApplicationUsageReport => ({
|
||||||
|
appId: 'appId',
|
||||||
|
viewId: 'viewId',
|
||||||
|
numberOfClicks: 0,
|
||||||
|
minutesOnScreen: 0,
|
||||||
|
...parts,
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('storeApplicationUsage', () => {
|
||||||
|
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
|
||||||
|
let timestamp: Date;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
repository = savedObjectsRepositoryMock.create();
|
||||||
|
timestamp = new Date();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not call `repository.incrementUsageCounters` when the report list is empty', async () => {
|
||||||
|
await storeApplicationUsage(repository, [], timestamp);
|
||||||
|
expect(repository.incrementCounter).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('calls `repository.incrementUsageCounters` with the correct parameters', async () => {
|
||||||
|
const report = createReport({
|
||||||
|
appId: 'app1',
|
||||||
|
viewId: 'view1',
|
||||||
|
numberOfClicks: 2,
|
||||||
|
minutesOnScreen: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
await storeApplicationUsage(repository, [report], timestamp);
|
||||||
|
|
||||||
|
expect(repository.incrementCounter).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
expect(repository.incrementCounter).toHaveBeenCalledWith(
|
||||||
|
...expectedIncrementParams(report, timestamp)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('aggregates reports with the same appId/viewId tuple', async () => {
|
||||||
|
const report1 = createReport({
|
||||||
|
appId: 'app1',
|
||||||
|
viewId: 'view1',
|
||||||
|
numberOfClicks: 2,
|
||||||
|
minutesOnScreen: 5,
|
||||||
|
});
|
||||||
|
const report2 = createReport({
|
||||||
|
appId: 'app1',
|
||||||
|
viewId: 'view2',
|
||||||
|
numberOfClicks: 1,
|
||||||
|
minutesOnScreen: 7,
|
||||||
|
});
|
||||||
|
const report3 = createReport({
|
||||||
|
appId: 'app1',
|
||||||
|
viewId: 'view1',
|
||||||
|
numberOfClicks: 3,
|
||||||
|
minutesOnScreen: 9,
|
||||||
|
});
|
||||||
|
|
||||||
|
await storeApplicationUsage(repository, [report1, report2, report3], timestamp);
|
||||||
|
|
||||||
|
expect(repository.incrementCounter).toHaveBeenCalledTimes(2);
|
||||||
|
|
||||||
|
expect(repository.incrementCounter).toHaveBeenCalledWith(
|
||||||
|
...expectedIncrementParams(
|
||||||
|
{
|
||||||
|
appId: 'app1',
|
||||||
|
viewId: 'view1',
|
||||||
|
numberOfClicks: report1.numberOfClicks + report3.numberOfClicks,
|
||||||
|
minutesOnScreen: report1.minutesOnScreen + report3.minutesOnScreen,
|
||||||
|
},
|
||||||
|
timestamp
|
||||||
|
)
|
||||||
|
);
|
||||||
|
expect(repository.incrementCounter).toHaveBeenCalledWith(
|
||||||
|
...expectedIncrementParams(report2, timestamp)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const expectedIncrementParams = (
|
||||||
|
{ appId, viewId, minutesOnScreen, numberOfClicks }: ApplicationUsageReport,
|
||||||
|
timestamp: Date
|
||||||
|
) => {
|
||||||
|
const dayId = moment(timestamp).format('YYYY-MM-DD');
|
||||||
|
return [
|
||||||
|
'application_usage_daily',
|
||||||
|
getDailyId({ appId, viewId, dayId }),
|
||||||
|
[
|
||||||
|
{ fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
|
||||||
|
{ fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
|
||||||
|
],
|
||||||
|
{
|
||||||
|
upsertAttributes: {
|
||||||
|
appId,
|
||||||
|
viewId,
|
||||||
|
timestamp: moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
};
|
|
@ -0,0 +1,87 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import moment from 'moment';
|
||||||
|
import { Writable } from '@kbn/utility-types';
|
||||||
|
import { ISavedObjectsRepository } from 'src/core/server';
|
||||||
|
import { ApplicationUsageReport } from './schema';
|
||||||
|
import { getDailyId } from '../../common/application_usage';
|
||||||
|
|
||||||
|
type WritableApplicationUsageReport = Writable<ApplicationUsageReport>;
|
||||||
|
|
||||||
|
export const storeApplicationUsage = async (
|
||||||
|
repository: ISavedObjectsRepository,
|
||||||
|
appUsages: ApplicationUsageReport[],
|
||||||
|
timestamp: Date
|
||||||
|
) => {
|
||||||
|
if (!appUsages.length) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dayId = getDayId(timestamp);
|
||||||
|
const aggregatedReports = aggregateAppUsages(appUsages);
|
||||||
|
|
||||||
|
return Promise.allSettled(
|
||||||
|
aggregatedReports.map(async (report) => incrementUsageCounters(repository, report, dayId))
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const aggregateAppUsages = (appUsages: ApplicationUsageReport[]) => {
|
||||||
|
return [
|
||||||
|
...appUsages
|
||||||
|
.reduce((map, appUsage) => {
|
||||||
|
const key = getKey(appUsage);
|
||||||
|
const aggregated: WritableApplicationUsageReport = map.get(key) ?? {
|
||||||
|
appId: appUsage.appId,
|
||||||
|
viewId: appUsage.viewId,
|
||||||
|
minutesOnScreen: 0,
|
||||||
|
numberOfClicks: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
aggregated.minutesOnScreen += appUsage.minutesOnScreen;
|
||||||
|
aggregated.numberOfClicks += appUsage.numberOfClicks;
|
||||||
|
|
||||||
|
map.set(key, aggregated);
|
||||||
|
return map;
|
||||||
|
}, new Map<string, ApplicationUsageReport>())
|
||||||
|
.values(),
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
const incrementUsageCounters = (
|
||||||
|
repository: ISavedObjectsRepository,
|
||||||
|
{ appId, viewId, numberOfClicks, minutesOnScreen }: WritableApplicationUsageReport,
|
||||||
|
dayId: string
|
||||||
|
) => {
|
||||||
|
const dailyId = getDailyId({ appId, viewId, dayId });
|
||||||
|
|
||||||
|
return repository.incrementCounter(
|
||||||
|
'application_usage_daily',
|
||||||
|
dailyId,
|
||||||
|
[
|
||||||
|
{ fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
|
||||||
|
{ fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
|
||||||
|
],
|
||||||
|
{
|
||||||
|
upsertAttributes: {
|
||||||
|
appId,
|
||||||
|
viewId,
|
||||||
|
timestamp: getTimestamp(dayId),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const getKey = ({ viewId, appId }: ApplicationUsageReport) => `${appId}___${viewId}`;
|
||||||
|
|
||||||
|
const getDayId = (timestamp: Date) => moment(timestamp).format('YYYY-MM-DD');
|
||||||
|
|
||||||
|
const getTimestamp = (dayId: string) => {
|
||||||
|
// Concatenating the day in YYYY-MM-DD form to T00:00:00Z to reduce the TZ effects
|
||||||
|
return moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString();
|
||||||
|
};
|
|
@ -0,0 +1,12 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const storeApplicationUsageMock = jest.fn();
|
||||||
|
jest.doMock('./store_application_usage', () => ({
|
||||||
|
storeApplicationUsage: storeApplicationUsageMock,
|
||||||
|
}));
|
|
@ -6,6 +6,8 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { storeApplicationUsageMock } from './store_report.test.mocks';
|
||||||
|
|
||||||
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
|
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
|
||||||
import { storeReport } from './store_report';
|
import { storeReport } from './store_report';
|
||||||
import { ReportSchemaType } from './schema';
|
import { ReportSchemaType } from './schema';
|
||||||
|
@ -16,8 +18,17 @@ describe('store_report', () => {
|
||||||
const momentTimestamp = moment();
|
const momentTimestamp = moment();
|
||||||
const date = momentTimestamp.format('DDMMYYYY');
|
const date = momentTimestamp.format('DDMMYYYY');
|
||||||
|
|
||||||
|
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
repository = savedObjectsRepositoryMock.create();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
storeApplicationUsageMock.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
test('stores report for all types of data', async () => {
|
test('stores report for all types of data', async () => {
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
|
||||||
const report: ReportSchemaType = {
|
const report: ReportSchemaType = {
|
||||||
reportVersion: 3,
|
reportVersion: 3,
|
||||||
userAgent: {
|
userAgent: {
|
||||||
|
@ -53,9 +64,9 @@ describe('store_report', () => {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
await storeReport(savedObjectClient, report);
|
await storeReport(repository, report);
|
||||||
|
|
||||||
expect(savedObjectClient.create).toHaveBeenCalledWith(
|
expect(repository.create).toHaveBeenCalledWith(
|
||||||
'ui-metric',
|
'ui-metric',
|
||||||
{ count: 1 },
|
{ count: 1 },
|
||||||
{
|
{
|
||||||
|
@ -63,51 +74,45 @@ describe('store_report', () => {
|
||||||
overwrite: true,
|
overwrite: true,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
|
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
|
||||||
1,
|
1,
|
||||||
'ui-metric',
|
'ui-metric',
|
||||||
'test-app-name:test-event-name',
|
'test-app-name:test-event-name',
|
||||||
[{ fieldName: 'count', incrementBy: 3 }]
|
[{ fieldName: 'count', incrementBy: 3 }]
|
||||||
);
|
);
|
||||||
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
|
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
|
||||||
2,
|
2,
|
||||||
'ui-counter',
|
'ui-counter',
|
||||||
`test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`,
|
`test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`,
|
||||||
[{ fieldName: 'count', incrementBy: 1 }]
|
[{ fieldName: 'count', incrementBy: 1 }]
|
||||||
);
|
);
|
||||||
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
|
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
|
||||||
3,
|
3,
|
||||||
'ui-counter',
|
'ui-counter',
|
||||||
`test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`,
|
`test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`,
|
||||||
[{ fieldName: 'count', incrementBy: 2 }]
|
[{ fieldName: 'count', incrementBy: 2 }]
|
||||||
);
|
);
|
||||||
expect(savedObjectClient.bulkCreate).toHaveBeenNthCalledWith(1, [
|
|
||||||
{
|
expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1);
|
||||||
type: 'application_usage_transactional',
|
expect(storeApplicationUsageMock).toHaveBeenCalledWith(
|
||||||
attributes: {
|
repository,
|
||||||
numberOfClicks: 3,
|
Object.values(report.application_usage as Record<string, any>),
|
||||||
minutesOnScreen: 10,
|
expect.any(Date)
|
||||||
appId: 'appId',
|
);
|
||||||
viewId: 'appId_view',
|
|
||||||
timestamp: expect.any(Date),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test('it should not fail if nothing to store', async () => {
|
test('it should not fail if nothing to store', async () => {
|
||||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
|
||||||
const report: ReportSchemaType = {
|
const report: ReportSchemaType = {
|
||||||
reportVersion: 3,
|
reportVersion: 3,
|
||||||
userAgent: void 0,
|
userAgent: void 0,
|
||||||
uiCounter: void 0,
|
uiCounter: void 0,
|
||||||
application_usage: void 0,
|
application_usage: void 0,
|
||||||
};
|
};
|
||||||
await storeReport(savedObjectClient, report);
|
await storeReport(repository, report);
|
||||||
|
|
||||||
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
|
expect(repository.bulkCreate).not.toHaveBeenCalled();
|
||||||
expect(savedObjectClient.incrementCounter).not.toHaveBeenCalled();
|
expect(repository.incrementCounter).not.toHaveBeenCalled();
|
||||||
expect(savedObjectClient.create).not.toHaveBeenCalled();
|
expect(repository.create).not.toHaveBeenCalled();
|
||||||
expect(savedObjectClient.create).not.toHaveBeenCalled();
|
expect(repository.create).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -10,6 +10,7 @@ import { ISavedObjectsRepository } from 'src/core/server';
|
||||||
import moment from 'moment';
|
import moment from 'moment';
|
||||||
import { chain, sumBy } from 'lodash';
|
import { chain, sumBy } from 'lodash';
|
||||||
import { ReportSchemaType } from './schema';
|
import { ReportSchemaType } from './schema';
|
||||||
|
import { storeApplicationUsage } from './store_application_usage';
|
||||||
|
|
||||||
export async function storeReport(
|
export async function storeReport(
|
||||||
internalRepository: ISavedObjectsRepository,
|
internalRepository: ISavedObjectsRepository,
|
||||||
|
@ -17,11 +18,11 @@ export async function storeReport(
|
||||||
) {
|
) {
|
||||||
const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : [];
|
const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : [];
|
||||||
const userAgents = report.userAgent ? Object.entries(report.userAgent) : [];
|
const userAgents = report.userAgent ? Object.entries(report.userAgent) : [];
|
||||||
const appUsage = report.application_usage ? Object.values(report.application_usage) : [];
|
const appUsages = report.application_usage ? Object.values(report.application_usage) : [];
|
||||||
|
|
||||||
const momentTimestamp = moment();
|
const momentTimestamp = moment();
|
||||||
const timestamp = momentTimestamp.toDate();
|
|
||||||
const date = momentTimestamp.format('DDMMYYYY');
|
const date = momentTimestamp.format('DDMMYYYY');
|
||||||
|
const timestamp = momentTimestamp.toDate();
|
||||||
|
|
||||||
return Promise.allSettled([
|
return Promise.allSettled([
|
||||||
// User Agent
|
// User Agent
|
||||||
|
@ -64,21 +65,6 @@ export async function storeReport(
|
||||||
];
|
];
|
||||||
}),
|
}),
|
||||||
// Application Usage
|
// Application Usage
|
||||||
...[
|
storeApplicationUsage(internalRepository, appUsages, timestamp),
|
||||||
(async () => {
|
|
||||||
if (!appUsage.length) return [];
|
|
||||||
const { saved_objects: savedObjects } = await internalRepository.bulkCreate(
|
|
||||||
appUsage.map((metric) => ({
|
|
||||||
type: 'application_usage_transactional',
|
|
||||||
attributes: {
|
|
||||||
...metric,
|
|
||||||
timestamp,
|
|
||||||
},
|
|
||||||
}))
|
|
||||||
);
|
|
||||||
|
|
||||||
return savedObjects;
|
|
||||||
})(),
|
|
||||||
],
|
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
|
@ -156,7 +156,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
describe('application usage limits', () => {
|
describe('application usage limits', () => {
|
||||||
function createSavedObject(viewId?: string) {
|
function createSavedObject(viewId?: string) {
|
||||||
return supertest
|
return supertest
|
||||||
.post('/api/saved_objects/application_usage_transactional')
|
.post('/api/saved_objects/application_usage_daily')
|
||||||
.send({
|
.send({
|
||||||
attributes: {
|
attributes: {
|
||||||
appId: 'test-app',
|
appId: 'test-app',
|
||||||
|
@ -184,7 +184,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
savedObjectIds.map((savedObjectId) => {
|
savedObjectIds.map((savedObjectId) => {
|
||||||
return supertest
|
return supertest
|
||||||
.delete(`/api/saved_objects/application_usage_transactional/${savedObjectId}`)
|
.delete(`/api/saved_objects/application_usage_daily/${savedObjectId}`)
|
||||||
.expect(200);
|
.expect(200);
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
@ -230,7 +230,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
.post('/api/saved_objects/_bulk_create')
|
.post('/api/saved_objects/_bulk_create')
|
||||||
.send(
|
.send(
|
||||||
new Array(10001).fill(0).map(() => ({
|
new Array(10001).fill(0).map(() => ({
|
||||||
type: 'application_usage_transactional',
|
type: 'application_usage_daily',
|
||||||
attributes: {
|
attributes: {
|
||||||
appId: 'test-app',
|
appId: 'test-app',
|
||||||
minutesOnScreen: 1,
|
minutesOnScreen: 1,
|
||||||
|
@ -248,13 +248,12 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
// The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout
|
// The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout
|
||||||
await es.deleteByQuery({
|
await es.deleteByQuery({
|
||||||
index: '.kibana',
|
index: '.kibana',
|
||||||
body: { query: { term: { type: 'application_usage_transactional' } } },
|
body: { query: { term: { type: 'application_usage_daily' } } },
|
||||||
conflicts: 'proceed',
|
conflicts: 'proceed',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// flaky https://github.com/elastic/kibana/issues/94513
|
it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
|
||||||
it.skip("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
|
|
||||||
const stats = await retrieveTelemetry(supertest);
|
const stats = await retrieveTelemetry(supertest);
|
||||||
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
|
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
|
||||||
'test-app': {
|
'test-app': {
|
||||||
|
|
Loading…
Reference in a new issue