[Application Usage] use incrementCounter on daily reports instead of creating transactional documents (#94923)
* initial implementation * add test for upsertAttributes * update generated doc * remove comment * removal transactional documents from the collector aggregation logic * rename generic type * add missing doc file * split rollups into distinct files * for api integ test * extract types to their own file * only roll transactional documents during startup * proper fix is better fix * perform daily rolling until success * unskip flaky test * fix unit tests
This commit is contained in:
parent
5d59c2c8e1
commit
32212644da
|
@ -8,7 +8,7 @@
|
|||
<b>Signature:</b>
|
||||
|
||||
```typescript
|
||||
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions
|
||||
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown> extends SavedObjectsBaseOptions
|
||||
```
|
||||
|
||||
## Properties
|
||||
|
@ -18,4 +18,5 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
|
|||
| [initialize](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.initialize.md) | <code>boolean</code> | (default=false) If true, sets all the counter fields to 0 if they don't already exist. Existing fields will be left as-is and won't be incremented. |
|
||||
| [migrationVersion](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.migrationversion.md) | <code>SavedObjectsMigrationVersion</code> | [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) |
|
||||
| [refresh](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.refresh.md) | <code>MutatingOperationRefreshSetting</code> | (default='wait\_for') The Elasticsearch refresh setting for this operation. See [MutatingOperationRefreshSetting](./kibana-plugin-core-server.mutatingoperationrefreshsetting.md) |
|
||||
| [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md) | <code>Attributes</code> | Attributes to use when upserting the document if it doesn't exist. |
|
||||
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
|
||||
|
||||
[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) > [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md)
|
||||
|
||||
## SavedObjectsIncrementCounterOptions.upsertAttributes property
|
||||
|
||||
Attributes to use when upserting the document if it doesn't exist.
|
||||
|
||||
<b>Signature:</b>
|
||||
|
||||
```typescript
|
||||
upsertAttributes?: Attributes;
|
||||
```
|
|
@ -9,7 +9,7 @@ Increments all the specified counter fields (by one by default). Creates the doc
|
|||
<b>Signature:</b>
|
||||
|
||||
```typescript
|
||||
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions): Promise<SavedObject<T>>;
|
||||
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions<T>): Promise<SavedObject<T>>;
|
||||
```
|
||||
|
||||
## Parameters
|
||||
|
@ -19,7 +19,7 @@ incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<str
|
|||
| type | <code>string</code> | The type of saved object whose fields should be incremented |
|
||||
| id | <code>string</code> | The id of the document whose fields should be incremented |
|
||||
| counterFields | <code>Array<string | SavedObjectsIncrementCounterField></code> | An array of field names to increment or an array of [SavedObjectsIncrementCounterField](./kibana-plugin-core-server.savedobjectsincrementcounterfield.md) |
|
||||
| options | <code>SavedObjectsIncrementCounterOptions</code> | [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
|
||||
| options | <code>SavedObjectsIncrementCounterOptions<T></code> | [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
|
||||
|
||||
<b>Returns:</b>
|
||||
|
||||
|
@ -52,5 +52,19 @@ repository
|
|||
'stats.apiCalls',
|
||||
])
|
||||
|
||||
// Increment the apiCalls field counter by 4
|
||||
repository
|
||||
.incrementCounter('dashboard_counter_type', 'counter_id', [
|
||||
{ fieldName: 'stats.apiCalls' incrementBy: 4 },
|
||||
])
|
||||
|
||||
// Initialize the document with arbitrary fields if not present
|
||||
repository.incrementCounter<{ appId: string }>(
|
||||
'dashboard_counter_type',
|
||||
'counter_id',
|
||||
[ 'stats.apiCalls'],
|
||||
{ upsertAttributes: { appId: 'myId' } }
|
||||
)
|
||||
|
||||
```
|
||||
|
||||
|
|
|
@ -23,6 +23,7 @@ import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock
|
|||
import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks';
|
||||
import { esKuery } from '../../es_query';
|
||||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||
|
||||
const { nodeTypes } = esKuery;
|
||||
|
||||
jest.mock('./search_dsl/search_dsl', () => ({ getSearchDsl: jest.fn() }));
|
||||
|
@ -3654,6 +3655,33 @@ describe('SavedObjectsRepository', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it(`uses the 'upsertAttributes' option when specified`, async () => {
|
||||
const upsertAttributes = {
|
||||
foo: 'bar',
|
||||
hello: 'dolly',
|
||||
};
|
||||
await incrementCounterSuccess(type, id, counterFields, { namespace, upsertAttributes });
|
||||
expect(client.update).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
body: expect.objectContaining({
|
||||
upsert: expect.objectContaining({
|
||||
[type]: {
|
||||
foo: 'bar',
|
||||
hello: 'dolly',
|
||||
...counterFields.reduce((aggs, field) => {
|
||||
return {
|
||||
...aggs,
|
||||
[field]: 1,
|
||||
};
|
||||
}, {}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
expect.anything()
|
||||
);
|
||||
});
|
||||
|
||||
it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => {
|
||||
await incrementCounterSuccess(type, id, counterFields, { namespace });
|
||||
expect(client.update).toHaveBeenCalledWith(
|
||||
|
|
|
@ -76,10 +76,16 @@ import {
|
|||
// BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository
|
||||
// so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient.
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
|
||||
type Left = { tag: 'Left'; error: Record<string, any> };
|
||||
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
|
||||
type Right = { tag: 'Right'; value: Record<string, any> };
|
||||
interface Left {
|
||||
tag: 'Left';
|
||||
error: Record<string, any>;
|
||||
}
|
||||
|
||||
interface Right {
|
||||
tag: 'Right';
|
||||
value: Record<string, any>;
|
||||
}
|
||||
|
||||
type Either = Left | Right;
|
||||
const isLeft = (either: Either): either is Left => either.tag === 'Left';
|
||||
const isRight = (either: Either): either is Right => either.tag === 'Right';
|
||||
|
@ -98,7 +104,8 @@ export interface SavedObjectsRepositoryOptions {
|
|||
/**
|
||||
* @public
|
||||
*/
|
||||
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
|
||||
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown>
|
||||
extends SavedObjectsBaseOptions {
|
||||
/**
|
||||
* (default=false) If true, sets all the counter fields to 0 if they don't
|
||||
* already exist. Existing fields will be left as-is and won't be incremented.
|
||||
|
@ -111,6 +118,10 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
|
|||
* operation. See {@link MutatingOperationRefreshSetting}
|
||||
*/
|
||||
refresh?: MutatingOperationRefreshSetting;
|
||||
/**
|
||||
* Attributes to use when upserting the document if it doesn't exist.
|
||||
*/
|
||||
upsertAttributes?: Attributes;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1694,6 +1705,20 @@ export class SavedObjectsRepository {
|
|||
* .incrementCounter('dashboard_counter_type', 'counter_id', [
|
||||
* 'stats.apiCalls',
|
||||
* ])
|
||||
*
|
||||
* // Increment the apiCalls field counter by 4
|
||||
* repository
|
||||
* .incrementCounter('dashboard_counter_type', 'counter_id', [
|
||||
* { fieldName: 'stats.apiCalls' incrementBy: 4 },
|
||||
* ])
|
||||
*
|
||||
* // Initialize the document with arbitrary fields if not present
|
||||
* repository.incrementCounter<{ appId: string }>(
|
||||
* 'dashboard_counter_type',
|
||||
* 'counter_id',
|
||||
* [ 'stats.apiCalls'],
|
||||
* { upsertAttributes: { appId: 'myId' } }
|
||||
* )
|
||||
* ```
|
||||
*
|
||||
* @param type - The type of saved object whose fields should be incremented
|
||||
|
@ -1706,7 +1731,7 @@ export class SavedObjectsRepository {
|
|||
type: string,
|
||||
id: string,
|
||||
counterFields: Array<string | SavedObjectsIncrementCounterField>,
|
||||
options: SavedObjectsIncrementCounterOptions = {}
|
||||
options: SavedObjectsIncrementCounterOptions<T> = {}
|
||||
): Promise<SavedObject<T>> {
|
||||
if (typeof type !== 'string') {
|
||||
throw new Error('"type" argument must be a string');
|
||||
|
@ -1728,12 +1753,16 @@ export class SavedObjectsRepository {
|
|||
throw SavedObjectsErrorHelpers.createUnsupportedTypeError(type);
|
||||
}
|
||||
|
||||
const { migrationVersion, refresh = DEFAULT_REFRESH_SETTING, initialize = false } = options;
|
||||
const {
|
||||
migrationVersion,
|
||||
refresh = DEFAULT_REFRESH_SETTING,
|
||||
initialize = false,
|
||||
upsertAttributes,
|
||||
} = options;
|
||||
|
||||
const normalizedCounterFields = counterFields.map((counterField) => {
|
||||
const fieldName = typeof counterField === 'string' ? counterField : counterField.fieldName;
|
||||
const incrementBy = typeof counterField === 'string' ? 1 : counterField.incrementBy || 1;
|
||||
|
||||
return {
|
||||
fieldName,
|
||||
incrementBy: initialize ? 0 : incrementBy,
|
||||
|
@ -1757,11 +1786,14 @@ export class SavedObjectsRepository {
|
|||
type,
|
||||
...(savedObjectNamespace && { namespace: savedObjectNamespace }),
|
||||
...(savedObjectNamespaces && { namespaces: savedObjectNamespaces }),
|
||||
attributes: normalizedCounterFields.reduce((acc, counterField) => {
|
||||
attributes: {
|
||||
...(upsertAttributes ?? {}),
|
||||
...normalizedCounterFields.reduce((acc, counterField) => {
|
||||
const { fieldName, incrementBy } = counterField;
|
||||
acc[fieldName] = incrementBy;
|
||||
return acc;
|
||||
}, {} as Record<string, number>),
|
||||
},
|
||||
migrationVersion,
|
||||
updated_at: time,
|
||||
});
|
||||
|
|
|
@ -2735,11 +2735,12 @@ export interface SavedObjectsIncrementCounterField {
|
|||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
|
||||
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown> extends SavedObjectsBaseOptions {
|
||||
initialize?: boolean;
|
||||
// (undocumented)
|
||||
migrationVersion?: SavedObjectsMigrationVersion;
|
||||
refresh?: MutatingOperationRefreshSetting;
|
||||
upsertAttributes?: Attributes;
|
||||
}
|
||||
|
||||
// @public
|
||||
|
@ -2839,7 +2840,7 @@ export class SavedObjectsRepository {
|
|||
// (undocumented)
|
||||
find<T = unknown>(options: SavedObjectsFindOptions): Promise<SavedObjectsFindResponse<T>>;
|
||||
get<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObject<T>>;
|
||||
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions): Promise<SavedObject<T>>;
|
||||
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions<T>): Promise<SavedObject<T>>;
|
||||
openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise<SavedObjectsOpenPointInTimeResponse>;
|
||||
removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise<SavedObjectsRemoveReferencesToResponse>;
|
||||
resolve<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObjectsResolveResponse<T>>;
|
||||
|
|
|
@ -12,15 +12,9 @@
|
|||
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Roll daily indices every 30 minutes.
|
||||
* This means that, assuming a user can visit all the 44 apps we can possibly report
|
||||
* in the 3 minutes interval the browser reports to the server, up to 22 users can have the same
|
||||
* behaviour and we wouldn't need to paginate in the transactional documents (less than 10k docs).
|
||||
*
|
||||
* Based on a more normal expected use case, the users could visit up to 5 apps in those 3 minutes,
|
||||
* allowing up to 200 users before reaching the limit.
|
||||
* Roll daily indices every 24h
|
||||
*/
|
||||
export const ROLL_DAILY_INDICES_INTERVAL = 30 * 60 * 1000;
|
||||
export const ROLL_DAILY_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Start rolling indices after 5 minutes up
|
||||
|
|
|
@ -7,3 +7,4 @@
|
|||
*/
|
||||
|
||||
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';
|
||||
export { rollDailyData as migrateTransactionalDocs } from './rollups';
|
||||
|
|
|
@ -6,21 +6,16 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { rollDailyData, rollTotals } from './rollups';
|
||||
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||
import { SavedObjectsErrorHelpers } from '../../../../../core/server';
|
||||
import {
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
SAVED_OBJECTS_TOTAL_TYPE,
|
||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||
} from './saved_objects_types';
|
||||
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
|
||||
import { SavedObjectsErrorHelpers } from '../../../../../../core/server';
|
||||
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TRANSACTIONAL_TYPE } from '../saved_objects_types';
|
||||
import { rollDailyData } from './daily';
|
||||
|
||||
describe('rollDailyData', () => {
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
|
||||
test('returns undefined if no savedObjectsClient initialised yet', async () => {
|
||||
await expect(rollDailyData(logger, undefined)).resolves.toBe(undefined);
|
||||
test('returns false if no savedObjectsClient initialised yet', async () => {
|
||||
await expect(rollDailyData(logger, undefined)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
test('handle empty results', async () => {
|
||||
|
@ -33,7 +28,7 @@ describe('rollDailyData', () => {
|
|||
throw new Error(`Unexpected type [${type}]`);
|
||||
}
|
||||
});
|
||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
|
||||
expect(savedObjectClient.get).not.toBeCalled();
|
||||
expect(savedObjectClient.bulkCreate).not.toBeCalled();
|
||||
expect(savedObjectClient.delete).not.toBeCalled();
|
||||
|
@ -101,7 +96,7 @@ describe('rollDailyData', () => {
|
|||
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
|
||||
});
|
||||
|
||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
|
||||
expect(savedObjectClient.get).toHaveBeenCalledTimes(2);
|
||||
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
|
@ -196,7 +191,7 @@ describe('rollDailyData', () => {
|
|||
throw new Error('Something went terribly wrong');
|
||||
});
|
||||
|
||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(false);
|
||||
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
|
||||
expect(savedObjectClient.get).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
|
@ -206,185 +201,3 @@ describe('rollDailyData', () => {
|
|||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollTotals', () => {
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
|
||||
test('returns undefined if no savedObjectsClient initialised yet', async () => {
|
||||
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
|
||||
});
|
||||
|
||||
test('handle empty results', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
||||
switch (type) {
|
||||
case SAVED_OBJECTS_DAILY_TYPE:
|
||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||
return { saved_objects: [], total: 0, page, per_page: perPage };
|
||||
default:
|
||||
throw new Error(`Unexpected type [${type}]`);
|
||||
}
|
||||
});
|
||||
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
test('migrate some documents', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
||||
switch (type) {
|
||||
case SAVED_OBJECTS_DAILY_TYPE:
|
||||
return {
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'appId-2:2020-01-01',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
timestamp: '2020-01-01T10:31:00.000Z',
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-1:2020-01-01',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
timestamp: '2020-01-01T11:31:00.000Z',
|
||||
minutesOnScreen: 2.5,
|
||||
numberOfClicks: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-1:2020-01-01:viewId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: 'viewId-1',
|
||||
timestamp: '2020-01-01T11:31:00.000Z',
|
||||
minutesOnScreen: 1,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
total: 3,
|
||||
page,
|
||||
per_page: perPage,
|
||||
};
|
||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||
return {
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'appId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-1___viewId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 4,
|
||||
numberOfClicks: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-2___viewId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 1,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
total: 3,
|
||||
page,
|
||||
per_page: perPage,
|
||||
};
|
||||
default:
|
||||
throw new Error(`Unexpected type [${type}]`);
|
||||
}
|
||||
});
|
||||
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
|
||||
[
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-1',
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
||||
minutesOnScreen: 3.0,
|
||||
numberOfClicks: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-1___viewId-1',
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 5.0,
|
||||
numberOfClicks: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-2___viewId-1',
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 1.0,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-2',
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
{ overwrite: true }
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId-2:2020-01-01'
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId-1:2020-01-01'
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId-1:2020-01-01:viewId-1'
|
||||
);
|
||||
});
|
||||
});
|
|
@ -6,18 +6,20 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { ISavedObjectsRepository, SavedObject, Logger } from 'kibana/server';
|
||||
import moment from 'moment';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import {
|
||||
ISavedObjectsRepository,
|
||||
SavedObject,
|
||||
SavedObjectsErrorHelpers,
|
||||
} from '../../../../../../core/server';
|
||||
import { getDailyId } from '../../../../../usage_collection/common/application_usage';
|
||||
import {
|
||||
ApplicationUsageDaily,
|
||||
ApplicationUsageTotal,
|
||||
ApplicationUsageTransactional,
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
SAVED_OBJECTS_TOTAL_TYPE,
|
||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||
} from './saved_objects_types';
|
||||
import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||
} from '../saved_objects_types';
|
||||
|
||||
/**
|
||||
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
|
||||
|
@ -27,18 +29,17 @@ type ApplicationUsageDailyWithVersion = Pick<
|
|||
'version' | 'attributes'
|
||||
>;
|
||||
|
||||
export function serializeKey(appId: string, viewId: string) {
|
||||
return `${appId}___${viewId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregates all the transactional events into daily aggregates
|
||||
* @param logger
|
||||
* @param savedObjectsClient
|
||||
*/
|
||||
export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
|
||||
export async function rollDailyData(
|
||||
logger: Logger,
|
||||
savedObjectsClient?: ISavedObjectsRepository
|
||||
): Promise<boolean> {
|
||||
if (!savedObjectsClient) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
|
@ -58,10 +59,7 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
|
|||
} = doc;
|
||||
const dayId = moment(timestamp).format('YYYY-MM-DD');
|
||||
|
||||
const dailyId =
|
||||
!viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
|
||||
? `${appId}:${dayId}`
|
||||
: `${appId}:${dayId}:${viewId}`;
|
||||
const dailyId = getDailyId({ dayId, appId, viewId });
|
||||
|
||||
const existingDoc =
|
||||
toCreate.get(dailyId) ||
|
||||
|
@ -103,9 +101,11 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
|
|||
}
|
||||
}
|
||||
} while (toCreate.size > 0);
|
||||
return true;
|
||||
} catch (err) {
|
||||
logger.debug(`Failed to rollup transactional to daily entries`);
|
||||
logger.debug(err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -125,7 +125,11 @@ async function getDailyDoc(
|
|||
dayId: string
|
||||
): Promise<ApplicationUsageDailyWithVersion> {
|
||||
try {
|
||||
return await savedObjectsClient.get<ApplicationUsageDaily>(SAVED_OBJECTS_DAILY_TYPE, id);
|
||||
const { attributes, version } = await savedObjectsClient.get<ApplicationUsageDaily>(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
id
|
||||
);
|
||||
return { attributes, version };
|
||||
} catch (err) {
|
||||
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
|
||||
return {
|
||||
|
@ -142,91 +146,3 @@ async function getDailyDoc(
|
|||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
|
||||
* @param logger
|
||||
* @param savedObjectsClient
|
||||
*/
|
||||
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
|
||||
if (!savedObjectsClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const [
|
||||
{ saved_objects: rawApplicationUsageTotals },
|
||||
{ saved_objects: rawApplicationUsageDaily },
|
||||
] = await Promise.all([
|
||||
savedObjectsClient.find<ApplicationUsageTotal>({
|
||||
perPage: 10000,
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
}),
|
||||
savedObjectsClient.find<ApplicationUsageDaily>({
|
||||
perPage: 10000,
|
||||
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
|
||||
}),
|
||||
]);
|
||||
|
||||
const existingTotals = rawApplicationUsageTotals.reduce(
|
||||
(
|
||||
acc,
|
||||
{
|
||||
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
|
||||
}
|
||||
) => {
|
||||
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
||||
|
||||
return {
|
||||
...acc,
|
||||
// No need to sum because there should be 1 document per appId only
|
||||
[key]: { appId, viewId, numberOfClicks, minutesOnScreen },
|
||||
};
|
||||
},
|
||||
{} as Record<
|
||||
string,
|
||||
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
|
||||
>
|
||||
);
|
||||
|
||||
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
|
||||
const {
|
||||
appId,
|
||||
viewId = MAIN_APP_DEFAULT_VIEW_ID,
|
||||
numberOfClicks,
|
||||
minutesOnScreen,
|
||||
} = attributes;
|
||||
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
||||
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
|
||||
|
||||
return {
|
||||
...acc,
|
||||
[key]: {
|
||||
appId,
|
||||
viewId,
|
||||
numberOfClicks: numberOfClicks + existing.numberOfClicks,
|
||||
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
|
||||
},
|
||||
};
|
||||
}, existingTotals);
|
||||
|
||||
await Promise.all([
|
||||
Object.entries(totals).length &&
|
||||
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
|
||||
Object.entries(totals).map(([id, entry]) => ({
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id,
|
||||
attributes: entry,
|
||||
})),
|
||||
{ overwrite: true }
|
||||
),
|
||||
...rawApplicationUsageDaily.map(
|
||||
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
|
||||
),
|
||||
]);
|
||||
} catch (err) {
|
||||
logger.debug(`Failed to rollup daily entries to totals`);
|
||||
logger.debug(err);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export { rollDailyData } from './daily';
|
||||
export { rollTotals } from './total';
|
||||
export { serializeKey } from './utils';
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
|
||||
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from '../saved_objects_types';
|
||||
import { rollTotals } from './total';
|
||||
|
||||
describe('rollTotals', () => {
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
|
||||
test('returns undefined if no savedObjectsClient initialised yet', async () => {
|
||||
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
|
||||
});
|
||||
|
||||
test('handle empty results', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
||||
switch (type) {
|
||||
case SAVED_OBJECTS_DAILY_TYPE:
|
||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||
return { saved_objects: [], total: 0, page, per_page: perPage };
|
||||
default:
|
||||
throw new Error(`Unexpected type [${type}]`);
|
||||
}
|
||||
});
|
||||
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
test('migrate some documents', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
|
||||
switch (type) {
|
||||
case SAVED_OBJECTS_DAILY_TYPE:
|
||||
return {
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'appId-2:2020-01-01',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
timestamp: '2020-01-01T10:31:00.000Z',
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-1:2020-01-01',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
timestamp: '2020-01-01T11:31:00.000Z',
|
||||
minutesOnScreen: 2.5,
|
||||
numberOfClicks: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-1:2020-01-01:viewId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: 'viewId-1',
|
||||
timestamp: '2020-01-01T11:31:00.000Z',
|
||||
minutesOnScreen: 1,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
total: 3,
|
||||
page,
|
||||
per_page: perPage,
|
||||
};
|
||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||
return {
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'appId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-1___viewId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 4,
|
||||
numberOfClicks: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'appId-2___viewId-1',
|
||||
type,
|
||||
score: 0,
|
||||
references: [],
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 1,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
total: 3,
|
||||
page,
|
||||
per_page: perPage,
|
||||
};
|
||||
default:
|
||||
throw new Error(`Unexpected type [${type}]`);
|
||||
}
|
||||
});
|
||||
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
|
||||
[
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-1',
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
||||
minutesOnScreen: 3.0,
|
||||
numberOfClicks: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-1___viewId-1',
|
||||
attributes: {
|
||||
appId: 'appId-1',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 5.0,
|
||||
numberOfClicks: 3,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-2___viewId-1',
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
viewId: 'viewId-1',
|
||||
minutesOnScreen: 1.0,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id: 'appId-2',
|
||||
attributes: {
|
||||
appId: 'appId-2',
|
||||
viewId: MAIN_APP_DEFAULT_VIEW_ID,
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
{ overwrite: true }
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId-2:2020-01-01'
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId-1:2020-01-01'
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId-1:2020-01-01:viewId-1'
|
||||
);
|
||||
});
|
||||
});
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { ISavedObjectsRepository } from 'kibana/server';
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
|
||||
import {
|
||||
ApplicationUsageDaily,
|
||||
ApplicationUsageTotal,
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
SAVED_OBJECTS_TOTAL_TYPE,
|
||||
} from '../saved_objects_types';
|
||||
import { serializeKey } from './utils';
|
||||
|
||||
/**
|
||||
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
|
||||
* @param logger
|
||||
* @param savedObjectsClient
|
||||
*/
|
||||
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
|
||||
if (!savedObjectsClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const [
|
||||
{ saved_objects: rawApplicationUsageTotals },
|
||||
{ saved_objects: rawApplicationUsageDaily },
|
||||
] = await Promise.all([
|
||||
savedObjectsClient.find<ApplicationUsageTotal>({
|
||||
perPage: 10000,
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
}),
|
||||
savedObjectsClient.find<ApplicationUsageDaily>({
|
||||
perPage: 10000,
|
||||
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
|
||||
}),
|
||||
]);
|
||||
|
||||
const existingTotals = rawApplicationUsageTotals.reduce(
|
||||
(
|
||||
acc,
|
||||
{
|
||||
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
|
||||
}
|
||||
) => {
|
||||
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
||||
|
||||
return {
|
||||
...acc,
|
||||
// No need to sum because there should be 1 document per appId only
|
||||
[key]: { appId, viewId, numberOfClicks, minutesOnScreen },
|
||||
};
|
||||
},
|
||||
{} as Record<
|
||||
string,
|
||||
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
|
||||
>
|
||||
);
|
||||
|
||||
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
|
||||
const {
|
||||
appId,
|
||||
viewId = MAIN_APP_DEFAULT_VIEW_ID,
|
||||
numberOfClicks,
|
||||
minutesOnScreen,
|
||||
} = attributes;
|
||||
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
|
||||
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
|
||||
|
||||
return {
|
||||
...acc,
|
||||
[key]: {
|
||||
appId,
|
||||
viewId,
|
||||
numberOfClicks: numberOfClicks + existing.numberOfClicks,
|
||||
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
|
||||
},
|
||||
};
|
||||
}, existingTotals);
|
||||
|
||||
await Promise.all([
|
||||
Object.entries(totals).length &&
|
||||
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
|
||||
Object.entries(totals).map(([id, entry]) => ({
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
id,
|
||||
attributes: entry,
|
||||
})),
|
||||
{ overwrite: true }
|
||||
),
|
||||
...rawApplicationUsageDaily.map(
|
||||
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
|
||||
),
|
||||
]);
|
||||
} catch (err) {
|
||||
logger.debug(`Failed to rollup daily entries to totals`);
|
||||
logger.debug(err);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export function serializeKey(appId: string, viewId: string) {
|
||||
return `${appId}___${viewId}`;
|
||||
}
|
|
@ -6,7 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
|
||||
import type { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
|
||||
|
||||
/**
|
||||
* Used for accumulating the totals of all the stats older than 90d
|
||||
|
@ -17,6 +17,7 @@ export interface ApplicationUsageTotal extends SavedObjectAttributes {
|
|||
minutesOnScreen: number;
|
||||
numberOfClicks: number;
|
||||
}
|
||||
|
||||
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
|
||||
|
||||
/**
|
||||
|
@ -25,6 +26,8 @@ export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
|
|||
export interface ApplicationUsageTransactional extends ApplicationUsageTotal {
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/** @deprecated transactional type is no longer used, and only preserved for backward compatibility */
|
||||
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
|
||||
|
||||
/**
|
||||
|
@ -62,6 +65,7 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
|
|||
});
|
||||
|
||||
// Type for storing ApplicationUsageTransactional (declaring empty mappings because we don't use the internal fields for query/aggregations)
|
||||
// Remark: this type is deprecated and only here for BWC reasons.
|
||||
registerType({
|
||||
name: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||
hidden: false,
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
*/
|
||||
|
||||
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
|
||||
import { ApplicationUsageTelemetryReport } from './telemetry_application_usage_collector';
|
||||
import { ApplicationUsageTelemetryReport } from './types';
|
||||
|
||||
const commonSchema: MakeSchemaFrom<ApplicationUsageTelemetryReport[string]> = {
|
||||
appId: { type: 'keyword', _meta: { description: 'The application being tracked' } },
|
||||
|
|
|
@ -11,74 +11,99 @@ import {
|
|||
Collector,
|
||||
createUsageCollectionSetupMock,
|
||||
} from '../../../../usage_collection/server/usage_collection.mock';
|
||||
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
|
||||
import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
|
||||
import {
|
||||
registerApplicationUsageCollector,
|
||||
transformByApplicationViews,
|
||||
ApplicationUsageViews,
|
||||
} from './telemetry_application_usage_collector';
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||
import {
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
SAVED_OBJECTS_TOTAL_TYPE,
|
||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||
} from './saved_objects_types';
|
||||
import { ApplicationUsageViews } from './types';
|
||||
|
||||
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from './saved_objects_types';
|
||||
|
||||
// use fake timers to avoid triggering rollups during tests
|
||||
jest.useFakeTimers();
|
||||
|
||||
describe('telemetry_application_usage', () => {
|
||||
jest.useFakeTimers();
|
||||
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
|
||||
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
|
||||
let collector: Collector<unknown>;
|
||||
let usageCollectionMock: ReturnType<typeof createUsageCollectionSetupMock>;
|
||||
let savedObjectClient: ReturnType<typeof savedObjectsRepositoryMock.create>;
|
||||
let getSavedObjectClient: jest.MockedFunction<() => undefined | typeof savedObjectClient>;
|
||||
|
||||
const usageCollectionMock = createUsageCollectionSetupMock();
|
||||
const registerType = jest.fn();
|
||||
const mockedFetchContext = createCollectorFetchContextMock();
|
||||
|
||||
beforeEach(() => {
|
||||
logger = loggingSystemMock.createLogger();
|
||||
usageCollectionMock = createUsageCollectionSetupMock();
|
||||
savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
getSavedObjectClient = jest.fn().mockReturnValue(savedObjectClient);
|
||||
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
|
||||
collector = new Collector(logger, config);
|
||||
return createUsageCollectionSetupMock().makeUsageCollector(config);
|
||||
});
|
||||
|
||||
const getUsageCollector = jest.fn();
|
||||
const registerType = jest.fn();
|
||||
const mockedFetchContext = createCollectorFetchContextMock();
|
||||
|
||||
beforeAll(() =>
|
||||
registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector)
|
||||
registerApplicationUsageCollector(
|
||||
logger,
|
||||
usageCollectionMock,
|
||||
registerType,
|
||||
getSavedObjectClient
|
||||
);
|
||||
afterAll(() => jest.clearAllTimers());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllTimers();
|
||||
});
|
||||
|
||||
test('registered collector is set', () => {
|
||||
expect(collector).not.toBeUndefined();
|
||||
});
|
||||
|
||||
test('if no savedObjectClient initialised, return undefined', async () => {
|
||||
getSavedObjectClient.mockReturnValue(undefined);
|
||||
|
||||
expect(collector.isReady()).toBe(false);
|
||||
expect(await collector.fetch(mockedFetchContext)).toBeUndefined();
|
||||
jest.runTimersToTime(ROLL_INDICES_START);
|
||||
});
|
||||
|
||||
test('calls `savedObjectsClient.find` with the correct parameters', async () => {
|
||||
savedObjectClient.find.mockResolvedValue({
|
||||
saved_objects: [],
|
||||
total: 0,
|
||||
per_page: 20,
|
||||
page: 0,
|
||||
});
|
||||
|
||||
await collector.fetch(mockedFetchContext);
|
||||
|
||||
expect(savedObjectClient.find).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(savedObjectClient.find).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
})
|
||||
);
|
||||
expect(savedObjectClient.find).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('when savedObjectClient is initialised, return something', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
savedObjectClient.find.mockImplementation(
|
||||
async () =>
|
||||
({
|
||||
savedObjectClient.find.mockResolvedValue({
|
||||
saved_objects: [],
|
||||
total: 0,
|
||||
} as any)
|
||||
);
|
||||
getUsageCollector.mockImplementation(() => savedObjectClient);
|
||||
|
||||
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
|
||||
per_page: 20,
|
||||
page: 0,
|
||||
});
|
||||
|
||||
expect(collector.isReady()).toBe(true);
|
||||
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({});
|
||||
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('it only gets 10k even when there are more documents (ES limitation)', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
const total = 10000;
|
||||
test('it aggregates total and daily data', async () => {
|
||||
savedObjectClient.find.mockImplementation(async (opts) => {
|
||||
switch (opts.type) {
|
||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||
|
@ -95,18 +120,6 @@ describe('telemetry_application_usage', () => {
|
|||
],
|
||||
total: 1,
|
||||
} as any;
|
||||
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
|
||||
const doc = {
|
||||
id: 'test-id',
|
||||
attributes: {
|
||||
appId: 'appId',
|
||||
timestamp: new Date().toISOString(),
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
};
|
||||
const savedObjects = new Array(total).fill(doc);
|
||||
return { saved_objects: savedObjects, total: total + 1 };
|
||||
case SAVED_OBJECTS_DAILY_TYPE:
|
||||
return {
|
||||
saved_objects: [
|
||||
|
@ -125,122 +138,21 @@ describe('telemetry_application_usage', () => {
|
|||
}
|
||||
});
|
||||
|
||||
getUsageCollector.mockImplementation(() => savedObjectClient);
|
||||
|
||||
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
|
||||
|
||||
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
|
||||
appId: {
|
||||
appId: 'appId',
|
||||
viewId: 'main',
|
||||
clicks_total: total + 1 + 10,
|
||||
clicks_7_days: total + 1,
|
||||
clicks_30_days: total + 1,
|
||||
clicks_90_days: total + 1,
|
||||
minutes_on_screen_total: (total + 1) * 0.5 + 10,
|
||||
minutes_on_screen_7_days: (total + 1) * 0.5,
|
||||
minutes_on_screen_30_days: (total + 1) * 0.5,
|
||||
minutes_on_screen_90_days: (total + 1) * 0.5,
|
||||
clicks_total: 1 + 10,
|
||||
clicks_7_days: 1,
|
||||
clicks_30_days: 1,
|
||||
clicks_90_days: 1,
|
||||
minutes_on_screen_total: 0.5 + 10,
|
||||
minutes_on_screen_7_days: 0.5,
|
||||
minutes_on_screen_30_days: 0.5,
|
||||
minutes_on_screen_90_days: 0.5,
|
||||
views: [],
|
||||
},
|
||||
});
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
|
||||
[
|
||||
{
|
||||
id: 'appId',
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
attributes: {
|
||||
appId: 'appId',
|
||||
viewId: 'main',
|
||||
minutesOnScreen: 10.5,
|
||||
numberOfClicks: 11,
|
||||
},
|
||||
},
|
||||
],
|
||||
{ overwrite: true }
|
||||
);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
|
||||
expect(savedObjectClient.delete).toHaveBeenCalledWith(
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
'appId:YYYY-MM-DD'
|
||||
);
|
||||
});
|
||||
|
||||
test('old transactional data not migrated yet', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
savedObjectClient.find.mockImplementation(async (opts) => {
|
||||
switch (opts.type) {
|
||||
case SAVED_OBJECTS_TOTAL_TYPE:
|
||||
case SAVED_OBJECTS_DAILY_TYPE:
|
||||
return { saved_objects: [], total: 0 } as any;
|
||||
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
|
||||
return {
|
||||
saved_objects: [
|
||||
{
|
||||
id: 'test-id',
|
||||
attributes: {
|
||||
appId: 'appId',
|
||||
timestamp: new Date(0).toISOString(),
|
||||
minutesOnScreen: 0.5,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'test-id-2',
|
||||
attributes: {
|
||||
appId: 'appId',
|
||||
viewId: 'main',
|
||||
timestamp: new Date(0).toISOString(),
|
||||
minutesOnScreen: 2,
|
||||
numberOfClicks: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'test-id-3',
|
||||
attributes: {
|
||||
appId: 'appId',
|
||||
viewId: 'viewId-1',
|
||||
timestamp: new Date(0).toISOString(),
|
||||
minutesOnScreen: 1,
|
||||
numberOfClicks: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
total: 1,
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
getUsageCollector.mockImplementation(() => savedObjectClient);
|
||||
|
||||
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
|
||||
appId: {
|
||||
appId: 'appId',
|
||||
viewId: 'main',
|
||||
clicks_total: 3,
|
||||
clicks_7_days: 0,
|
||||
clicks_30_days: 0,
|
||||
clicks_90_days: 0,
|
||||
minutes_on_screen_total: 2.5,
|
||||
minutes_on_screen_7_days: 0,
|
||||
minutes_on_screen_30_days: 0,
|
||||
minutes_on_screen_90_days: 0,
|
||||
views: [
|
||||
{
|
||||
appId: 'appId',
|
||||
viewId: 'viewId-1',
|
||||
clicks_total: 1,
|
||||
clicks_7_days: 0,
|
||||
clicks_30_days: 0,
|
||||
clicks_90_days: 0,
|
||||
minutes_on_screen_total: 1,
|
||||
minutes_on_screen_7_days: 0,
|
||||
minutes_on_screen_30_days: 0,
|
||||
minutes_on_screen_90_days: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
@ -11,57 +11,21 @@ import { timer } from 'rxjs';
|
|||
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from 'kibana/server';
|
||||
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
|
||||
import { serializeKey } from './rollups';
|
||||
|
||||
import {
|
||||
ApplicationUsageDaily,
|
||||
ApplicationUsageTotal,
|
||||
ApplicationUsageTransactional,
|
||||
registerMappings,
|
||||
SAVED_OBJECTS_DAILY_TYPE,
|
||||
SAVED_OBJECTS_TOTAL_TYPE,
|
||||
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||
} from './saved_objects_types';
|
||||
import { applicationUsageSchema } from './schema';
|
||||
import { rollDailyData, rollTotals } from './rollups';
|
||||
import { rollTotals, rollDailyData, serializeKey } from './rollups';
|
||||
import {
|
||||
ROLL_TOTAL_INDICES_INTERVAL,
|
||||
ROLL_DAILY_INDICES_INTERVAL,
|
||||
ROLL_INDICES_START,
|
||||
} from './constants';
|
||||
|
||||
export interface ApplicationViewUsage {
|
||||
appId: string;
|
||||
viewId: string;
|
||||
clicks_total: number;
|
||||
clicks_7_days: number;
|
||||
clicks_30_days: number;
|
||||
clicks_90_days: number;
|
||||
minutes_on_screen_total: number;
|
||||
minutes_on_screen_7_days: number;
|
||||
minutes_on_screen_30_days: number;
|
||||
minutes_on_screen_90_days: number;
|
||||
}
|
||||
|
||||
export interface ApplicationUsageViews {
|
||||
[serializedKey: string]: ApplicationViewUsage;
|
||||
}
|
||||
|
||||
export interface ApplicationUsageTelemetryReport {
|
||||
[appId: string]: {
|
||||
appId: string;
|
||||
viewId: string;
|
||||
clicks_total: number;
|
||||
clicks_7_days: number;
|
||||
clicks_30_days: number;
|
||||
clicks_90_days: number;
|
||||
minutes_on_screen_total: number;
|
||||
minutes_on_screen_7_days: number;
|
||||
minutes_on_screen_30_days: number;
|
||||
minutes_on_screen_90_days: number;
|
||||
views?: ApplicationViewUsage[];
|
||||
};
|
||||
}
|
||||
import { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
|
||||
|
||||
export const transformByApplicationViews = (
|
||||
report: ApplicationUsageViews
|
||||
|
@ -92,6 +56,21 @@ export function registerApplicationUsageCollector(
|
|||
) {
|
||||
registerMappings(registerType);
|
||||
|
||||
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
|
||||
rollTotals(logger, getSavedObjectsClient())
|
||||
);
|
||||
|
||||
const dailyRollingSub = timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(
|
||||
async () => {
|
||||
const success = await rollDailyData(logger, getSavedObjectsClient());
|
||||
// we only need to roll the transactional documents once to assure BWC
|
||||
// once we rolling succeeds, we can stop.
|
||||
if (success) {
|
||||
dailyRollingSub.unsubscribe();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const collector = usageCollection.makeUsageCollector<ApplicationUsageTelemetryReport | undefined>(
|
||||
{
|
||||
type: 'application_usage',
|
||||
|
@ -105,7 +84,6 @@ export function registerApplicationUsageCollector(
|
|||
const [
|
||||
{ saved_objects: rawApplicationUsageTotals },
|
||||
{ saved_objects: rawApplicationUsageDaily },
|
||||
{ saved_objects: rawApplicationUsageTransactional },
|
||||
] = await Promise.all([
|
||||
savedObjectsClient.find<ApplicationUsageTotal>({
|
||||
type: SAVED_OBJECTS_TOTAL_TYPE,
|
||||
|
@ -115,10 +93,6 @@ export function registerApplicationUsageCollector(
|
|||
type: SAVED_OBJECTS_DAILY_TYPE,
|
||||
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
|
||||
}),
|
||||
savedObjectsClient.find<ApplicationUsageTransactional>({
|
||||
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
|
||||
perPage: 10000, // If we have more than those, we won't report the rest (they'll be rolled up to the daily soon enough to become a problem)
|
||||
}),
|
||||
]);
|
||||
|
||||
const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
|
||||
|
@ -156,10 +130,7 @@ export function registerApplicationUsageCollector(
|
|||
const nowMinus30 = moment().subtract(30, 'days');
|
||||
const nowMinus90 = moment().subtract(90, 'days');
|
||||
|
||||
const applicationUsage = [
|
||||
...rawApplicationUsageDaily,
|
||||
...rawApplicationUsageTransactional,
|
||||
].reduce(
|
||||
const applicationUsage = rawApplicationUsageDaily.reduce(
|
||||
(
|
||||
acc,
|
||||
{
|
||||
|
@ -224,11 +195,4 @@ export function registerApplicationUsageCollector(
|
|||
);
|
||||
|
||||
usageCollection.registerCollector(collector);
|
||||
|
||||
timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(() =>
|
||||
rollDailyData(logger, getSavedObjectsClient())
|
||||
);
|
||||
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
|
||||
rollTotals(logger, getSavedObjectsClient())
|
||||
);
|
||||
}
|
||||
|
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export interface ApplicationViewUsage {
|
||||
appId: string;
|
||||
viewId: string;
|
||||
clicks_total: number;
|
||||
clicks_7_days: number;
|
||||
clicks_30_days: number;
|
||||
clicks_90_days: number;
|
||||
minutes_on_screen_total: number;
|
||||
minutes_on_screen_7_days: number;
|
||||
minutes_on_screen_30_days: number;
|
||||
minutes_on_screen_90_days: number;
|
||||
}
|
||||
|
||||
export interface ApplicationUsageViews {
|
||||
[serializedKey: string]: ApplicationViewUsage;
|
||||
}
|
||||
|
||||
export interface ApplicationUsageTelemetryReport {
|
||||
[appId: string]: {
|
||||
appId: string;
|
||||
viewId: string;
|
||||
clicks_total: number;
|
||||
clicks_7_days: number;
|
||||
clicks_30_days: number;
|
||||
clicks_90_days: number;
|
||||
minutes_on_screen_total: number;
|
||||
minutes_on_screen_7_days: number;
|
||||
minutes_on_screen_30_days: number;
|
||||
minutes_on_screen_90_days: number;
|
||||
views?: ApplicationViewUsage[];
|
||||
};
|
||||
}
|
23
src/plugins/usage_collection/common/application_usage.ts
Normal file
23
src/plugins/usage_collection/common/application_usage.ts
Normal file
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { MAIN_APP_DEFAULT_VIEW_ID } from './constants';
|
||||
|
||||
export const getDailyId = ({
|
||||
appId,
|
||||
dayId,
|
||||
viewId,
|
||||
}: {
|
||||
viewId: string;
|
||||
appId: string;
|
||||
dayId: string;
|
||||
}) => {
|
||||
return !viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
|
||||
? `${appId}:${dayId}`
|
||||
: `${appId}:${dayId}:${viewId}`;
|
||||
};
|
|
@ -9,6 +9,13 @@
|
|||
import { schema, TypeOf } from '@kbn/config-schema';
|
||||
import { METRIC_TYPE } from '@kbn/analytics';
|
||||
|
||||
const applicationUsageReportSchema = schema.object({
|
||||
minutesOnScreen: schema.number(),
|
||||
numberOfClicks: schema.number(),
|
||||
appId: schema.string(),
|
||||
viewId: schema.string(),
|
||||
});
|
||||
|
||||
export const reportSchema = schema.object({
|
||||
reportVersion: schema.maybe(schema.oneOf([schema.literal(3)])),
|
||||
userAgent: schema.maybe(
|
||||
|
@ -38,17 +45,8 @@ export const reportSchema = schema.object({
|
|||
})
|
||||
)
|
||||
),
|
||||
application_usage: schema.maybe(
|
||||
schema.recordOf(
|
||||
schema.string(),
|
||||
schema.object({
|
||||
minutesOnScreen: schema.number(),
|
||||
numberOfClicks: schema.number(),
|
||||
appId: schema.string(),
|
||||
viewId: schema.string(),
|
||||
})
|
||||
)
|
||||
),
|
||||
application_usage: schema.maybe(schema.recordOf(schema.string(), applicationUsageReportSchema)),
|
||||
});
|
||||
|
||||
export type ReportSchemaType = TypeOf<typeof reportSchema>;
|
||||
export type ApplicationUsageReport = TypeOf<typeof applicationUsageReportSchema>;
|
||||
|
|
|
@ -0,0 +1,115 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
|
||||
import { getDailyId } from '../../common/application_usage';
|
||||
import { storeApplicationUsage } from './store_application_usage';
|
||||
import { ApplicationUsageReport } from './schema';
|
||||
|
||||
const createReport = (parts: Partial<ApplicationUsageReport>): ApplicationUsageReport => ({
|
||||
appId: 'appId',
|
||||
viewId: 'viewId',
|
||||
numberOfClicks: 0,
|
||||
minutesOnScreen: 0,
|
||||
...parts,
|
||||
});
|
||||
|
||||
describe('storeApplicationUsage', () => {
|
||||
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
|
||||
let timestamp: Date;
|
||||
|
||||
beforeEach(() => {
|
||||
repository = savedObjectsRepositoryMock.create();
|
||||
timestamp = new Date();
|
||||
});
|
||||
|
||||
it('does not call `repository.incrementUsageCounters` when the report list is empty', async () => {
|
||||
await storeApplicationUsage(repository, [], timestamp);
|
||||
expect(repository.incrementCounter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls `repository.incrementUsageCounters` with the correct parameters', async () => {
|
||||
const report = createReport({
|
||||
appId: 'app1',
|
||||
viewId: 'view1',
|
||||
numberOfClicks: 2,
|
||||
minutesOnScreen: 5,
|
||||
});
|
||||
|
||||
await storeApplicationUsage(repository, [report], timestamp);
|
||||
|
||||
expect(repository.incrementCounter).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(repository.incrementCounter).toHaveBeenCalledWith(
|
||||
...expectedIncrementParams(report, timestamp)
|
||||
);
|
||||
});
|
||||
|
||||
it('aggregates reports with the same appId/viewId tuple', async () => {
|
||||
const report1 = createReport({
|
||||
appId: 'app1',
|
||||
viewId: 'view1',
|
||||
numberOfClicks: 2,
|
||||
minutesOnScreen: 5,
|
||||
});
|
||||
const report2 = createReport({
|
||||
appId: 'app1',
|
||||
viewId: 'view2',
|
||||
numberOfClicks: 1,
|
||||
minutesOnScreen: 7,
|
||||
});
|
||||
const report3 = createReport({
|
||||
appId: 'app1',
|
||||
viewId: 'view1',
|
||||
numberOfClicks: 3,
|
||||
minutesOnScreen: 9,
|
||||
});
|
||||
|
||||
await storeApplicationUsage(repository, [report1, report2, report3], timestamp);
|
||||
|
||||
expect(repository.incrementCounter).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(repository.incrementCounter).toHaveBeenCalledWith(
|
||||
...expectedIncrementParams(
|
||||
{
|
||||
appId: 'app1',
|
||||
viewId: 'view1',
|
||||
numberOfClicks: report1.numberOfClicks + report3.numberOfClicks,
|
||||
minutesOnScreen: report1.minutesOnScreen + report3.minutesOnScreen,
|
||||
},
|
||||
timestamp
|
||||
)
|
||||
);
|
||||
expect(repository.incrementCounter).toHaveBeenCalledWith(
|
||||
...expectedIncrementParams(report2, timestamp)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const expectedIncrementParams = (
|
||||
{ appId, viewId, minutesOnScreen, numberOfClicks }: ApplicationUsageReport,
|
||||
timestamp: Date
|
||||
) => {
|
||||
const dayId = moment(timestamp).format('YYYY-MM-DD');
|
||||
return [
|
||||
'application_usage_daily',
|
||||
getDailyId({ appId, viewId, dayId }),
|
||||
[
|
||||
{ fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
|
||||
{ fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
|
||||
],
|
||||
{
|
||||
upsertAttributes: {
|
||||
appId,
|
||||
viewId,
|
||||
timestamp: moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString(),
|
||||
},
|
||||
},
|
||||
];
|
||||
};
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import moment from 'moment';
|
||||
import { Writable } from '@kbn/utility-types';
|
||||
import { ISavedObjectsRepository } from 'src/core/server';
|
||||
import { ApplicationUsageReport } from './schema';
|
||||
import { getDailyId } from '../../common/application_usage';
|
||||
|
||||
type WritableApplicationUsageReport = Writable<ApplicationUsageReport>;
|
||||
|
||||
export const storeApplicationUsage = async (
|
||||
repository: ISavedObjectsRepository,
|
||||
appUsages: ApplicationUsageReport[],
|
||||
timestamp: Date
|
||||
) => {
|
||||
if (!appUsages.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const dayId = getDayId(timestamp);
|
||||
const aggregatedReports = aggregateAppUsages(appUsages);
|
||||
|
||||
return Promise.allSettled(
|
||||
aggregatedReports.map(async (report) => incrementUsageCounters(repository, report, dayId))
|
||||
);
|
||||
};
|
||||
|
||||
const aggregateAppUsages = (appUsages: ApplicationUsageReport[]) => {
|
||||
return [
|
||||
...appUsages
|
||||
.reduce((map, appUsage) => {
|
||||
const key = getKey(appUsage);
|
||||
const aggregated: WritableApplicationUsageReport = map.get(key) ?? {
|
||||
appId: appUsage.appId,
|
||||
viewId: appUsage.viewId,
|
||||
minutesOnScreen: 0,
|
||||
numberOfClicks: 0,
|
||||
};
|
||||
|
||||
aggregated.minutesOnScreen += appUsage.minutesOnScreen;
|
||||
aggregated.numberOfClicks += appUsage.numberOfClicks;
|
||||
|
||||
map.set(key, aggregated);
|
||||
return map;
|
||||
}, new Map<string, ApplicationUsageReport>())
|
||||
.values(),
|
||||
];
|
||||
};
|
||||
|
||||
const incrementUsageCounters = (
|
||||
repository: ISavedObjectsRepository,
|
||||
{ appId, viewId, numberOfClicks, minutesOnScreen }: WritableApplicationUsageReport,
|
||||
dayId: string
|
||||
) => {
|
||||
const dailyId = getDailyId({ appId, viewId, dayId });
|
||||
|
||||
return repository.incrementCounter(
|
||||
'application_usage_daily',
|
||||
dailyId,
|
||||
[
|
||||
{ fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
|
||||
{ fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
|
||||
],
|
||||
{
|
||||
upsertAttributes: {
|
||||
appId,
|
||||
viewId,
|
||||
timestamp: getTimestamp(dayId),
|
||||
},
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const getKey = ({ viewId, appId }: ApplicationUsageReport) => `${appId}___${viewId}`;
|
||||
|
||||
const getDayId = (timestamp: Date) => moment(timestamp).format('YYYY-MM-DD');
|
||||
|
||||
const getTimestamp = (dayId: string) => {
|
||||
// Concatenating the day in YYYY-MM-DD form to T00:00:00Z to reduce the TZ effects
|
||||
return moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString();
|
||||
};
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export const storeApplicationUsageMock = jest.fn();
|
||||
jest.doMock('./store_application_usage', () => ({
|
||||
storeApplicationUsage: storeApplicationUsageMock,
|
||||
}));
|
|
@ -6,6 +6,8 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { storeApplicationUsageMock } from './store_report.test.mocks';
|
||||
|
||||
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
|
||||
import { storeReport } from './store_report';
|
||||
import { ReportSchemaType } from './schema';
|
||||
|
@ -16,8 +18,17 @@ describe('store_report', () => {
|
|||
const momentTimestamp = moment();
|
||||
const date = momentTimestamp.format('DDMMYYYY');
|
||||
|
||||
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
|
||||
|
||||
beforeEach(() => {
|
||||
repository = savedObjectsRepositoryMock.create();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
storeApplicationUsageMock.mockReset();
|
||||
});
|
||||
|
||||
test('stores report for all types of data', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
const report: ReportSchemaType = {
|
||||
reportVersion: 3,
|
||||
userAgent: {
|
||||
|
@ -53,9 +64,9 @@ describe('store_report', () => {
|
|||
},
|
||||
},
|
||||
};
|
||||
await storeReport(savedObjectClient, report);
|
||||
await storeReport(repository, report);
|
||||
|
||||
expect(savedObjectClient.create).toHaveBeenCalledWith(
|
||||
expect(repository.create).toHaveBeenCalledWith(
|
||||
'ui-metric',
|
||||
{ count: 1 },
|
||||
{
|
||||
|
@ -63,51 +74,45 @@ describe('store_report', () => {
|
|||
overwrite: true,
|
||||
}
|
||||
);
|
||||
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
|
||||
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'ui-metric',
|
||||
'test-app-name:test-event-name',
|
||||
[{ fieldName: 'count', incrementBy: 3 }]
|
||||
);
|
||||
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
|
||||
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'ui-counter',
|
||||
`test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`,
|
||||
[{ fieldName: 'count', incrementBy: 1 }]
|
||||
);
|
||||
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
|
||||
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
'ui-counter',
|
||||
`test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`,
|
||||
[{ fieldName: 'count', incrementBy: 2 }]
|
||||
);
|
||||
expect(savedObjectClient.bulkCreate).toHaveBeenNthCalledWith(1, [
|
||||
{
|
||||
type: 'application_usage_transactional',
|
||||
attributes: {
|
||||
numberOfClicks: 3,
|
||||
minutesOnScreen: 10,
|
||||
appId: 'appId',
|
||||
viewId: 'appId_view',
|
||||
timestamp: expect.any(Date),
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1);
|
||||
expect(storeApplicationUsageMock).toHaveBeenCalledWith(
|
||||
repository,
|
||||
Object.values(report.application_usage as Record<string, any>),
|
||||
expect.any(Date)
|
||||
);
|
||||
});
|
||||
|
||||
test('it should not fail if nothing to store', async () => {
|
||||
const savedObjectClient = savedObjectsRepositoryMock.create();
|
||||
const report: ReportSchemaType = {
|
||||
reportVersion: 3,
|
||||
userAgent: void 0,
|
||||
uiCounter: void 0,
|
||||
application_usage: void 0,
|
||||
};
|
||||
await storeReport(savedObjectClient, report);
|
||||
await storeReport(repository, report);
|
||||
|
||||
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
|
||||
expect(savedObjectClient.incrementCounter).not.toHaveBeenCalled();
|
||||
expect(savedObjectClient.create).not.toHaveBeenCalled();
|
||||
expect(savedObjectClient.create).not.toHaveBeenCalled();
|
||||
expect(repository.bulkCreate).not.toHaveBeenCalled();
|
||||
expect(repository.incrementCounter).not.toHaveBeenCalled();
|
||||
expect(repository.create).not.toHaveBeenCalled();
|
||||
expect(repository.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
@ -10,6 +10,7 @@ import { ISavedObjectsRepository } from 'src/core/server';
|
|||
import moment from 'moment';
|
||||
import { chain, sumBy } from 'lodash';
|
||||
import { ReportSchemaType } from './schema';
|
||||
import { storeApplicationUsage } from './store_application_usage';
|
||||
|
||||
export async function storeReport(
|
||||
internalRepository: ISavedObjectsRepository,
|
||||
|
@ -17,11 +18,11 @@ export async function storeReport(
|
|||
) {
|
||||
const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : [];
|
||||
const userAgents = report.userAgent ? Object.entries(report.userAgent) : [];
|
||||
const appUsage = report.application_usage ? Object.values(report.application_usage) : [];
|
||||
const appUsages = report.application_usage ? Object.values(report.application_usage) : [];
|
||||
|
||||
const momentTimestamp = moment();
|
||||
const timestamp = momentTimestamp.toDate();
|
||||
const date = momentTimestamp.format('DDMMYYYY');
|
||||
const timestamp = momentTimestamp.toDate();
|
||||
|
||||
return Promise.allSettled([
|
||||
// User Agent
|
||||
|
@ -64,21 +65,6 @@ export async function storeReport(
|
|||
];
|
||||
}),
|
||||
// Application Usage
|
||||
...[
|
||||
(async () => {
|
||||
if (!appUsage.length) return [];
|
||||
const { saved_objects: savedObjects } = await internalRepository.bulkCreate(
|
||||
appUsage.map((metric) => ({
|
||||
type: 'application_usage_transactional',
|
||||
attributes: {
|
||||
...metric,
|
||||
timestamp,
|
||||
},
|
||||
}))
|
||||
);
|
||||
|
||||
return savedObjects;
|
||||
})(),
|
||||
],
|
||||
storeApplicationUsage(internalRepository, appUsages, timestamp),
|
||||
]);
|
||||
}
|
||||
|
|
|
@ -156,7 +156,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
describe('application usage limits', () => {
|
||||
function createSavedObject(viewId?: string) {
|
||||
return supertest
|
||||
.post('/api/saved_objects/application_usage_transactional')
|
||||
.post('/api/saved_objects/application_usage_daily')
|
||||
.send({
|
||||
attributes: {
|
||||
appId: 'test-app',
|
||||
|
@ -184,7 +184,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
await Promise.all(
|
||||
savedObjectIds.map((savedObjectId) => {
|
||||
return supertest
|
||||
.delete(`/api/saved_objects/application_usage_transactional/${savedObjectId}`)
|
||||
.delete(`/api/saved_objects/application_usage_daily/${savedObjectId}`)
|
||||
.expect(200);
|
||||
})
|
||||
);
|
||||
|
@ -230,7 +230,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
.post('/api/saved_objects/_bulk_create')
|
||||
.send(
|
||||
new Array(10001).fill(0).map(() => ({
|
||||
type: 'application_usage_transactional',
|
||||
type: 'application_usage_daily',
|
||||
attributes: {
|
||||
appId: 'test-app',
|
||||
minutesOnScreen: 1,
|
||||
|
@ -248,13 +248,12 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
// The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout
|
||||
await es.deleteByQuery({
|
||||
index: '.kibana',
|
||||
body: { query: { term: { type: 'application_usage_transactional' } } },
|
||||
body: { query: { term: { type: 'application_usage_daily' } } },
|
||||
conflicts: 'proceed',
|
||||
});
|
||||
});
|
||||
|
||||
// flaky https://github.com/elastic/kibana/issues/94513
|
||||
it.skip("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
|
||||
it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
|
||||
const stats = await retrieveTelemetry(supertest);
|
||||
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
|
||||
'test-app': {
|
||||
|
|
Loading…
Reference in a new issue