maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Application Usage] use incrementCounter on daily reports instead of creating transactional documents (#94923)

Browse source 
* initial implementation

* add test for upsertAttributes

* update generated doc

* remove comment

* removal transactional documents from the collector aggregation logic

* rename generic type

* add missing doc file

* split rollups into distinct files

* for api integ test

* extract types to their own file

* only roll transactional documents during startup

* proper fix is better fix

* perform daily rolling until success

* unskip flaky test

* fix unit tests
This commit is contained in:
Pierre Gayvallet 2021-03-26 08:11:35 +01:00 committed by GitHub
parent 5d59c2c8e1
commit 32212644da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 882 additions and 602 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.md
View file

@ -8,7 +8,7 @@
<b>Signature:</b>
```typescript
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown> extends SavedObjectsBaseOptions
```
## Properties
@ -18,4 +18,5 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
| [initialize](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.initialize.md) | <code>boolean</code> | (default=false) If true, sets all the counter fields to 0 if they don't already exist. Existing fields will be left as-is and won't be incremented. |
| [migrationVersion](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.migrationversion.md) | <code>SavedObjectsMigrationVersion</code> | [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) |
| [refresh](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.refresh.md) | <code>MutatingOperationRefreshSetting</code> | (default='wait\_for') The Elasticsearch refresh setting for this operation. See [MutatingOperationRefreshSetting](./kibana-plugin-core-server.mutatingoperationrefreshsetting.md) |
| [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md) | <code>Attributes</code> | Attributes to use when upserting the document if it doesn't exist. |

13
docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md Normal file
View file

@ -0,0 +1,13 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->
[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) &gt; [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md)
## SavedObjectsIncrementCounterOptions.upsertAttributes property
Attributes to use when upserting the document if it doesn't exist.
<b>Signature:</b>
```typescript
upsertAttributes?: Attributes;
```

18
docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md
View file

@ -9,7 +9,7 @@ Increments all the specified counter fields (by one by default). Creates the doc
<b>Signature:</b>
```typescript
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions): Promise<SavedObject<T>>;
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions<T>): Promise<SavedObject<T>>;
```
## Parameters
@ -19,7 +19,7 @@ incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<str
| type | <code>string</code> | The type of saved object whose fields should be incremented |
| id | <code>string</code> | The id of the document whose fields should be incremented |
| counterFields | <code>Array&lt;string &#124; SavedObjectsIncrementCounterField&gt;</code> | An array of field names to increment or an array of [SavedObjectsIncrementCounterField](./kibana-plugin-core-server.savedobjectsincrementcounterfield.md) |
| options | <code>SavedObjectsIncrementCounterOptions</code> | [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
| options | <code>SavedObjectsIncrementCounterOptions&lt;T&gt;</code> | [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
<b>Returns:</b>
@ -52,5 +52,19 @@ repository
'stats.apiCalls',
])
// Increment the apiCalls field counter by 4
repository
.incrementCounter('dashboard_counter_type', 'counter_id', [
{ fieldName: 'stats.apiCalls' incrementBy: 4 },
])
// Initialize the document with arbitrary fields if not present
repository.incrementCounter<{ appId: string }>(
'dashboard_counter_type',
'counter_id',
[ 'stats.apiCalls'],
{ upsertAttributes: { appId: 'myId' } }
)
```

28
src/core/server/saved_objects/service/lib/repository.test.js
View file

@ -23,6 +23,7 @@ import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock
import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks';
import { esKuery } from '../../es_query';
import { errors as EsErrors } from '@elastic/elasticsearch';
const { nodeTypes } = esKuery;
jest.mock('./search_dsl/search_dsl', () => ({ getSearchDsl: jest.fn() }));
@ -3654,6 +3655,33 @@ describe('SavedObjectsRepository', () => {
);
});
it(`uses the 'upsertAttributes' option when specified`, async () => {
const upsertAttributes = {
foo: 'bar',
hello: 'dolly',
};
await incrementCounterSuccess(type, id, counterFields, { namespace, upsertAttributes });
expect(client.update).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.objectContaining({
upsert: expect.objectContaining({
[type]: {
foo: 'bar',
hello: 'dolly',
...counterFields.reduce((aggs, field) => {
return {
...aggs,
[field]: 1,
};
}, {}),
},
}),
}),
}),
expect.anything()
);
});
it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => {
await incrementCounterSuccess(type, id, counterFields, { namespace });
expect(client.update).toHaveBeenCalledWith(

58
src/core/server/saved_objects/service/lib/repository.ts
View file

@ -76,10 +76,16 @@ import {
// BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository
// so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient.
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
type Left = { tag: 'Left'; error: Record<string, any> };
// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
type Right = { tag: 'Right'; value: Record<string, any> };
interface Left {
tag: 'Left';
error: Record<string, any>;
}
interface Right {
tag: 'Right';
value: Record<string, any>;
}
type Either = Left | Right;
const isLeft = (either: Either): either is Left => either.tag === 'Left';
const isRight = (either: Either): either is Right => either.tag === 'Right';
@ -98,7 +104,8 @@ export interface SavedObjectsRepositoryOptions {
/**
* @public
*/
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown>
extends SavedObjectsBaseOptions {
/**
* (default=false) If true, sets all the counter fields to 0 if they don't
* already exist. Existing fields will be left as-is and won't be incremented.
@ -111,6 +118,10 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
* operation. See {@link MutatingOperationRefreshSetting}
*/
refresh?: MutatingOperationRefreshSetting;
/**
* Attributes to use when upserting the document if it doesn't exist.
*/
upsertAttributes?: Attributes;
}
/**
@ -1694,6 +1705,20 @@ export class SavedObjectsRepository {
* .incrementCounter('dashboard_counter_type', 'counter_id', [
* 'stats.apiCalls',
* ])
*
* // Increment the apiCalls field counter by 4
* repository
* .incrementCounter('dashboard_counter_type', 'counter_id', [
* { fieldName: 'stats.apiCalls' incrementBy: 4 },
* ])
*
* // Initialize the document with arbitrary fields if not present
* repository.incrementCounter<{ appId: string }>(
* 'dashboard_counter_type',
* 'counter_id',
* [ 'stats.apiCalls'],
* { upsertAttributes: { appId: 'myId' } }
* )
* ```
*
* @param type - The type of saved object whose fields should be incremented
@ -1706,7 +1731,7 @@ export class SavedObjectsRepository {
type: string,
id: string,
counterFields: Array<string | SavedObjectsIncrementCounterField>,
options: SavedObjectsIncrementCounterOptions = {}
options: SavedObjectsIncrementCounterOptions<T> = {}
): Promise<SavedObject<T>> {
if (typeof type !== 'string') {
throw new Error('"type" argument must be a string');
@ -1728,12 +1753,16 @@ export class SavedObjectsRepository {
throw SavedObjectsErrorHelpers.createUnsupportedTypeError(type);
}
const { migrationVersion, refresh = DEFAULT_REFRESH_SETTING, initialize = false } = options;
const {
migrationVersion,
refresh = DEFAULT_REFRESH_SETTING,
initialize = false,
upsertAttributes,
} = options;
const normalizedCounterFields = counterFields.map((counterField) => {
const fieldName = typeof counterField === 'string' ? counterField : counterField.fieldName;
const incrementBy = typeof counterField === 'string' ? 1 : counterField.incrementBy || 1;
return {
fieldName,
incrementBy: initialize ? 0 : incrementBy,
@ -1757,11 +1786,14 @@ export class SavedObjectsRepository {
type,
...(savedObjectNamespace && { namespace: savedObjectNamespace }),
...(savedObjectNamespaces && { namespaces: savedObjectNamespaces }),
attributes: normalizedCounterFields.reduce((acc, counterField) => {
const { fieldName, incrementBy } = counterField;
acc[fieldName] = incrementBy;
return acc;
}, {} as Record<string, number>),
attributes: {
...(upsertAttributes ?? {}),
...normalizedCounterFields.reduce((acc, counterField) => {
const { fieldName, incrementBy } = counterField;
acc[fieldName] = incrementBy;
return acc;
}, {} as Record<string, number>),
},
migrationVersion,
updated_at: time,
});

5
src/core/server/server.api.md
View file

@ -2735,11 +2735,12 @@ export interface SavedObjectsIncrementCounterField {
}
// @public (undocumented)
export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
export interface SavedObjectsIncrementCounterOptions<Attributes = unknown> extends SavedObjectsBaseOptions {
initialize?: boolean;
// (undocumented)
migrationVersion?: SavedObjectsMigrationVersion;
refresh?: MutatingOperationRefreshSetting;
upsertAttributes?: Attributes;
}
// @public
@ -2839,7 +2840,7 @@ export class SavedObjectsRepository {
// (undocumented)
find<T = unknown>(options: SavedObjectsFindOptions): Promise<SavedObjectsFindResponse<T>>;
get<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObject<T>>;
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions): Promise<SavedObject<T>>;
incrementCounter<T = unknown>(type: string, id: string, counterFields: Array<string | SavedObjectsIncrementCounterField>, options?: SavedObjectsIncrementCounterOptions<T>): Promise<SavedObject<T>>;
openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise<SavedObjectsOpenPointInTimeResponse>;
removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise<SavedObjectsRemoveReferencesToResponse>;
resolve<T = unknown>(type: string, id: string, options?: SavedObjectsBaseOptions): Promise<SavedObjectsResolveResponse<T>>;

10
src/plugins/kibana_usage_collection/server/collectors/application_usage/constants.ts
View file

@ -12,15 +12,9 @@
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Roll daily indices every 30 minutes.
* This means that, assuming a user can visit all the 44 apps we can possibly report
* in the 3 minutes interval the browser reports to the server, up to 22 users can have the same
* behaviour and we wouldn't need to paginate in the transactional documents (less than 10k docs).
*
* Based on a more normal expected use case, the users could visit up to 5 apps in those 3 minutes,
* allowing up to 200 users before reaching the limit.
* Roll daily indices every 24h
*/
export const ROLL_DAILY_INDICES_INTERVAL = 30 * 60 * 1000;
export const ROLL_DAILY_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up

1
src/plugins/kibana_usage_collection/server/collectors/application_usage/index.ts
View file

@ -7,3 +7,4 @@
*/
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';
export { rollDailyData as migrateTransactionalDocs } from './rollups';

205
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.test.ts → src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.test.ts
View file

@ -6,21 +6,16 @@
* Side Public License, v 1.
*/
import { rollDailyData, rollTotals } from './rollups';
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
import { SavedObjectsErrorHelpers } from '../../../../../core/server';
import {
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
import { SavedObjectsErrorHelpers } from '../../../../../../core/server';
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TRANSACTIONAL_TYPE } from '../saved_objects_types';
import { rollDailyData } from './daily';
describe('rollDailyData', () => {
const logger = loggingSystemMock.createLogger();
test('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollDailyData(logger, undefined)).resolves.toBe(undefined);
test('returns false if no savedObjectsClient initialised yet', async () => {
await expect(rollDailyData(logger, undefined)).resolves.toBe(false);
});
test('handle empty results', async () => {
@ -33,7 +28,7 @@ describe('rollDailyData', () => {
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
expect(savedObjectClient.get).not.toBeCalled();
expect(savedObjectClient.bulkCreate).not.toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
@ -101,7 +96,7 @@ describe('rollDailyData', () => {
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
expect(savedObjectClient.get).toHaveBeenCalledTimes(2);
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
1,
@ -196,7 +191,7 @@ describe('rollDailyData', () => {
throw new Error('Something went terribly wrong');
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(false);
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
expect(savedObjectClient.get).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
@ -206,185 +201,3 @@ describe('rollDailyData', () => {
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
});
describe('rollTotals', () => {
const logger = loggingSystemMock.createLogger();
test('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
});
test('handle empty results', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_DAILY_TYPE:
case SAVED_OBJECTS_TOTAL_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
test('migrate some documents', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_DAILY_TYPE:
return {
saved_objects: [
{
id: 'appId-2:2020-01-01',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-2',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'appId-1:2020-01-01',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 2.5,
numberOfClicks: 2,
},
},
{
id: 'appId-1:2020-01-01:viewId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
viewId: 'viewId-1',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 1,
numberOfClicks: 1,
},
},
],
total: 3,
page,
per_page: perPage,
};
case SAVED_OBJECTS_TOTAL_TYPE:
return {
saved_objects: [
{
id: 'appId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'appId-1___viewId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
viewId: 'viewId-1',
minutesOnScreen: 4,
numberOfClicks: 2,
},
},
{
id: 'appId-2___viewId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-2',
viewId: 'viewId-1',
minutesOnScreen: 1,
numberOfClicks: 1,
},
},
],
total: 3,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-1',
attributes: {
appId: 'appId-1',
viewId: MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen: 3.0,
numberOfClicks: 3,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-1___viewId-1',
attributes: {
appId: 'appId-1',
viewId: 'viewId-1',
minutesOnScreen: 5.0,
numberOfClicks: 3,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-2___viewId-1',
attributes: {
appId: 'appId-2',
viewId: 'viewId-1',
minutesOnScreen: 1.0,
numberOfClicks: 1,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-2',
attributes: {
appId: 'appId-2',
viewId: MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-2:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01:viewId-1'
);
});
});

126
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.ts → src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.ts
View file

@ -6,18 +6,20 @@
* Side Public License, v 1.
*/
import { ISavedObjectsRepository, SavedObject, Logger } from 'kibana/server';
import moment from 'moment';
import type { Logger } from '@kbn/logging';
import {
ISavedObjectsRepository,
SavedObject,
SavedObjectsErrorHelpers,
} from '../../../../../../core/server';
import { getDailyId } from '../../../../../usage_collection/common/application_usage';
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
ApplicationUsageTransactional,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
} from '../saved_objects_types';
/**
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
@ -27,18 +29,17 @@ type ApplicationUsageDailyWithVersion = Pick<
'version' | 'attributes'
>;
export function serializeKey(appId: string, viewId: string) {
return `${appId}___${viewId}`;
}
/**
* Aggregates all the transactional events into daily aggregates
* @param logger
* @param savedObjectsClient
*/
export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
export async function rollDailyData(
logger: Logger,
savedObjectsClient?: ISavedObjectsRepository
): Promise<boolean> {
if (!savedObjectsClient) {
return;
return false;
}
try {
@ -58,10 +59,7 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
} = doc;
const dayId = moment(timestamp).format('YYYY-MM-DD');
const dailyId =
!viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
? `${appId}:${dayId}`
: `${appId}:${dayId}:${viewId}`;
const dailyId = getDailyId({ dayId, appId, viewId });
const existingDoc =
toCreate.get(dailyId) ||
@ -103,9 +101,11 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
}
}
} while (toCreate.size > 0);
return true;
} catch (err) {
logger.debug(`Failed to rollup transactional to daily entries`);
logger.debug(err);
return false;
}
}
@ -125,7 +125,11 @@ async function getDailyDoc(
dayId: string
): Promise<ApplicationUsageDailyWithVersion> {
try {
return await savedObjectsClient.get<ApplicationUsageDaily>(SAVED_OBJECTS_DAILY_TYPE, id);
const { attributes, version } = await savedObjectsClient.get<ApplicationUsageDaily>(
SAVED_OBJECTS_DAILY_TYPE,
id
);
return { attributes, version };
} catch (err) {
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
return {
@ -142,91 +146,3 @@ async function getDailyDoc(
throw err;
}
}
/**
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
* @param logger
* @param savedObjectsClient
*/
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
if (!savedObjectsClient) {
return;
}
try {
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
perPage: 10000,
type: SAVED_OBJECTS_TOTAL_TYPE,
}),
savedObjectsClient.find<ApplicationUsageDaily>({
perPage: 10000,
type: SAVED_OBJECTS_DAILY_TYPE,
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
}),
]);
const existingTotals = rawApplicationUsageTotals.reduce(
(
acc,
{
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
}
) => {
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
return {
...acc,
// No need to sum because there should be 1 document per appId only
[key]: { appId, viewId, numberOfClicks, minutesOnScreen },
};
},
{} as Record<
string,
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
>
);
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
const {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
numberOfClicks,
minutesOnScreen,
} = attributes;
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
return {
...acc,
[key]: {
appId,
viewId,
numberOfClicks: numberOfClicks + existing.numberOfClicks,
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
},
};
}, existingTotals);
await Promise.all([
Object.entries(totals).length &&
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
Object.entries(totals).map(([id, entry]) => ({
type: SAVED_OBJECTS_TOTAL_TYPE,
id,
attributes: entry,
})),
{ overwrite: true }
),
...rawApplicationUsageDaily.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
),
]);
} catch (err) {
logger.debug(`Failed to rollup daily entries to totals`);
logger.debug(err);
}
}

11
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/index.ts Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export { rollDailyData } from './daily';
export { rollTotals } from './total';
export { serializeKey } from './utils';

194
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.test.ts Normal file
View file

@ -0,0 +1,194 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from '../saved_objects_types';
import { rollTotals } from './total';
describe('rollTotals', () => {
const logger = loggingSystemMock.createLogger();
test('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
});
test('handle empty results', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_DAILY_TYPE:
case SAVED_OBJECTS_TOTAL_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
test('migrate some documents', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_DAILY_TYPE:
return {
saved_objects: [
{
id: 'appId-2:2020-01-01',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-2',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'appId-1:2020-01-01',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 2.5,
numberOfClicks: 2,
},
},
{
id: 'appId-1:2020-01-01:viewId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
viewId: 'viewId-1',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 1,
numberOfClicks: 1,
},
},
],
total: 3,
page,
per_page: perPage,
};
case SAVED_OBJECTS_TOTAL_TYPE:
return {
saved_objects: [
{
id: 'appId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'appId-1___viewId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
viewId: 'viewId-1',
minutesOnScreen: 4,
numberOfClicks: 2,
},
},
{
id: 'appId-2___viewId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-2',
viewId: 'viewId-1',
minutesOnScreen: 1,
numberOfClicks: 1,
},
},
],
total: 3,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-1',
attributes: {
appId: 'appId-1',
viewId: MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen: 3.0,
numberOfClicks: 3,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-1___viewId-1',
attributes: {
appId: 'appId-1',
viewId: 'viewId-1',
minutesOnScreen: 5.0,
numberOfClicks: 3,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-2___viewId-1',
attributes: {
appId: 'appId-2',
viewId: 'viewId-1',
minutesOnScreen: 1.0,
numberOfClicks: 1,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-2',
attributes: {
appId: 'appId-2',
viewId: MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-2:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01:viewId-1'
);
});
});

106
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.ts Normal file
View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { Logger } from '@kbn/logging';
import type { ISavedObjectsRepository } from 'kibana/server';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
} from '../saved_objects_types';
import { serializeKey } from './utils';
/**
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
* @param logger
* @param savedObjectsClient
*/
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
if (!savedObjectsClient) {
return;
}
try {
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
perPage: 10000,
type: SAVED_OBJECTS_TOTAL_TYPE,
}),
savedObjectsClient.find<ApplicationUsageDaily>({
perPage: 10000,
type: SAVED_OBJECTS_DAILY_TYPE,
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
}),
]);
const existingTotals = rawApplicationUsageTotals.reduce(
(
acc,
{
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
}
) => {
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
return {
...acc,
// No need to sum because there should be 1 document per appId only
[key]: { appId, viewId, numberOfClicks, minutesOnScreen },
};
},
{} as Record<
string,
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
>
);
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
const {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
numberOfClicks,
minutesOnScreen,
} = attributes;
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
return {
...acc,
[key]: {
appId,
viewId,
numberOfClicks: numberOfClicks + existing.numberOfClicks,
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
},
};
}, existingTotals);
await Promise.all([
Object.entries(totals).length &&
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
Object.entries(totals).map(([id, entry]) => ({
type: SAVED_OBJECTS_TOTAL_TYPE,
id,
attributes: entry,
})),
{ overwrite: true }
),
...rawApplicationUsageDaily.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
),
]);
} catch (err) {
logger.debug(`Failed to rollup daily entries to totals`);
logger.debug(err);
}
}

11
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/utils.ts Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export function serializeKey(appId: string, viewId: string) {
return `${appId}___${viewId}`;
}

6
src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts
View file

@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
import { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
import type { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
/**
* Used for accumulating the totals of all the stats older than 90d
@ -17,6 +17,7 @@ export interface ApplicationUsageTotal extends SavedObjectAttributes {
minutesOnScreen: number;
numberOfClicks: number;
}
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
/**
@ -25,6 +26,8 @@ export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
export interface ApplicationUsageTransactional extends ApplicationUsageTotal {
timestamp: string;
}
/** @deprecated transactional type is no longer used, and only preserved for backward compatibility */
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
/**
@ -62,6 +65,7 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
});
// Type for storing ApplicationUsageTransactional (declaring empty mappings because we don't use the internal fields for query/aggregations)
// Remark: this type is deprecated and only here for BWC reasons.
registerType({
name: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
hidden: false,

2
src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
View file

@ -7,7 +7,7 @@
*/
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
import { ApplicationUsageTelemetryReport } from './telemetry_application_usage_collector';
import { ApplicationUsageTelemetryReport } from './types';
const commonSchema: MakeSchemaFrom<ApplicationUsageTelemetryReport[string]> = {
appId: { type: 'keyword', _meta: { description: 'The application being tracked' } },

232
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
View file

@ -11,74 +11,99 @@ import {
Collector,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import {
registerApplicationUsageCollector,
transformByApplicationViews,
ApplicationUsageViews,
} from './telemetry_application_usage_collector';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
import {
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { ApplicationUsageViews } from './types';
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from './saved_objects_types';
// use fake timers to avoid triggering rollups during tests
jest.useFakeTimers();
describe('telemetry_application_usage', () => {
jest.useFakeTimers();
const logger = loggingSystemMock.createLogger();
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
let collector: Collector<unknown>;
let usageCollectionMock: ReturnType<typeof createUsageCollectionSetupMock>;
let savedObjectClient: ReturnType<typeof savedObjectsRepositoryMock.create>;
let getSavedObjectClient: jest.MockedFunction<() => undefined | typeof savedObjectClient>;
const usageCollectionMock = createUsageCollectionSetupMock();
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = new Collector(logger, config);
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
const getUsageCollector = jest.fn();
const registerType = jest.fn();
const mockedFetchContext = createCollectorFetchContextMock();
beforeAll(() =>
registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector)
);
afterAll(() => jest.clearAllTimers());
beforeEach(() => {
logger = loggingSystemMock.createLogger();
usageCollectionMock = createUsageCollectionSetupMock();
savedObjectClient = savedObjectsRepositoryMock.create();
getSavedObjectClient = jest.fn().mockReturnValue(savedObjectClient);
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = new Collector(logger, config);
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
registerApplicationUsageCollector(
logger,
usageCollectionMock,
registerType,
getSavedObjectClient
);
});
afterEach(() => {
jest.clearAllTimers();
});
test('registered collector is set', () => {
expect(collector).not.toBeUndefined();
});
test('if no savedObjectClient initialised, return undefined', async () => {
getSavedObjectClient.mockReturnValue(undefined);
expect(collector.isReady()).toBe(false);
expect(await collector.fetch(mockedFetchContext)).toBeUndefined();
jest.runTimersToTime(ROLL_INDICES_START);
});
test('calls `savedObjectsClient.find` with the correct parameters', async () => {
savedObjectClient.find.mockResolvedValue({
saved_objects: [],
total: 0,
per_page: 20,
page: 0,
});
await collector.fetch(mockedFetchContext);
expect(savedObjectClient.find).toHaveBeenCalledTimes(2);
expect(savedObjectClient.find).toHaveBeenCalledWith(
expect.objectContaining({
type: SAVED_OBJECTS_TOTAL_TYPE,
})
);
expect(savedObjectClient.find).toHaveBeenCalledWith(
expect.objectContaining({
type: SAVED_OBJECTS_DAILY_TYPE,
})
);
});
test('when savedObjectClient is initialised, return something', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(
async () =>
({
saved_objects: [],
total: 0,
} as any)
);
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
savedObjectClient.find.mockResolvedValue({
saved_objects: [],
total: 0,
per_page: 20,
page: 0,
});
expect(collector.isReady()).toBe(true);
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({});
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
});
test('it only gets 10k even when there are more documents (ES limitation)', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
const total = 10000;
test('it aggregates total and daily data', async () => {
savedObjectClient.find.mockImplementation(async (opts) => {
switch (opts.type) {
case SAVED_OBJECTS_TOTAL_TYPE:
@ -95,18 +120,6 @@ describe('telemetry_application_usage', () => {
],
total: 1,
} as any;
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
const doc = {
id: 'test-id',
attributes: {
appId: 'appId',
timestamp: new Date().toISOString(),
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
};
const savedObjects = new Array(total).fill(doc);
return { saved_objects: savedObjects, total: total + 1 };
case SAVED_OBJECTS_DAILY_TYPE:
return {
saved_objects: [
@ -125,122 +138,21 @@ describe('telemetry_application_usage', () => {
}
});
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
appId: {
appId: 'appId',
viewId: 'main',
clicks_total: total + 1 + 10,
clicks_7_days: total + 1,
clicks_30_days: total + 1,
clicks_90_days: total + 1,
minutes_on_screen_total: (total + 1) * 0.5 + 10,
minutes_on_screen_7_days: (total + 1) * 0.5,
minutes_on_screen_30_days: (total + 1) * 0.5,
minutes_on_screen_90_days: (total + 1) * 0.5,
clicks_total: 1 + 10,
clicks_7_days: 1,
clicks_30_days: 1,
clicks_90_days: 1,
minutes_on_screen_total: 0.5 + 10,
minutes_on_screen_7_days: 0.5,
minutes_on_screen_30_days: 0.5,
minutes_on_screen_90_days: 0.5,
views: [],
},
});
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
id: 'appId',
type: SAVED_OBJECTS_TOTAL_TYPE,
attributes: {
appId: 'appId',
viewId: 'main',
minutesOnScreen: 10.5,
numberOfClicks: 11,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId:YYYY-MM-DD'
);
});
test('old transactional data not migrated yet', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async (opts) => {
switch (opts.type) {
case SAVED_OBJECTS_TOTAL_TYPE:
case SAVED_OBJECTS_DAILY_TYPE:
return { saved_objects: [], total: 0 } as any;
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
return {
saved_objects: [
{
id: 'test-id',
attributes: {
appId: 'appId',
timestamp: new Date(0).toISOString(),
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'test-id-2',
attributes: {
appId: 'appId',
viewId: 'main',
timestamp: new Date(0).toISOString(),
minutesOnScreen: 2,
numberOfClicks: 2,
},
},
{
id: 'test-id-3',
attributes: {
appId: 'appId',
viewId: 'viewId-1',
timestamp: new Date(0).toISOString(),
minutesOnScreen: 1,
numberOfClicks: 1,
},
},
],
total: 1,
};
}
});
getUsageCollector.mockImplementation(() => savedObjectClient);
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
appId: {
appId: 'appId',
viewId: 'main',
clicks_total: 3,
clicks_7_days: 0,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: 2.5,
minutes_on_screen_7_days: 0,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
views: [
{
appId: 'appId',
viewId: 'viewId-1',
clicks_total: 1,
clicks_7_days: 0,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: 1,
minutes_on_screen_7_days: 0,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
},
],
},
});
});
});

72
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
View file

@ -11,57 +11,21 @@ import { timer } from 'rxjs';
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from 'kibana/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
import { serializeKey } from './rollups';
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
ApplicationUsageTransactional,
registerMappings,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { applicationUsageSchema } from './schema';
import { rollDailyData, rollTotals } from './rollups';
import { rollTotals, rollDailyData, serializeKey } from './rollups';
import {
ROLL_TOTAL_INDICES_INTERVAL,
ROLL_DAILY_INDICES_INTERVAL,
ROLL_INDICES_START,
} from './constants';
export interface ApplicationViewUsage {
appId: string;
viewId: string;
clicks_total: number;
clicks_7_days: number;
clicks_30_days: number;
clicks_90_days: number;
minutes_on_screen_total: number;
minutes_on_screen_7_days: number;
minutes_on_screen_30_days: number;
minutes_on_screen_90_days: number;
}
export interface ApplicationUsageViews {
[serializedKey: string]: ApplicationViewUsage;
}
export interface ApplicationUsageTelemetryReport {
[appId: string]: {
appId: string;
viewId: string;
clicks_total: number;
clicks_7_days: number;
clicks_30_days: number;
clicks_90_days: number;
minutes_on_screen_total: number;
minutes_on_screen_7_days: number;
minutes_on_screen_30_days: number;
minutes_on_screen_90_days: number;
views?: ApplicationViewUsage[];
};
}
import { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
export const transformByApplicationViews = (
report: ApplicationUsageViews
@ -92,6 +56,21 @@ export function registerApplicationUsageCollector(
) {
registerMappings(registerType);
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
rollTotals(logger, getSavedObjectsClient())
);
const dailyRollingSub = timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(
async () => {
const success = await rollDailyData(logger, getSavedObjectsClient());
// we only need to roll the transactional documents once to assure BWC
// once we rolling succeeds, we can stop.
if (success) {
dailyRollingSub.unsubscribe();
}
}
);
const collector = usageCollection.makeUsageCollector<ApplicationUsageTelemetryReport | undefined>(
{
type: 'application_usage',
@ -105,7 +84,6 @@ export function registerApplicationUsageCollector(
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
{ saved_objects: rawApplicationUsageTransactional },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
type: SAVED_OBJECTS_TOTAL_TYPE,
@ -115,10 +93,6 @@ export function registerApplicationUsageCollector(
type: SAVED_OBJECTS_DAILY_TYPE,
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
}),
savedObjectsClient.find<ApplicationUsageTransactional>({
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
perPage: 10000, // If we have more than those, we won't report the rest (they'll be rolled up to the daily soon enough to become a problem)
}),
]);
const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
@ -156,10 +130,7 @@ export function registerApplicationUsageCollector(
const nowMinus30 = moment().subtract(30, 'days');
const nowMinus90 = moment().subtract(90, 'days');
const applicationUsage = [
...rawApplicationUsageDaily,
...rawApplicationUsageTransactional,
].reduce(
const applicationUsage = rawApplicationUsageDaily.reduce(
(
acc,
{
@ -224,11 +195,4 @@ export function registerApplicationUsageCollector(
);
usageCollection.registerCollector(collector);
timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(() =>
rollDailyData(logger, getSavedObjectsClient())
);
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
rollTotals(logger, getSavedObjectsClient())
);
}

40
src/plugins/kibana_usage_collection/server/collectors/application_usage/types.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export interface ApplicationViewUsage {
appId: string;
viewId: string;
clicks_total: number;
clicks_7_days: number;
clicks_30_days: number;
clicks_90_days: number;
minutes_on_screen_total: number;
minutes_on_screen_7_days: number;
minutes_on_screen_30_days: number;
minutes_on_screen_90_days: number;
}
export interface ApplicationUsageViews {
[serializedKey: string]: ApplicationViewUsage;
}
export interface ApplicationUsageTelemetryReport {
[appId: string]: {
appId: string;
viewId: string;
clicks_total: number;
clicks_7_days: number;
clicks_30_days: number;
clicks_90_days: number;
minutes_on_screen_total: number;
minutes_on_screen_7_days: number;
minutes_on_screen_30_days: number;
minutes_on_screen_90_days: number;
views?: ApplicationViewUsage[];
};
}

23
src/plugins/usage_collection/common/application_usage.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { MAIN_APP_DEFAULT_VIEW_ID } from './constants';
export const getDailyId = ({
appId,
dayId,
viewId,
}: {
viewId: string;
appId: string;
dayId: string;
}) => {
return !viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
? `${appId}:${dayId}`
: `${appId}:${dayId}:${viewId}`;
};

20
src/plugins/usage_collection/server/report/schema.ts
View file

@ -9,6 +9,13 @@
import { schema, TypeOf } from '@kbn/config-schema';
import { METRIC_TYPE } from '@kbn/analytics';
const applicationUsageReportSchema = schema.object({
minutesOnScreen: schema.number(),
numberOfClicks: schema.number(),
appId: schema.string(),
viewId: schema.string(),
});
export const reportSchema = schema.object({
reportVersion: schema.maybe(schema.oneOf([schema.literal(3)])),
userAgent: schema.maybe(
@ -38,17 +45,8 @@ export const reportSchema = schema.object({
})
)
),
application_usage: schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
minutesOnScreen: schema.number(),
numberOfClicks: schema.number(),
appId: schema.string(),
viewId: schema.string(),
})
)
),
application_usage: schema.maybe(schema.recordOf(schema.string(), applicationUsageReportSchema)),
});
export type ReportSchemaType = TypeOf<typeof reportSchema>;
export type ApplicationUsageReport = TypeOf<typeof applicationUsageReportSchema>;

115
src/plugins/usage_collection/server/report/store_application_usage.test.ts Normal file
View file

@ -0,0 +1,115 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { getDailyId } from '../../common/application_usage';
import { storeApplicationUsage } from './store_application_usage';
import { ApplicationUsageReport } from './schema';
const createReport = (parts: Partial<ApplicationUsageReport>): ApplicationUsageReport => ({
appId: 'appId',
viewId: 'viewId',
numberOfClicks: 0,
minutesOnScreen: 0,
...parts,
});
describe('storeApplicationUsage', () => {
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
let timestamp: Date;
beforeEach(() => {
repository = savedObjectsRepositoryMock.create();
timestamp = new Date();
});
it('does not call `repository.incrementUsageCounters` when the report list is empty', async () => {
await storeApplicationUsage(repository, [], timestamp);
expect(repository.incrementCounter).not.toHaveBeenCalled();
});
it('calls `repository.incrementUsageCounters` with the correct parameters', async () => {
const report = createReport({
appId: 'app1',
viewId: 'view1',
numberOfClicks: 2,
minutesOnScreen: 5,
});
await storeApplicationUsage(repository, [report], timestamp);
expect(repository.incrementCounter).toHaveBeenCalledTimes(1);
expect(repository.incrementCounter).toHaveBeenCalledWith(
...expectedIncrementParams(report, timestamp)
);
});
it('aggregates reports with the same appId/viewId tuple', async () => {
const report1 = createReport({
appId: 'app1',
viewId: 'view1',
numberOfClicks: 2,
minutesOnScreen: 5,
});
const report2 = createReport({
appId: 'app1',
viewId: 'view2',
numberOfClicks: 1,
minutesOnScreen: 7,
});
const report3 = createReport({
appId: 'app1',
viewId: 'view1',
numberOfClicks: 3,
minutesOnScreen: 9,
});
await storeApplicationUsage(repository, [report1, report2, report3], timestamp);
expect(repository.incrementCounter).toHaveBeenCalledTimes(2);
expect(repository.incrementCounter).toHaveBeenCalledWith(
...expectedIncrementParams(
{
appId: 'app1',
viewId: 'view1',
numberOfClicks: report1.numberOfClicks + report3.numberOfClicks,
minutesOnScreen: report1.minutesOnScreen + report3.minutesOnScreen,
},
timestamp
)
);
expect(repository.incrementCounter).toHaveBeenCalledWith(
...expectedIncrementParams(report2, timestamp)
);
});
});
const expectedIncrementParams = (
{ appId, viewId, minutesOnScreen, numberOfClicks }: ApplicationUsageReport,
timestamp: Date
) => {
const dayId = moment(timestamp).format('YYYY-MM-DD');
return [
'application_usage_daily',
getDailyId({ appId, viewId, dayId }),
[
{ fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
{ fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
],
{
upsertAttributes: {
appId,
viewId,
timestamp: moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString(),
},
},
];
};

87
src/plugins/usage_collection/server/report/store_application_usage.ts Normal file
View file

@ -0,0 +1,87 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import { Writable } from '@kbn/utility-types';
import { ISavedObjectsRepository } from 'src/core/server';
import { ApplicationUsageReport } from './schema';
import { getDailyId } from '../../common/application_usage';
type WritableApplicationUsageReport = Writable<ApplicationUsageReport>;
export const storeApplicationUsage = async (
repository: ISavedObjectsRepository,
appUsages: ApplicationUsageReport[],
timestamp: Date
) => {
if (!appUsages.length) {
return;
}
const dayId = getDayId(timestamp);
const aggregatedReports = aggregateAppUsages(appUsages);
return Promise.allSettled(
aggregatedReports.map(async (report) => incrementUsageCounters(repository, report, dayId))
);
};
const aggregateAppUsages = (appUsages: ApplicationUsageReport[]) => {
return [
...appUsages
.reduce((map, appUsage) => {
const key = getKey(appUsage);
const aggregated: WritableApplicationUsageReport = map.get(key) ?? {
appId: appUsage.appId,
viewId: appUsage.viewId,
minutesOnScreen: 0,
numberOfClicks: 0,
};
aggregated.minutesOnScreen += appUsage.minutesOnScreen;
aggregated.numberOfClicks += appUsage.numberOfClicks;
map.set(key, aggregated);
return map;
}, new Map<string, ApplicationUsageReport>())
.values(),
];
};
const incrementUsageCounters = (
repository: ISavedObjectsRepository,
{ appId, viewId, numberOfClicks, minutesOnScreen }: WritableApplicationUsageReport,
dayId: string
) => {
const dailyId = getDailyId({ appId, viewId, dayId });
return repository.incrementCounter(
'application_usage_daily',
dailyId,
[
{ fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
{ fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
],
{
upsertAttributes: {
appId,
viewId,
timestamp: getTimestamp(dayId),
},
}
);
};
const getKey = ({ viewId, appId }: ApplicationUsageReport) => `${appId}___${viewId}`;
const getDayId = (timestamp: Date) => moment(timestamp).format('YYYY-MM-DD');
const getTimestamp = (dayId: string) => {
// Concatenating the day in YYYY-MM-DD form to T00:00:00Z to reduce the TZ effects
return moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString();
};

12
src/plugins/usage_collection/server/report/store_report.test.mocks.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export const storeApplicationUsageMock = jest.fn();
jest.doMock('./store_application_usage', () => ({
storeApplicationUsage: storeApplicationUsageMock,
}));

53
src/plugins/usage_collection/server/report/store_report.test.ts
View file

@ -6,6 +6,8 @@
* Side Public License, v 1.
*/
import { storeApplicationUsageMock } from './store_report.test.mocks';
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { storeReport } from './store_report';
import { ReportSchemaType } from './schema';
@ -16,8 +18,17 @@ describe('store_report', () => {
const momentTimestamp = moment();
const date = momentTimestamp.format('DDMMYYYY');
let repository: ReturnType<typeof savedObjectsRepositoryMock.create>;
beforeEach(() => {
repository = savedObjectsRepositoryMock.create();
});
afterEach(() => {
storeApplicationUsageMock.mockReset();
});
test('stores report for all types of data', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
const report: ReportSchemaType = {
reportVersion: 3,
userAgent: {
@ -53,9 +64,9 @@ describe('store_report', () => {
},
},
};
await storeReport(savedObjectClient, report);
await storeReport(repository, report);
expect(savedObjectClient.create).toHaveBeenCalledWith(
expect(repository.create).toHaveBeenCalledWith(
'ui-metric',
{ count: 1 },
{
@ -63,51 +74,45 @@ describe('store_report', () => {
overwrite: true,
}
);
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
1,
'ui-metric',
'test-app-name:test-event-name',
[{ fieldName: 'count', incrementBy: 3 }]
);
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
2,
'ui-counter',
`test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`,
[{ fieldName: 'count', incrementBy: 1 }]
);
expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
expect(repository.incrementCounter).toHaveBeenNthCalledWith(
3,
'ui-counter',
`test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`,
[{ fieldName: 'count', incrementBy: 2 }]
);
expect(savedObjectClient.bulkCreate).toHaveBeenNthCalledWith(1, [
{
type: 'application_usage_transactional',
attributes: {
numberOfClicks: 3,
minutesOnScreen: 10,
appId: 'appId',
viewId: 'appId_view',
timestamp: expect.any(Date),
},
},
]);
expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1);
expect(storeApplicationUsageMock).toHaveBeenCalledWith(
repository,
Object.values(report.application_usage as Record<string, any>),
expect.any(Date)
);
});
test('it should not fail if nothing to store', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
const report: ReportSchemaType = {
reportVersion: 3,
userAgent: void 0,
uiCounter: void 0,
application_usage: void 0,
};
await storeReport(savedObjectClient, report);
await storeReport(repository, report);
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
expect(savedObjectClient.incrementCounter).not.toHaveBeenCalled();
expect(savedObjectClient.create).not.toHaveBeenCalled();
expect(savedObjectClient.create).not.toHaveBeenCalled();
expect(repository.bulkCreate).not.toHaveBeenCalled();
expect(repository.incrementCounter).not.toHaveBeenCalled();
expect(repository.create).not.toHaveBeenCalled();
expect(repository.create).not.toHaveBeenCalled();
});
});

22
src/plugins/usage_collection/server/report/store_report.ts
View file

@ -10,6 +10,7 @@ import { ISavedObjectsRepository } from 'src/core/server';
import moment from 'moment';
import { chain, sumBy } from 'lodash';
import { ReportSchemaType } from './schema';
import { storeApplicationUsage } from './store_application_usage';
export async function storeReport(
internalRepository: ISavedObjectsRepository,
@ -17,11 +18,11 @@ export async function storeReport(
) {
const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : [];
const userAgents = report.userAgent ? Object.entries(report.userAgent) : [];
const appUsage = report.application_usage ? Object.values(report.application_usage) : [];
const appUsages = report.application_usage ? Object.values(report.application_usage) : [];
const momentTimestamp = moment();
const timestamp = momentTimestamp.toDate();
const date = momentTimestamp.format('DDMMYYYY');
const timestamp = momentTimestamp.toDate();
return Promise.allSettled([
// User Agent
@ -64,21 +65,6 @@ export async function storeReport(
];
}),
// Application Usage
...[
(async () => {
if (!appUsage.length) return [];
const { saved_objects: savedObjects } = await internalRepository.bulkCreate(
appUsage.map((metric) => ({
type: 'application_usage_transactional',
attributes: {
...metric,
timestamp,
},
}))
);
return savedObjects;
})(),
],
storeApplicationUsage(internalRepository, appUsages, timestamp),
]);
}

11
test/api_integration/apis/telemetry/telemetry_local.ts
View file

@ -156,7 +156,7 @@ export default function ({ getService }: FtrProviderContext) {
describe('application usage limits', () => {
function createSavedObject(viewId?: string) {
return supertest
.post('/api/saved_objects/application_usage_transactional')
.post('/api/saved_objects/application_usage_daily')
.send({
attributes: {
appId: 'test-app',
@ -184,7 +184,7 @@ export default function ({ getService }: FtrProviderContext) {
await Promise.all(
savedObjectIds.map((savedObjectId) => {
return supertest
.delete(`/api/saved_objects/application_usage_transactional/${savedObjectId}`)
.delete(`/api/saved_objects/application_usage_daily/${savedObjectId}`)
.expect(200);
})
);
@ -230,7 +230,7 @@ export default function ({ getService }: FtrProviderContext) {
.post('/api/saved_objects/_bulk_create')
.send(
new Array(10001).fill(0).map(() => ({
type: 'application_usage_transactional',
type: 'application_usage_daily',
attributes: {
appId: 'test-app',
minutesOnScreen: 1,
@ -248,13 +248,12 @@ export default function ({ getService }: FtrProviderContext) {
// The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout
await es.deleteByQuery({
index: '.kibana',
body: { query: { term: { type: 'application_usage_transactional' } } },
body: { query: { term: { type: 'application_usage_daily' } } },
conflicts: 'proceed',
});
});
// flaky https://github.com/elastic/kibana/issues/94513
it.skip("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
const stats = await retrieveTelemetry(supertest);
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
'test-app': {