diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.md
index 68e9bb09456c..8da2458cf007 100644
--- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.md
+++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.md
@@ -8,7 +8,7 @@
Signature:
```typescript
-export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions
+export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions
```
## Properties
@@ -18,4 +18,5 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
| [initialize](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.initialize.md) | boolean
| (default=false) If true, sets all the counter fields to 0 if they don't already exist. Existing fields will be left as-is and won't be incremented. |
| [migrationVersion](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.migrationversion.md) | SavedObjectsMigrationVersion
| [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) |
| [refresh](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.refresh.md) | MutatingOperationRefreshSetting
| (default='wait\_for') The Elasticsearch refresh setting for this operation. See [MutatingOperationRefreshSetting](./kibana-plugin-core-server.mutatingoperationrefreshsetting.md) |
+| [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md) | Attributes
| Attributes to use when upserting the document if it doesn't exist. |
diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md
new file mode 100644
index 000000000000..d5657dd65771
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md
@@ -0,0 +1,13 @@
+
+
+[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) > [upsertAttributes](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.upsertattributes.md)
+
+## SavedObjectsIncrementCounterOptions.upsertAttributes property
+
+Attributes to use when upserting the document if it doesn't exist.
+
+Signature:
+
+```typescript
+upsertAttributes?: Attributes;
+```
diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md
index eb18e064c84e..59d98bf4d607 100644
--- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md
+++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md
@@ -9,7 +9,7 @@ Increments all the specified counter fields (by one by default). Creates the doc
Signature:
```typescript
-incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>;
+incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>;
```
## Parameters
@@ -19,7 +19,7 @@ incrementCounter(type: string, id: string, counterFields: Arraystring | The type of saved object whose fields should be incremented |
| id | string
| The id of the document whose fields should be incremented |
| counterFields | Array<string | SavedObjectsIncrementCounterField>
| An array of field names to increment or an array of [SavedObjectsIncrementCounterField](./kibana-plugin-core-server.savedobjectsincrementcounterfield.md) |
-| options | SavedObjectsIncrementCounterOptions
| [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
+| options | SavedObjectsIncrementCounterOptions<T>
| [SavedObjectsIncrementCounterOptions](./kibana-plugin-core-server.savedobjectsincrementcounteroptions.md) |
Returns:
@@ -52,5 +52,19 @@ repository
'stats.apiCalls',
])
+// Increment the apiCalls field counter by 4
+repository
+ .incrementCounter('dashboard_counter_type', 'counter_id', [
+ { fieldName: 'stats.apiCalls' incrementBy: 4 },
+ ])
+
+// Initialize the document with arbitrary fields if not present
+repository.incrementCounter<{ appId: string }>(
+ 'dashboard_counter_type',
+ 'counter_id',
+ [ 'stats.apiCalls'],
+ { upsertAttributes: { appId: 'myId' } }
+)
+
```
diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js
index 37572c83e4c8..ce48e8dc9a31 100644
--- a/src/core/server/saved_objects/service/lib/repository.test.js
+++ b/src/core/server/saved_objects/service/lib/repository.test.js
@@ -23,6 +23,7 @@ import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock
import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks';
import { esKuery } from '../../es_query';
import { errors as EsErrors } from '@elastic/elasticsearch';
+
const { nodeTypes } = esKuery;
jest.mock('./search_dsl/search_dsl', () => ({ getSearchDsl: jest.fn() }));
@@ -3654,6 +3655,33 @@ describe('SavedObjectsRepository', () => {
);
});
+ it(`uses the 'upsertAttributes' option when specified`, async () => {
+ const upsertAttributes = {
+ foo: 'bar',
+ hello: 'dolly',
+ };
+ await incrementCounterSuccess(type, id, counterFields, { namespace, upsertAttributes });
+ expect(client.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({
+ upsert: expect.objectContaining({
+ [type]: {
+ foo: 'bar',
+ hello: 'dolly',
+ ...counterFields.reduce((aggs, field) => {
+ return {
+ ...aggs,
+ [field]: 1,
+ };
+ }, {}),
+ },
+ }),
+ }),
+ }),
+ expect.anything()
+ );
+ });
+
it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => {
await incrementCounterSuccess(type, id, counterFields, { namespace });
expect(client.update).toHaveBeenCalledWith(
diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts
index aa1e62c1652c..6e2a1d6ec051 100644
--- a/src/core/server/saved_objects/service/lib/repository.ts
+++ b/src/core/server/saved_objects/service/lib/repository.ts
@@ -76,10 +76,16 @@ import {
// BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository
// so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient.
-// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
-type Left = { tag: 'Left'; error: Record };
-// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
-type Right = { tag: 'Right'; value: Record };
+interface Left {
+ tag: 'Left';
+ error: Record;
+}
+
+interface Right {
+ tag: 'Right';
+ value: Record;
+}
+
type Either = Left | Right;
const isLeft = (either: Either): either is Left => either.tag === 'Left';
const isRight = (either: Either): either is Right => either.tag === 'Right';
@@ -98,7 +104,8 @@ export interface SavedObjectsRepositoryOptions {
/**
* @public
*/
-export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
+export interface SavedObjectsIncrementCounterOptions
+ extends SavedObjectsBaseOptions {
/**
* (default=false) If true, sets all the counter fields to 0 if they don't
* already exist. Existing fields will be left as-is and won't be incremented.
@@ -111,6 +118,10 @@ export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOpt
* operation. See {@link MutatingOperationRefreshSetting}
*/
refresh?: MutatingOperationRefreshSetting;
+ /**
+ * Attributes to use when upserting the document if it doesn't exist.
+ */
+ upsertAttributes?: Attributes;
}
/**
@@ -1694,6 +1705,20 @@ export class SavedObjectsRepository {
* .incrementCounter('dashboard_counter_type', 'counter_id', [
* 'stats.apiCalls',
* ])
+ *
+ * // Increment the apiCalls field counter by 4
+ * repository
+ * .incrementCounter('dashboard_counter_type', 'counter_id', [
+ * { fieldName: 'stats.apiCalls' incrementBy: 4 },
+ * ])
+ *
+ * // Initialize the document with arbitrary fields if not present
+ * repository.incrementCounter<{ appId: string }>(
+ * 'dashboard_counter_type',
+ * 'counter_id',
+ * [ 'stats.apiCalls'],
+ * { upsertAttributes: { appId: 'myId' } }
+ * )
* ```
*
* @param type - The type of saved object whose fields should be incremented
@@ -1706,7 +1731,7 @@ export class SavedObjectsRepository {
type: string,
id: string,
counterFields: Array,
- options: SavedObjectsIncrementCounterOptions = {}
+ options: SavedObjectsIncrementCounterOptions = {}
): Promise> {
if (typeof type !== 'string') {
throw new Error('"type" argument must be a string');
@@ -1728,12 +1753,16 @@ export class SavedObjectsRepository {
throw SavedObjectsErrorHelpers.createUnsupportedTypeError(type);
}
- const { migrationVersion, refresh = DEFAULT_REFRESH_SETTING, initialize = false } = options;
+ const {
+ migrationVersion,
+ refresh = DEFAULT_REFRESH_SETTING,
+ initialize = false,
+ upsertAttributes,
+ } = options;
const normalizedCounterFields = counterFields.map((counterField) => {
const fieldName = typeof counterField === 'string' ? counterField : counterField.fieldName;
const incrementBy = typeof counterField === 'string' ? 1 : counterField.incrementBy || 1;
-
return {
fieldName,
incrementBy: initialize ? 0 : incrementBy,
@@ -1757,11 +1786,14 @@ export class SavedObjectsRepository {
type,
...(savedObjectNamespace && { namespace: savedObjectNamespace }),
...(savedObjectNamespaces && { namespaces: savedObjectNamespaces }),
- attributes: normalizedCounterFields.reduce((acc, counterField) => {
- const { fieldName, incrementBy } = counterField;
- acc[fieldName] = incrementBy;
- return acc;
- }, {} as Record),
+ attributes: {
+ ...(upsertAttributes ?? {}),
+ ...normalizedCounterFields.reduce((acc, counterField) => {
+ const { fieldName, incrementBy } = counterField;
+ acc[fieldName] = incrementBy;
+ return acc;
+ }, {} as Record),
+ },
migrationVersion,
updated_at: time,
});
diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md
index 73f8a4407516..cf1647ef5cec 100644
--- a/src/core/server/server.api.md
+++ b/src/core/server/server.api.md
@@ -2735,11 +2735,12 @@ export interface SavedObjectsIncrementCounterField {
}
// @public (undocumented)
-export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
+export interface SavedObjectsIncrementCounterOptions extends SavedObjectsBaseOptions {
initialize?: boolean;
// (undocumented)
migrationVersion?: SavedObjectsMigrationVersion;
refresh?: MutatingOperationRefreshSetting;
+ upsertAttributes?: Attributes;
}
// @public
@@ -2839,7 +2840,7 @@ export class SavedObjectsRepository {
// (undocumented)
find(options: SavedObjectsFindOptions): Promise>;
get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>;
- incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>;
+ incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>;
openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise;
removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise;
resolve(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>;
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/constants.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/constants.ts
index 1910ba054bf8..f072f044925b 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/constants.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/constants.ts
@@ -12,15 +12,9 @@
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
- * Roll daily indices every 30 minutes.
- * This means that, assuming a user can visit all the 44 apps we can possibly report
- * in the 3 minutes interval the browser reports to the server, up to 22 users can have the same
- * behaviour and we wouldn't need to paginate in the transactional documents (less than 10k docs).
- *
- * Based on a more normal expected use case, the users could visit up to 5 apps in those 3 minutes,
- * allowing up to 200 users before reaching the limit.
+ * Roll daily indices every 24h
*/
-export const ROLL_DAILY_INDICES_INTERVAL = 30 * 60 * 1000;
+export const ROLL_DAILY_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/index.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/index.ts
index 676f5fddc16e..2d2d07d9d189 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/index.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/index.ts
@@ -7,3 +7,4 @@
*/
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';
+export { rollDailyData as migrateTransactionalDocs } from './rollups';
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.test.ts
similarity index 51%
rename from src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.test.ts
rename to src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.test.ts
index 7d86bc41e0b9..5acd1fb9c9c3 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.test.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.test.ts
@@ -6,21 +6,16 @@
* Side Public License, v 1.
*/
-import { rollDailyData, rollTotals } from './rollups';
-import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
-import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
-import { SavedObjectsErrorHelpers } from '../../../../../core/server';
-import {
- SAVED_OBJECTS_DAILY_TYPE,
- SAVED_OBJECTS_TOTAL_TYPE,
- SAVED_OBJECTS_TRANSACTIONAL_TYPE,
-} from './saved_objects_types';
+import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
+import { SavedObjectsErrorHelpers } from '../../../../../../core/server';
+import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TRANSACTIONAL_TYPE } from '../saved_objects_types';
+import { rollDailyData } from './daily';
describe('rollDailyData', () => {
const logger = loggingSystemMock.createLogger();
- test('returns undefined if no savedObjectsClient initialised yet', async () => {
- await expect(rollDailyData(logger, undefined)).resolves.toBe(undefined);
+ test('returns false if no savedObjectsClient initialised yet', async () => {
+ await expect(rollDailyData(logger, undefined)).resolves.toBe(false);
});
test('handle empty results', async () => {
@@ -33,7 +28,7 @@ describe('rollDailyData', () => {
throw new Error(`Unexpected type [${type}]`);
}
});
- await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
+ await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
expect(savedObjectClient.get).not.toBeCalled();
expect(savedObjectClient.bulkCreate).not.toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
@@ -101,7 +96,7 @@ describe('rollDailyData', () => {
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
});
- await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
+ await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
expect(savedObjectClient.get).toHaveBeenCalledTimes(2);
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
1,
@@ -196,7 +191,7 @@ describe('rollDailyData', () => {
throw new Error('Something went terribly wrong');
});
- await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
+ await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(false);
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
expect(savedObjectClient.get).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
@@ -206,185 +201,3 @@ describe('rollDailyData', () => {
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
});
-
-describe('rollTotals', () => {
- const logger = loggingSystemMock.createLogger();
-
- test('returns undefined if no savedObjectsClient initialised yet', async () => {
- await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
- });
-
- test('handle empty results', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
- savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
- switch (type) {
- case SAVED_OBJECTS_DAILY_TYPE:
- case SAVED_OBJECTS_TOTAL_TYPE:
- return { saved_objects: [], total: 0, page, per_page: perPage };
- default:
- throw new Error(`Unexpected type [${type}]`);
- }
- });
- await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
- expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
- expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
- });
-
- test('migrate some documents', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
- savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
- switch (type) {
- case SAVED_OBJECTS_DAILY_TYPE:
- return {
- saved_objects: [
- {
- id: 'appId-2:2020-01-01',
- type,
- score: 0,
- references: [],
- attributes: {
- appId: 'appId-2',
- timestamp: '2020-01-01T10:31:00.000Z',
- minutesOnScreen: 0.5,
- numberOfClicks: 1,
- },
- },
- {
- id: 'appId-1:2020-01-01',
- type,
- score: 0,
- references: [],
- attributes: {
- appId: 'appId-1',
- timestamp: '2020-01-01T11:31:00.000Z',
- minutesOnScreen: 2.5,
- numberOfClicks: 2,
- },
- },
- {
- id: 'appId-1:2020-01-01:viewId-1',
- type,
- score: 0,
- references: [],
- attributes: {
- appId: 'appId-1',
- viewId: 'viewId-1',
- timestamp: '2020-01-01T11:31:00.000Z',
- minutesOnScreen: 1,
- numberOfClicks: 1,
- },
- },
- ],
- total: 3,
- page,
- per_page: perPage,
- };
- case SAVED_OBJECTS_TOTAL_TYPE:
- return {
- saved_objects: [
- {
- id: 'appId-1',
- type,
- score: 0,
- references: [],
- attributes: {
- appId: 'appId-1',
- minutesOnScreen: 0.5,
- numberOfClicks: 1,
- },
- },
- {
- id: 'appId-1___viewId-1',
- type,
- score: 0,
- references: [],
- attributes: {
- appId: 'appId-1',
- viewId: 'viewId-1',
- minutesOnScreen: 4,
- numberOfClicks: 2,
- },
- },
- {
- id: 'appId-2___viewId-1',
- type,
- score: 0,
- references: [],
- attributes: {
- appId: 'appId-2',
- viewId: 'viewId-1',
- minutesOnScreen: 1,
- numberOfClicks: 1,
- },
- },
- ],
- total: 3,
- page,
- per_page: perPage,
- };
- default:
- throw new Error(`Unexpected type [${type}]`);
- }
- });
- await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
- expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
- expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
- [
- {
- type: SAVED_OBJECTS_TOTAL_TYPE,
- id: 'appId-1',
- attributes: {
- appId: 'appId-1',
- viewId: MAIN_APP_DEFAULT_VIEW_ID,
- minutesOnScreen: 3.0,
- numberOfClicks: 3,
- },
- },
- {
- type: SAVED_OBJECTS_TOTAL_TYPE,
- id: 'appId-1___viewId-1',
- attributes: {
- appId: 'appId-1',
- viewId: 'viewId-1',
- minutesOnScreen: 5.0,
- numberOfClicks: 3,
- },
- },
- {
- type: SAVED_OBJECTS_TOTAL_TYPE,
- id: 'appId-2___viewId-1',
- attributes: {
- appId: 'appId-2',
- viewId: 'viewId-1',
- minutesOnScreen: 1.0,
- numberOfClicks: 1,
- },
- },
- {
- type: SAVED_OBJECTS_TOTAL_TYPE,
- id: 'appId-2',
- attributes: {
- appId: 'appId-2',
- viewId: MAIN_APP_DEFAULT_VIEW_ID,
- minutesOnScreen: 0.5,
- numberOfClicks: 1,
- },
- },
- ],
- { overwrite: true }
- );
- expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
- expect(savedObjectClient.delete).toHaveBeenCalledWith(
- SAVED_OBJECTS_DAILY_TYPE,
- 'appId-2:2020-01-01'
- );
- expect(savedObjectClient.delete).toHaveBeenCalledWith(
- SAVED_OBJECTS_DAILY_TYPE,
- 'appId-1:2020-01-01'
- );
- expect(savedObjectClient.delete).toHaveBeenCalledWith(
- SAVED_OBJECTS_DAILY_TYPE,
- 'appId-1:2020-01-01:viewId-1'
- );
- });
-});
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.ts
similarity index 55%
rename from src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.ts
rename to src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.ts
index df7e7662b49c..a7873c7d5dfe 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.ts
@@ -6,18 +6,20 @@
* Side Public License, v 1.
*/
-import { ISavedObjectsRepository, SavedObject, Logger } from 'kibana/server';
import moment from 'moment';
+import type { Logger } from '@kbn/logging';
+import {
+ ISavedObjectsRepository,
+ SavedObject,
+ SavedObjectsErrorHelpers,
+} from '../../../../../../core/server';
+import { getDailyId } from '../../../../../usage_collection/common/application_usage';
import {
ApplicationUsageDaily,
- ApplicationUsageTotal,
ApplicationUsageTransactional,
SAVED_OBJECTS_DAILY_TYPE,
- SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
-} from './saved_objects_types';
-import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
-import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
+} from '../saved_objects_types';
/**
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
@@ -27,18 +29,17 @@ type ApplicationUsageDailyWithVersion = Pick<
'version' | 'attributes'
>;
-export function serializeKey(appId: string, viewId: string) {
- return `${appId}___${viewId}`;
-}
-
/**
* Aggregates all the transactional events into daily aggregates
* @param logger
* @param savedObjectsClient
*/
-export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
+export async function rollDailyData(
+ logger: Logger,
+ savedObjectsClient?: ISavedObjectsRepository
+): Promise {
if (!savedObjectsClient) {
- return;
+ return false;
}
try {
@@ -58,10 +59,7 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
} = doc;
const dayId = moment(timestamp).format('YYYY-MM-DD');
- const dailyId =
- !viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
- ? `${appId}:${dayId}`
- : `${appId}:${dayId}:${viewId}`;
+ const dailyId = getDailyId({ dayId, appId, viewId });
const existingDoc =
toCreate.get(dailyId) ||
@@ -103,9 +101,11 @@ export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedO
}
}
} while (toCreate.size > 0);
+ return true;
} catch (err) {
logger.debug(`Failed to rollup transactional to daily entries`);
logger.debug(err);
+ return false;
}
}
@@ -125,7 +125,11 @@ async function getDailyDoc(
dayId: string
): Promise {
try {
- return await savedObjectsClient.get(SAVED_OBJECTS_DAILY_TYPE, id);
+ const { attributes, version } = await savedObjectsClient.get(
+ SAVED_OBJECTS_DAILY_TYPE,
+ id
+ );
+ return { attributes, version };
} catch (err) {
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
return {
@@ -142,91 +146,3 @@ async function getDailyDoc(
throw err;
}
}
-
-/**
- * Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
- * @param logger
- * @param savedObjectsClient
- */
-export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
- if (!savedObjectsClient) {
- return;
- }
-
- try {
- const [
- { saved_objects: rawApplicationUsageTotals },
- { saved_objects: rawApplicationUsageDaily },
- ] = await Promise.all([
- savedObjectsClient.find({
- perPage: 10000,
- type: SAVED_OBJECTS_TOTAL_TYPE,
- }),
- savedObjectsClient.find({
- perPage: 10000,
- type: SAVED_OBJECTS_DAILY_TYPE,
- filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
- }),
- ]);
-
- const existingTotals = rawApplicationUsageTotals.reduce(
- (
- acc,
- {
- attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
- }
- ) => {
- const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
-
- return {
- ...acc,
- // No need to sum because there should be 1 document per appId only
- [key]: { appId, viewId, numberOfClicks, minutesOnScreen },
- };
- },
- {} as Record<
- string,
- { appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
- >
- );
-
- const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
- const {
- appId,
- viewId = MAIN_APP_DEFAULT_VIEW_ID,
- numberOfClicks,
- minutesOnScreen,
- } = attributes;
- const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
- const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
-
- return {
- ...acc,
- [key]: {
- appId,
- viewId,
- numberOfClicks: numberOfClicks + existing.numberOfClicks,
- minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
- },
- };
- }, existingTotals);
-
- await Promise.all([
- Object.entries(totals).length &&
- savedObjectsClient.bulkCreate(
- Object.entries(totals).map(([id, entry]) => ({
- type: SAVED_OBJECTS_TOTAL_TYPE,
- id,
- attributes: entry,
- })),
- { overwrite: true }
- ),
- ...rawApplicationUsageDaily.map(
- ({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
- ),
- ]);
- } catch (err) {
- logger.debug(`Failed to rollup daily entries to totals`);
- logger.debug(err);
- }
-}
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/index.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/index.ts
new file mode 100644
index 000000000000..8f3d83613aa9
--- /dev/null
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/index.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { rollDailyData } from './daily';
+export { rollTotals } from './total';
+export { serializeKey } from './utils';
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.test.ts
new file mode 100644
index 000000000000..9fea955ab5d8
--- /dev/null
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.test.ts
@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks';
+import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
+import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from '../saved_objects_types';
+import { rollTotals } from './total';
+
+describe('rollTotals', () => {
+ const logger = loggingSystemMock.createLogger();
+
+ test('returns undefined if no savedObjectsClient initialised yet', async () => {
+ await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
+ });
+
+ test('handle empty results', async () => {
+ const savedObjectClient = savedObjectsRepositoryMock.create();
+ savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
+ switch (type) {
+ case SAVED_OBJECTS_DAILY_TYPE:
+ case SAVED_OBJECTS_TOTAL_TYPE:
+ return { saved_objects: [], total: 0, page, per_page: perPage };
+ default:
+ throw new Error(`Unexpected type [${type}]`);
+ }
+ });
+ await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
+ expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
+ expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
+ });
+
+ test('migrate some documents', async () => {
+ const savedObjectClient = savedObjectsRepositoryMock.create();
+ savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
+ switch (type) {
+ case SAVED_OBJECTS_DAILY_TYPE:
+ return {
+ saved_objects: [
+ {
+ id: 'appId-2:2020-01-01',
+ type,
+ score: 0,
+ references: [],
+ attributes: {
+ appId: 'appId-2',
+ timestamp: '2020-01-01T10:31:00.000Z',
+ minutesOnScreen: 0.5,
+ numberOfClicks: 1,
+ },
+ },
+ {
+ id: 'appId-1:2020-01-01',
+ type,
+ score: 0,
+ references: [],
+ attributes: {
+ appId: 'appId-1',
+ timestamp: '2020-01-01T11:31:00.000Z',
+ minutesOnScreen: 2.5,
+ numberOfClicks: 2,
+ },
+ },
+ {
+ id: 'appId-1:2020-01-01:viewId-1',
+ type,
+ score: 0,
+ references: [],
+ attributes: {
+ appId: 'appId-1',
+ viewId: 'viewId-1',
+ timestamp: '2020-01-01T11:31:00.000Z',
+ minutesOnScreen: 1,
+ numberOfClicks: 1,
+ },
+ },
+ ],
+ total: 3,
+ page,
+ per_page: perPage,
+ };
+ case SAVED_OBJECTS_TOTAL_TYPE:
+ return {
+ saved_objects: [
+ {
+ id: 'appId-1',
+ type,
+ score: 0,
+ references: [],
+ attributes: {
+ appId: 'appId-1',
+ minutesOnScreen: 0.5,
+ numberOfClicks: 1,
+ },
+ },
+ {
+ id: 'appId-1___viewId-1',
+ type,
+ score: 0,
+ references: [],
+ attributes: {
+ appId: 'appId-1',
+ viewId: 'viewId-1',
+ minutesOnScreen: 4,
+ numberOfClicks: 2,
+ },
+ },
+ {
+ id: 'appId-2___viewId-1',
+ type,
+ score: 0,
+ references: [],
+ attributes: {
+ appId: 'appId-2',
+ viewId: 'viewId-1',
+ minutesOnScreen: 1,
+ numberOfClicks: 1,
+ },
+ },
+ ],
+ total: 3,
+ page,
+ per_page: perPage,
+ };
+ default:
+ throw new Error(`Unexpected type [${type}]`);
+ }
+ });
+ await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
+ expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
+ expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
+ [
+ {
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ id: 'appId-1',
+ attributes: {
+ appId: 'appId-1',
+ viewId: MAIN_APP_DEFAULT_VIEW_ID,
+ minutesOnScreen: 3.0,
+ numberOfClicks: 3,
+ },
+ },
+ {
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ id: 'appId-1___viewId-1',
+ attributes: {
+ appId: 'appId-1',
+ viewId: 'viewId-1',
+ minutesOnScreen: 5.0,
+ numberOfClicks: 3,
+ },
+ },
+ {
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ id: 'appId-2___viewId-1',
+ attributes: {
+ appId: 'appId-2',
+ viewId: 'viewId-1',
+ minutesOnScreen: 1.0,
+ numberOfClicks: 1,
+ },
+ },
+ {
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ id: 'appId-2',
+ attributes: {
+ appId: 'appId-2',
+ viewId: MAIN_APP_DEFAULT_VIEW_ID,
+ minutesOnScreen: 0.5,
+ numberOfClicks: 1,
+ },
+ },
+ ],
+ { overwrite: true }
+ );
+ expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
+ expect(savedObjectClient.delete).toHaveBeenCalledWith(
+ SAVED_OBJECTS_DAILY_TYPE,
+ 'appId-2:2020-01-01'
+ );
+ expect(savedObjectClient.delete).toHaveBeenCalledWith(
+ SAVED_OBJECTS_DAILY_TYPE,
+ 'appId-1:2020-01-01'
+ );
+ expect(savedObjectClient.delete).toHaveBeenCalledWith(
+ SAVED_OBJECTS_DAILY_TYPE,
+ 'appId-1:2020-01-01:viewId-1'
+ );
+ });
+});
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.ts
new file mode 100644
index 000000000000..e27c7b897d99
--- /dev/null
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.ts
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { Logger } from '@kbn/logging';
+import type { ISavedObjectsRepository } from 'kibana/server';
+import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../../usage_collection/common/constants';
+import {
+ ApplicationUsageDaily,
+ ApplicationUsageTotal,
+ SAVED_OBJECTS_DAILY_TYPE,
+ SAVED_OBJECTS_TOTAL_TYPE,
+} from '../saved_objects_types';
+import { serializeKey } from './utils';
+
+/**
+ * Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
+ * @param logger
+ * @param savedObjectsClient
+ */
+export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
+ if (!savedObjectsClient) {
+ return;
+ }
+
+ try {
+ const [
+ { saved_objects: rawApplicationUsageTotals },
+ { saved_objects: rawApplicationUsageDaily },
+ ] = await Promise.all([
+ savedObjectsClient.find({
+ perPage: 10000,
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ }),
+ savedObjectsClient.find({
+ perPage: 10000,
+ type: SAVED_OBJECTS_DAILY_TYPE,
+ filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
+ }),
+ ]);
+
+ const existingTotals = rawApplicationUsageTotals.reduce(
+ (
+ acc,
+ {
+ attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
+ }
+ ) => {
+ const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
+
+ return {
+ ...acc,
+ // No need to sum because there should be 1 document per appId only
+ [key]: { appId, viewId, numberOfClicks, minutesOnScreen },
+ };
+ },
+ {} as Record<
+ string,
+ { appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
+ >
+ );
+
+ const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
+ const {
+ appId,
+ viewId = MAIN_APP_DEFAULT_VIEW_ID,
+ numberOfClicks,
+ minutesOnScreen,
+ } = attributes;
+ const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
+ const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
+
+ return {
+ ...acc,
+ [key]: {
+ appId,
+ viewId,
+ numberOfClicks: numberOfClicks + existing.numberOfClicks,
+ minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
+ },
+ };
+ }, existingTotals);
+
+ await Promise.all([
+ Object.entries(totals).length &&
+ savedObjectsClient.bulkCreate(
+ Object.entries(totals).map(([id, entry]) => ({
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ id,
+ attributes: entry,
+ })),
+ { overwrite: true }
+ ),
+ ...rawApplicationUsageDaily.map(
+ ({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
+ ),
+ ]);
+ } catch (err) {
+ logger.debug(`Failed to rollup daily entries to totals`);
+ logger.debug(err);
+ }
+}
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/utils.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/utils.ts
new file mode 100644
index 000000000000..8be00e628788
--- /dev/null
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/utils.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export function serializeKey(appId: string, viewId: string) {
+ return `${appId}___${viewId}`;
+}
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts
index 9e71b5c3b032..f2b996f3af97 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts
@@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
-import { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
+import type { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
/**
* Used for accumulating the totals of all the stats older than 90d
@@ -17,6 +17,7 @@ export interface ApplicationUsageTotal extends SavedObjectAttributes {
minutesOnScreen: number;
numberOfClicks: number;
}
+
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
/**
@@ -25,6 +26,8 @@ export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
export interface ApplicationUsageTransactional extends ApplicationUsageTotal {
timestamp: string;
}
+
+/** @deprecated transactional type is no longer used, and only preserved for backward compatibility */
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
/**
@@ -62,6 +65,7 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
});
// Type for storing ApplicationUsageTransactional (declaring empty mappings because we don't use the internal fields for query/aggregations)
+ // Remark: this type is deprecated and only here for BWC reasons.
registerType({
name: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
hidden: false,
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
index 062d751ef454..693e9132fe53 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
@@ -7,7 +7,7 @@
*/
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
-import { ApplicationUsageTelemetryReport } from './telemetry_application_usage_collector';
+import { ApplicationUsageTelemetryReport } from './types';
const commonSchema: MakeSchemaFrom = {
appId: { type: 'keyword', _meta: { description: 'The application being tracked' } },
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
index 3e8434d44603..f1b21af5506e 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts
@@ -11,74 +11,99 @@ import {
Collector,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
-
+import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
-import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import {
registerApplicationUsageCollector,
transformByApplicationViews,
- ApplicationUsageViews,
} from './telemetry_application_usage_collector';
-import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
-import {
- SAVED_OBJECTS_DAILY_TYPE,
- SAVED_OBJECTS_TOTAL_TYPE,
- SAVED_OBJECTS_TRANSACTIONAL_TYPE,
-} from './saved_objects_types';
+import { ApplicationUsageViews } from './types';
+
+import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TOTAL_TYPE } from './saved_objects_types';
+
+// use fake timers to avoid triggering rollups during tests
+jest.useFakeTimers();
describe('telemetry_application_usage', () => {
- jest.useFakeTimers();
-
- const logger = loggingSystemMock.createLogger();
-
+ let logger: ReturnType;
let collector: Collector;
+ let usageCollectionMock: ReturnType;
+ let savedObjectClient: ReturnType;
+ let getSavedObjectClient: jest.MockedFunction<() => undefined | typeof savedObjectClient>;
- const usageCollectionMock = createUsageCollectionSetupMock();
- usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
- collector = new Collector(logger, config);
- return createUsageCollectionSetupMock().makeUsageCollector(config);
- });
-
- const getUsageCollector = jest.fn();
const registerType = jest.fn();
const mockedFetchContext = createCollectorFetchContextMock();
- beforeAll(() =>
- registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector)
- );
- afterAll(() => jest.clearAllTimers());
+ beforeEach(() => {
+ logger = loggingSystemMock.createLogger();
+ usageCollectionMock = createUsageCollectionSetupMock();
+ savedObjectClient = savedObjectsRepositoryMock.create();
+ getSavedObjectClient = jest.fn().mockReturnValue(savedObjectClient);
+ usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
+ collector = new Collector(logger, config);
+ return createUsageCollectionSetupMock().makeUsageCollector(config);
+ });
+ registerApplicationUsageCollector(
+ logger,
+ usageCollectionMock,
+ registerType,
+ getSavedObjectClient
+ );
+ });
+
+ afterEach(() => {
+ jest.clearAllTimers();
+ });
test('registered collector is set', () => {
expect(collector).not.toBeUndefined();
});
test('if no savedObjectClient initialised, return undefined', async () => {
+ getSavedObjectClient.mockReturnValue(undefined);
+
expect(collector.isReady()).toBe(false);
expect(await collector.fetch(mockedFetchContext)).toBeUndefined();
- jest.runTimersToTime(ROLL_INDICES_START);
+ });
+
+ test('calls `savedObjectsClient.find` with the correct parameters', async () => {
+ savedObjectClient.find.mockResolvedValue({
+ saved_objects: [],
+ total: 0,
+ per_page: 20,
+ page: 0,
+ });
+
+ await collector.fetch(mockedFetchContext);
+
+ expect(savedObjectClient.find).toHaveBeenCalledTimes(2);
+
+ expect(savedObjectClient.find).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: SAVED_OBJECTS_TOTAL_TYPE,
+ })
+ );
+ expect(savedObjectClient.find).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: SAVED_OBJECTS_DAILY_TYPE,
+ })
+ );
});
test('when savedObjectClient is initialised, return something', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
- savedObjectClient.find.mockImplementation(
- async () =>
- ({
- saved_objects: [],
- total: 0,
- } as any)
- );
- getUsageCollector.mockImplementation(() => savedObjectClient);
-
- jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
+ savedObjectClient.find.mockResolvedValue({
+ saved_objects: [],
+ total: 0,
+ per_page: 20,
+ page: 0,
+ });
expect(collector.isReady()).toBe(true);
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({});
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
});
- test('it only gets 10k even when there are more documents (ES limitation)', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
- const total = 10000;
+ test('it aggregates total and daily data', async () => {
savedObjectClient.find.mockImplementation(async (opts) => {
switch (opts.type) {
case SAVED_OBJECTS_TOTAL_TYPE:
@@ -95,18 +120,6 @@ describe('telemetry_application_usage', () => {
],
total: 1,
} as any;
- case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
- const doc = {
- id: 'test-id',
- attributes: {
- appId: 'appId',
- timestamp: new Date().toISOString(),
- minutesOnScreen: 0.5,
- numberOfClicks: 1,
- },
- };
- const savedObjects = new Array(total).fill(doc);
- return { saved_objects: savedObjects, total: total + 1 };
case SAVED_OBJECTS_DAILY_TYPE:
return {
saved_objects: [
@@ -125,122 +138,21 @@ describe('telemetry_application_usage', () => {
}
});
- getUsageCollector.mockImplementation(() => savedObjectClient);
-
- jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
-
expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
appId: {
appId: 'appId',
viewId: 'main',
- clicks_total: total + 1 + 10,
- clicks_7_days: total + 1,
- clicks_30_days: total + 1,
- clicks_90_days: total + 1,
- minutes_on_screen_total: (total + 1) * 0.5 + 10,
- minutes_on_screen_7_days: (total + 1) * 0.5,
- minutes_on_screen_30_days: (total + 1) * 0.5,
- minutes_on_screen_90_days: (total + 1) * 0.5,
+ clicks_total: 1 + 10,
+ clicks_7_days: 1,
+ clicks_30_days: 1,
+ clicks_90_days: 1,
+ minutes_on_screen_total: 0.5 + 10,
+ minutes_on_screen_7_days: 0.5,
+ minutes_on_screen_30_days: 0.5,
+ minutes_on_screen_90_days: 0.5,
views: [],
},
});
- expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
- [
- {
- id: 'appId',
- type: SAVED_OBJECTS_TOTAL_TYPE,
- attributes: {
- appId: 'appId',
- viewId: 'main',
- minutesOnScreen: 10.5,
- numberOfClicks: 11,
- },
- },
- ],
- { overwrite: true }
- );
- expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
- expect(savedObjectClient.delete).toHaveBeenCalledWith(
- SAVED_OBJECTS_DAILY_TYPE,
- 'appId:YYYY-MM-DD'
- );
- });
-
- test('old transactional data not migrated yet', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
- savedObjectClient.find.mockImplementation(async (opts) => {
- switch (opts.type) {
- case SAVED_OBJECTS_TOTAL_TYPE:
- case SAVED_OBJECTS_DAILY_TYPE:
- return { saved_objects: [], total: 0 } as any;
- case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
- return {
- saved_objects: [
- {
- id: 'test-id',
- attributes: {
- appId: 'appId',
- timestamp: new Date(0).toISOString(),
- minutesOnScreen: 0.5,
- numberOfClicks: 1,
- },
- },
- {
- id: 'test-id-2',
- attributes: {
- appId: 'appId',
- viewId: 'main',
- timestamp: new Date(0).toISOString(),
- minutesOnScreen: 2,
- numberOfClicks: 2,
- },
- },
- {
- id: 'test-id-3',
- attributes: {
- appId: 'appId',
- viewId: 'viewId-1',
- timestamp: new Date(0).toISOString(),
- minutesOnScreen: 1,
- numberOfClicks: 1,
- },
- },
- ],
- total: 1,
- };
- }
- });
-
- getUsageCollector.mockImplementation(() => savedObjectClient);
-
- expect(await collector.fetch(mockedFetchContext)).toStrictEqual({
- appId: {
- appId: 'appId',
- viewId: 'main',
- clicks_total: 3,
- clicks_7_days: 0,
- clicks_30_days: 0,
- clicks_90_days: 0,
- minutes_on_screen_total: 2.5,
- minutes_on_screen_7_days: 0,
- minutes_on_screen_30_days: 0,
- minutes_on_screen_90_days: 0,
- views: [
- {
- appId: 'appId',
- viewId: 'viewId-1',
- clicks_total: 1,
- clicks_7_days: 0,
- clicks_30_days: 0,
- clicks_90_days: 0,
- minutes_on_screen_total: 1,
- minutes_on_screen_7_days: 0,
- minutes_on_screen_30_days: 0,
- minutes_on_screen_90_days: 0,
- },
- ],
- },
- });
});
});
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
index ee1b42e61a6c..a01f1bca4f0e 100644
--- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
@@ -11,57 +11,21 @@ import { timer } from 'rxjs';
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from 'kibana/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants';
-import { serializeKey } from './rollups';
-
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
- ApplicationUsageTransactional,
registerMappings,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
- SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { applicationUsageSchema } from './schema';
-import { rollDailyData, rollTotals } from './rollups';
+import { rollTotals, rollDailyData, serializeKey } from './rollups';
import {
ROLL_TOTAL_INDICES_INTERVAL,
ROLL_DAILY_INDICES_INTERVAL,
ROLL_INDICES_START,
} from './constants';
-
-export interface ApplicationViewUsage {
- appId: string;
- viewId: string;
- clicks_total: number;
- clicks_7_days: number;
- clicks_30_days: number;
- clicks_90_days: number;
- minutes_on_screen_total: number;
- minutes_on_screen_7_days: number;
- minutes_on_screen_30_days: number;
- minutes_on_screen_90_days: number;
-}
-
-export interface ApplicationUsageViews {
- [serializedKey: string]: ApplicationViewUsage;
-}
-
-export interface ApplicationUsageTelemetryReport {
- [appId: string]: {
- appId: string;
- viewId: string;
- clicks_total: number;
- clicks_7_days: number;
- clicks_30_days: number;
- clicks_90_days: number;
- minutes_on_screen_total: number;
- minutes_on_screen_7_days: number;
- minutes_on_screen_30_days: number;
- minutes_on_screen_90_days: number;
- views?: ApplicationViewUsage[];
- };
-}
+import { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
export const transformByApplicationViews = (
report: ApplicationUsageViews
@@ -92,6 +56,21 @@ export function registerApplicationUsageCollector(
) {
registerMappings(registerType);
+ timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
+ rollTotals(logger, getSavedObjectsClient())
+ );
+
+ const dailyRollingSub = timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(
+ async () => {
+ const success = await rollDailyData(logger, getSavedObjectsClient());
+ // we only need to roll the transactional documents once to assure BWC
+ // once we rolling succeeds, we can stop.
+ if (success) {
+ dailyRollingSub.unsubscribe();
+ }
+ }
+ );
+
const collector = usageCollection.makeUsageCollector(
{
type: 'application_usage',
@@ -105,7 +84,6 @@ export function registerApplicationUsageCollector(
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
- { saved_objects: rawApplicationUsageTransactional },
] = await Promise.all([
savedObjectsClient.find({
type: SAVED_OBJECTS_TOTAL_TYPE,
@@ -115,10 +93,6 @@ export function registerApplicationUsageCollector(
type: SAVED_OBJECTS_DAILY_TYPE,
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
}),
- savedObjectsClient.find({
- type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
- perPage: 10000, // If we have more than those, we won't report the rest (they'll be rolled up to the daily soon enough to become a problem)
- }),
]);
const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
@@ -156,10 +130,7 @@ export function registerApplicationUsageCollector(
const nowMinus30 = moment().subtract(30, 'days');
const nowMinus90 = moment().subtract(90, 'days');
- const applicationUsage = [
- ...rawApplicationUsageDaily,
- ...rawApplicationUsageTransactional,
- ].reduce(
+ const applicationUsage = rawApplicationUsageDaily.reduce(
(
acc,
{
@@ -224,11 +195,4 @@ export function registerApplicationUsageCollector(
);
usageCollection.registerCollector(collector);
-
- timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(() =>
- rollDailyData(logger, getSavedObjectsClient())
- );
- timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
- rollTotals(logger, getSavedObjectsClient())
- );
}
diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/types.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/types.ts
new file mode 100644
index 000000000000..bef835e922d8
--- /dev/null
+++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/types.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export interface ApplicationViewUsage {
+ appId: string;
+ viewId: string;
+ clicks_total: number;
+ clicks_7_days: number;
+ clicks_30_days: number;
+ clicks_90_days: number;
+ minutes_on_screen_total: number;
+ minutes_on_screen_7_days: number;
+ minutes_on_screen_30_days: number;
+ minutes_on_screen_90_days: number;
+}
+
+export interface ApplicationUsageViews {
+ [serializedKey: string]: ApplicationViewUsage;
+}
+
+export interface ApplicationUsageTelemetryReport {
+ [appId: string]: {
+ appId: string;
+ viewId: string;
+ clicks_total: number;
+ clicks_7_days: number;
+ clicks_30_days: number;
+ clicks_90_days: number;
+ minutes_on_screen_total: number;
+ minutes_on_screen_7_days: number;
+ minutes_on_screen_30_days: number;
+ minutes_on_screen_90_days: number;
+ views?: ApplicationViewUsage[];
+ };
+}
diff --git a/src/plugins/usage_collection/common/application_usage.ts b/src/plugins/usage_collection/common/application_usage.ts
new file mode 100644
index 000000000000..c9dd489000d3
--- /dev/null
+++ b/src/plugins/usage_collection/common/application_usage.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { MAIN_APP_DEFAULT_VIEW_ID } from './constants';
+
+export const getDailyId = ({
+ appId,
+ dayId,
+ viewId,
+}: {
+ viewId: string;
+ appId: string;
+ dayId: string;
+}) => {
+ return !viewId || viewId === MAIN_APP_DEFAULT_VIEW_ID
+ ? `${appId}:${dayId}`
+ : `${appId}:${dayId}:${viewId}`;
+};
diff --git a/src/plugins/usage_collection/server/report/schema.ts b/src/plugins/usage_collection/server/report/schema.ts
index 93203a33cd1e..350ec8d90e76 100644
--- a/src/plugins/usage_collection/server/report/schema.ts
+++ b/src/plugins/usage_collection/server/report/schema.ts
@@ -9,6 +9,13 @@
import { schema, TypeOf } from '@kbn/config-schema';
import { METRIC_TYPE } from '@kbn/analytics';
+const applicationUsageReportSchema = schema.object({
+ minutesOnScreen: schema.number(),
+ numberOfClicks: schema.number(),
+ appId: schema.string(),
+ viewId: schema.string(),
+});
+
export const reportSchema = schema.object({
reportVersion: schema.maybe(schema.oneOf([schema.literal(3)])),
userAgent: schema.maybe(
@@ -38,17 +45,8 @@ export const reportSchema = schema.object({
})
)
),
- application_usage: schema.maybe(
- schema.recordOf(
- schema.string(),
- schema.object({
- minutesOnScreen: schema.number(),
- numberOfClicks: schema.number(),
- appId: schema.string(),
- viewId: schema.string(),
- })
- )
- ),
+ application_usage: schema.maybe(schema.recordOf(schema.string(), applicationUsageReportSchema)),
});
export type ReportSchemaType = TypeOf;
+export type ApplicationUsageReport = TypeOf;
diff --git a/src/plugins/usage_collection/server/report/store_application_usage.test.ts b/src/plugins/usage_collection/server/report/store_application_usage.test.ts
new file mode 100644
index 000000000000..c4c9e5746e6c
--- /dev/null
+++ b/src/plugins/usage_collection/server/report/store_application_usage.test.ts
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import moment from 'moment';
+import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
+import { getDailyId } from '../../common/application_usage';
+import { storeApplicationUsage } from './store_application_usage';
+import { ApplicationUsageReport } from './schema';
+
+const createReport = (parts: Partial): ApplicationUsageReport => ({
+ appId: 'appId',
+ viewId: 'viewId',
+ numberOfClicks: 0,
+ minutesOnScreen: 0,
+ ...parts,
+});
+
+describe('storeApplicationUsage', () => {
+ let repository: ReturnType;
+ let timestamp: Date;
+
+ beforeEach(() => {
+ repository = savedObjectsRepositoryMock.create();
+ timestamp = new Date();
+ });
+
+ it('does not call `repository.incrementUsageCounters` when the report list is empty', async () => {
+ await storeApplicationUsage(repository, [], timestamp);
+ expect(repository.incrementCounter).not.toHaveBeenCalled();
+ });
+
+ it('calls `repository.incrementUsageCounters` with the correct parameters', async () => {
+ const report = createReport({
+ appId: 'app1',
+ viewId: 'view1',
+ numberOfClicks: 2,
+ minutesOnScreen: 5,
+ });
+
+ await storeApplicationUsage(repository, [report], timestamp);
+
+ expect(repository.incrementCounter).toHaveBeenCalledTimes(1);
+
+ expect(repository.incrementCounter).toHaveBeenCalledWith(
+ ...expectedIncrementParams(report, timestamp)
+ );
+ });
+
+ it('aggregates reports with the same appId/viewId tuple', async () => {
+ const report1 = createReport({
+ appId: 'app1',
+ viewId: 'view1',
+ numberOfClicks: 2,
+ minutesOnScreen: 5,
+ });
+ const report2 = createReport({
+ appId: 'app1',
+ viewId: 'view2',
+ numberOfClicks: 1,
+ minutesOnScreen: 7,
+ });
+ const report3 = createReport({
+ appId: 'app1',
+ viewId: 'view1',
+ numberOfClicks: 3,
+ minutesOnScreen: 9,
+ });
+
+ await storeApplicationUsage(repository, [report1, report2, report3], timestamp);
+
+ expect(repository.incrementCounter).toHaveBeenCalledTimes(2);
+
+ expect(repository.incrementCounter).toHaveBeenCalledWith(
+ ...expectedIncrementParams(
+ {
+ appId: 'app1',
+ viewId: 'view1',
+ numberOfClicks: report1.numberOfClicks + report3.numberOfClicks,
+ minutesOnScreen: report1.minutesOnScreen + report3.minutesOnScreen,
+ },
+ timestamp
+ )
+ );
+ expect(repository.incrementCounter).toHaveBeenCalledWith(
+ ...expectedIncrementParams(report2, timestamp)
+ );
+ });
+});
+
+const expectedIncrementParams = (
+ { appId, viewId, minutesOnScreen, numberOfClicks }: ApplicationUsageReport,
+ timestamp: Date
+) => {
+ const dayId = moment(timestamp).format('YYYY-MM-DD');
+ return [
+ 'application_usage_daily',
+ getDailyId({ appId, viewId, dayId }),
+ [
+ { fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
+ { fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
+ ],
+ {
+ upsertAttributes: {
+ appId,
+ viewId,
+ timestamp: moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString(),
+ },
+ },
+ ];
+};
diff --git a/src/plugins/usage_collection/server/report/store_application_usage.ts b/src/plugins/usage_collection/server/report/store_application_usage.ts
new file mode 100644
index 000000000000..2058b054fda8
--- /dev/null
+++ b/src/plugins/usage_collection/server/report/store_application_usage.ts
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import moment from 'moment';
+import { Writable } from '@kbn/utility-types';
+import { ISavedObjectsRepository } from 'src/core/server';
+import { ApplicationUsageReport } from './schema';
+import { getDailyId } from '../../common/application_usage';
+
+type WritableApplicationUsageReport = Writable;
+
+export const storeApplicationUsage = async (
+ repository: ISavedObjectsRepository,
+ appUsages: ApplicationUsageReport[],
+ timestamp: Date
+) => {
+ if (!appUsages.length) {
+ return;
+ }
+
+ const dayId = getDayId(timestamp);
+ const aggregatedReports = aggregateAppUsages(appUsages);
+
+ return Promise.allSettled(
+ aggregatedReports.map(async (report) => incrementUsageCounters(repository, report, dayId))
+ );
+};
+
+const aggregateAppUsages = (appUsages: ApplicationUsageReport[]) => {
+ return [
+ ...appUsages
+ .reduce((map, appUsage) => {
+ const key = getKey(appUsage);
+ const aggregated: WritableApplicationUsageReport = map.get(key) ?? {
+ appId: appUsage.appId,
+ viewId: appUsage.viewId,
+ minutesOnScreen: 0,
+ numberOfClicks: 0,
+ };
+
+ aggregated.minutesOnScreen += appUsage.minutesOnScreen;
+ aggregated.numberOfClicks += appUsage.numberOfClicks;
+
+ map.set(key, aggregated);
+ return map;
+ }, new Map())
+ .values(),
+ ];
+};
+
+const incrementUsageCounters = (
+ repository: ISavedObjectsRepository,
+ { appId, viewId, numberOfClicks, minutesOnScreen }: WritableApplicationUsageReport,
+ dayId: string
+) => {
+ const dailyId = getDailyId({ appId, viewId, dayId });
+
+ return repository.incrementCounter(
+ 'application_usage_daily',
+ dailyId,
+ [
+ { fieldName: 'numberOfClicks', incrementBy: numberOfClicks },
+ { fieldName: 'minutesOnScreen', incrementBy: minutesOnScreen },
+ ],
+ {
+ upsertAttributes: {
+ appId,
+ viewId,
+ timestamp: getTimestamp(dayId),
+ },
+ }
+ );
+};
+
+const getKey = ({ viewId, appId }: ApplicationUsageReport) => `${appId}___${viewId}`;
+
+const getDayId = (timestamp: Date) => moment(timestamp).format('YYYY-MM-DD');
+
+const getTimestamp = (dayId: string) => {
+ // Concatenating the day in YYYY-MM-DD form to T00:00:00Z to reduce the TZ effects
+ return moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString();
+};
diff --git a/src/plugins/usage_collection/server/report/store_report.test.mocks.ts b/src/plugins/usage_collection/server/report/store_report.test.mocks.ts
new file mode 100644
index 000000000000..d151e7d7a5dd
--- /dev/null
+++ b/src/plugins/usage_collection/server/report/store_report.test.mocks.ts
@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const storeApplicationUsageMock = jest.fn();
+jest.doMock('./store_application_usage', () => ({
+ storeApplicationUsage: storeApplicationUsageMock,
+}));
diff --git a/src/plugins/usage_collection/server/report/store_report.test.ts b/src/plugins/usage_collection/server/report/store_report.test.ts
index 7174a5406724..dfcdd1f8e7e4 100644
--- a/src/plugins/usage_collection/server/report/store_report.test.ts
+++ b/src/plugins/usage_collection/server/report/store_report.test.ts
@@ -6,6 +6,8 @@
* Side Public License, v 1.
*/
+import { storeApplicationUsageMock } from './store_report.test.mocks';
+
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { storeReport } from './store_report';
import { ReportSchemaType } from './schema';
@@ -16,8 +18,17 @@ describe('store_report', () => {
const momentTimestamp = moment();
const date = momentTimestamp.format('DDMMYYYY');
+ let repository: ReturnType;
+
+ beforeEach(() => {
+ repository = savedObjectsRepositoryMock.create();
+ });
+
+ afterEach(() => {
+ storeApplicationUsageMock.mockReset();
+ });
+
test('stores report for all types of data', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
const report: ReportSchemaType = {
reportVersion: 3,
userAgent: {
@@ -53,9 +64,9 @@ describe('store_report', () => {
},
},
};
- await storeReport(savedObjectClient, report);
+ await storeReport(repository, report);
- expect(savedObjectClient.create).toHaveBeenCalledWith(
+ expect(repository.create).toHaveBeenCalledWith(
'ui-metric',
{ count: 1 },
{
@@ -63,51 +74,45 @@ describe('store_report', () => {
overwrite: true,
}
);
- expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
+ expect(repository.incrementCounter).toHaveBeenNthCalledWith(
1,
'ui-metric',
'test-app-name:test-event-name',
[{ fieldName: 'count', incrementBy: 3 }]
);
- expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
+ expect(repository.incrementCounter).toHaveBeenNthCalledWith(
2,
'ui-counter',
`test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`,
[{ fieldName: 'count', incrementBy: 1 }]
);
- expect(savedObjectClient.incrementCounter).toHaveBeenNthCalledWith(
+ expect(repository.incrementCounter).toHaveBeenNthCalledWith(
3,
'ui-counter',
`test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`,
[{ fieldName: 'count', incrementBy: 2 }]
);
- expect(savedObjectClient.bulkCreate).toHaveBeenNthCalledWith(1, [
- {
- type: 'application_usage_transactional',
- attributes: {
- numberOfClicks: 3,
- minutesOnScreen: 10,
- appId: 'appId',
- viewId: 'appId_view',
- timestamp: expect.any(Date),
- },
- },
- ]);
+
+ expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1);
+ expect(storeApplicationUsageMock).toHaveBeenCalledWith(
+ repository,
+ Object.values(report.application_usage as Record),
+ expect.any(Date)
+ );
});
test('it should not fail if nothing to store', async () => {
- const savedObjectClient = savedObjectsRepositoryMock.create();
const report: ReportSchemaType = {
reportVersion: 3,
userAgent: void 0,
uiCounter: void 0,
application_usage: void 0,
};
- await storeReport(savedObjectClient, report);
+ await storeReport(repository, report);
- expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
- expect(savedObjectClient.incrementCounter).not.toHaveBeenCalled();
- expect(savedObjectClient.create).not.toHaveBeenCalled();
- expect(savedObjectClient.create).not.toHaveBeenCalled();
+ expect(repository.bulkCreate).not.toHaveBeenCalled();
+ expect(repository.incrementCounter).not.toHaveBeenCalled();
+ expect(repository.create).not.toHaveBeenCalled();
+ expect(repository.create).not.toHaveBeenCalled();
});
});
diff --git a/src/plugins/usage_collection/server/report/store_report.ts b/src/plugins/usage_collection/server/report/store_report.ts
index c3e04990d579..0545a54792d4 100644
--- a/src/plugins/usage_collection/server/report/store_report.ts
+++ b/src/plugins/usage_collection/server/report/store_report.ts
@@ -10,6 +10,7 @@ import { ISavedObjectsRepository } from 'src/core/server';
import moment from 'moment';
import { chain, sumBy } from 'lodash';
import { ReportSchemaType } from './schema';
+import { storeApplicationUsage } from './store_application_usage';
export async function storeReport(
internalRepository: ISavedObjectsRepository,
@@ -17,11 +18,11 @@ export async function storeReport(
) {
const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : [];
const userAgents = report.userAgent ? Object.entries(report.userAgent) : [];
- const appUsage = report.application_usage ? Object.values(report.application_usage) : [];
+ const appUsages = report.application_usage ? Object.values(report.application_usage) : [];
const momentTimestamp = moment();
- const timestamp = momentTimestamp.toDate();
const date = momentTimestamp.format('DDMMYYYY');
+ const timestamp = momentTimestamp.toDate();
return Promise.allSettled([
// User Agent
@@ -64,21 +65,6 @@ export async function storeReport(
];
}),
// Application Usage
- ...[
- (async () => {
- if (!appUsage.length) return [];
- const { saved_objects: savedObjects } = await internalRepository.bulkCreate(
- appUsage.map((metric) => ({
- type: 'application_usage_transactional',
- attributes: {
- ...metric,
- timestamp,
- },
- }))
- );
-
- return savedObjects;
- })(),
- ],
+ storeApplicationUsage(internalRepository, appUsages, timestamp),
]);
}
diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts
index a7b4da566b14..d0a09ee58d33 100644
--- a/test/api_integration/apis/telemetry/telemetry_local.ts
+++ b/test/api_integration/apis/telemetry/telemetry_local.ts
@@ -156,7 +156,7 @@ export default function ({ getService }: FtrProviderContext) {
describe('application usage limits', () => {
function createSavedObject(viewId?: string) {
return supertest
- .post('/api/saved_objects/application_usage_transactional')
+ .post('/api/saved_objects/application_usage_daily')
.send({
attributes: {
appId: 'test-app',
@@ -184,7 +184,7 @@ export default function ({ getService }: FtrProviderContext) {
await Promise.all(
savedObjectIds.map((savedObjectId) => {
return supertest
- .delete(`/api/saved_objects/application_usage_transactional/${savedObjectId}`)
+ .delete(`/api/saved_objects/application_usage_daily/${savedObjectId}`)
.expect(200);
})
);
@@ -230,7 +230,7 @@ export default function ({ getService }: FtrProviderContext) {
.post('/api/saved_objects/_bulk_create')
.send(
new Array(10001).fill(0).map(() => ({
- type: 'application_usage_transactional',
+ type: 'application_usage_daily',
attributes: {
appId: 'test-app',
minutesOnScreen: 1,
@@ -248,13 +248,12 @@ export default function ({ getService }: FtrProviderContext) {
// The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout
await es.deleteByQuery({
index: '.kibana',
- body: { query: { term: { type: 'application_usage_transactional' } } },
+ body: { query: { term: { type: 'application_usage_daily' } } },
conflicts: 'proceed',
});
});
- // flaky https://github.com/elastic/kibana/issues/94513
- it.skip("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
+ it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
const stats = await retrieveTelemetry(supertest);
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
'test-app': {