maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Application Usage] Daily rollups to overcome the find >10k SO limitation (#77610)

Browse source 
Co-authored-by: Christiane (Tina) Heiligers <christiane.heiligers@elastic.co>
This commit is contained in:
Alejandro Fernández Haro 2020-09-16 20:08:06 +01:00 committed by GitHub
parent e733d42d84
commit fd1aad0711
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 952 additions and 370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md
View file

@ -28,10 +28,10 @@ This collection occurs by default for every application registered via the menti
## Developer notes
In order to keep the count of the events, this collector uses 2 Saved Objects:
In order to keep the count of the events, this collector uses 3 Saved Objects:
1. `application_usage_transactional`: It stores each individually reported event (up to 90 days old). Grouped by `timestamp` and `appId`.
2. `application_usage_totals`: It stores the sum of all the events older than 90 days old per `appId`.
1. `application_usage_transactional`: It stores each individually reported event. Grouped by `timestamp` and `appId`. The reason for having these documents instead of editing `application_usage_daily` documents on very report is to provide faster response to the requests to `/api/ui_metric/report` (creating new documents instead of finding and editing existing ones) and to avoid conflicts when multiple users reach to the API concurrently.
2. `application_usage_daily`: Periodically, documents from `application_usage_transactional` are aggregated to daily summaries and deleted. Also grouped by `timestamp` and `appId`.
3. `application_usage_totals`: It stores the sum of all the events older than 90 days old, grouped by `appId`.
Both of them use the shared fields `appId: 'keyword'`, `numberOfClicks: 'long'` and `minutesOnScreen: 'float'`. `application_usage_transactional` also stores `timestamp: { type: 'date' }`.
but they are currently not added in the mappings because we don't use them for search purposes, and we need to be thoughtful with the number of mapped fields in the SavedObjects index ([#43673](https://github.com/elastic/kibana/issues/43673)).
All the types use the shared fields `appId: 'keyword'`, `numberOfClicks: 'long'` and `minutesOnScreen: 'float'`, but they are currently not added in the mappings because we don't use them for search purposes, and we need to be thoughtful with the number of mapped fields in the SavedObjects index ([#43673](https://github.com/elastic/kibana/issues/43673)). `application_usage_transactional` and `application_usage_daily` also store `timestamp: { type: 'date' }`.

151
src/plugins/kibana_usage_collection/server/collectors/application_usage/index.test.ts
View file

@ -1,151 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { savedObjectsRepositoryMock } from '../../../../../core/server/mocks';
import {
CollectorOptions,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
import { registerApplicationUsageCollector } from './';
import {
ROLL_INDICES_INTERVAL,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './telemetry_application_usage_collector';
describe('telemetry_application_usage', () => {
jest.useFakeTimers();
let collector: CollectorOptions;
const usageCollectionMock = createUsageCollectionSetupMock();
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = config;
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
const getUsageCollector = jest.fn();
const registerType = jest.fn();
const callCluster = jest.fn();
beforeAll(() =>
registerApplicationUsageCollector(usageCollectionMock, registerType, getUsageCollector)
);
afterAll(() => jest.clearAllTimers());
test('registered collector is set', () => {
expect(collector).not.toBeUndefined();
});
test('if no savedObjectClient initialised, return undefined', async () => {
expect(await collector.fetch(callCluster)).toBeUndefined();
jest.runTimersToTime(ROLL_INDICES_INTERVAL);
});
test('when savedObjectClient is initialised, return something', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(
async () =>
({
saved_objects: [],
total: 0,
} as any)
);
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_INDICES_INTERVAL); // Force rollTotals to run
expect(await collector.fetch(callCluster)).toStrictEqual({});
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
});
test('paging in findAll works', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let total = 201;
savedObjectClient.find.mockImplementation(async (opts) => {
if (opts.type === SAVED_OBJECTS_TOTAL_TYPE) {
return {
saved_objects: [
{
id: 'appId',
attributes: {
appId: 'appId',
minutesOnScreen: 10,
numberOfClicks: 10,
},
},
],
total: 1,
} as any;
}
if ((opts.page || 1) > 2) {
return { saved_objects: [], total };
}
const doc = {
id: 'test-id',
attributes: {
appId: 'appId',
timestamp: new Date().toISOString(),
minutesOnScreen: 1,
numberOfClicks: 1,
},
};
const savedObjects = new Array(opts.perPage).fill(doc);
total = savedObjects.length * 2 + 1;
return { saved_objects: savedObjects, total };
});
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_INDICES_INTERVAL); // Force rollTotals to run
expect(await collector.fetch(callCluster)).toStrictEqual({
appId: {
clicks_total: total - 1 + 10,
clicks_7_days: total - 1,
clicks_30_days: total - 1,
clicks_90_days: total - 1,
minutes_on_screen_total: total - 1 + 10,
minutes_on_screen_7_days: total - 1,
minutes_on_screen_30_days: total - 1,
minutes_on_screen_90_days: total - 1,
},
});
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
id: 'appId',
type: SAVED_OBJECTS_TOTAL_TYPE,
attributes: {
appId: 'appId',
minutesOnScreen: total - 1 + 10,
numberOfClicks: total - 1 + 10,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(total - 1);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id'
);
});
});

299
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.test.ts Normal file
View file

@ -0,0 +1,299 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { rollDailyData, rollTotals } from './rollups';
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
import { SavedObjectsErrorHelpers } from '../../../../../core/server';
import {
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
describe('rollDailyData', () => {
const logger = loggingSystemMock.createLogger();
test('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollDailyData(logger, undefined)).resolves.toBe(undefined);
});
test('handle empty results', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.get).not.toBeCalled();
expect(savedObjectClient.bulkCreate).not.toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
});
test('migrate some docs', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let timesCalled = 0;
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
if (timesCalled++ > 0) {
return { saved_objects: [], total: 0, page, per_page: perPage };
}
return {
saved_objects: [
{
id: 'test-id-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'test-id-2',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 1.5,
numberOfClicks: 2,
},
},
],
total: 2,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
savedObjectClient.get.mockImplementation(async (type, id) => {
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
expect(savedObjectClient.get).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId:2020-01-01'
);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
type: SAVED_OBJECTS_DAILY_TYPE,
id: 'appId:2020-01-01',
attributes: {
appId: 'appId',
timestamp: '2020-01-01T00:00:00.000Z',
minutesOnScreen: 2.0,
numberOfClicks: 3,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(2);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id-1'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id-2'
);
});
test('error getting the daily document', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let timesCalled = 0;
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
if (timesCalled++ > 0) {
return { saved_objects: [], total: 0, page, per_page: perPage };
}
return {
saved_objects: [
{
id: 'test-id-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
total: 1,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
savedObjectClient.get.mockImplementation(async (type, id) => {
throw new Error('Something went terribly wrong');
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
expect(savedObjectClient.get).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId:2020-01-01'
);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
});
describe('rollTotals', () => {
const logger = loggingSystemMock.createLogger();
test('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(rollTotals(logger, undefined)).resolves.toBe(undefined);
});
test('handle empty results', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_DAILY_TYPE:
case SAVED_OBJECTS_TOTAL_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
test('migrate some documents', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_DAILY_TYPE:
return {
saved_objects: [
{
id: 'appId-2:2020-01-01',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-2',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'appId-1:2020-01-01',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 1.5,
numberOfClicks: 2,
},
},
],
total: 2,
page,
per_page: perPage,
};
case SAVED_OBJECTS_TOTAL_TYPE:
return {
saved_objects: [
{
id: 'appId-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId-1',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
total: 1,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollTotals(logger, savedObjectClient)).resolves.toBe(undefined);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-1',
attributes: {
appId: 'appId-1',
minutesOnScreen: 2.0,
numberOfClicks: 3,
},
},
{
type: SAVED_OBJECTS_TOTAL_TYPE,
id: 'appId-2',
attributes: {
appId: 'appId-2',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(2);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-2:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01'
);
});
});

202
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups.ts Normal file
View file

@ -0,0 +1,202 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { ISavedObjectsRepository, SavedObject, Logger } from 'kibana/server';
import moment from 'moment';
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
ApplicationUsageTransactional,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
/**
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
*/
type ApplicationUsageDailyWithVersion = Pick<
SavedObject<ApplicationUsageDaily>,
'version' | 'attributes'
>;
/**
* Aggregates all the transactional events into daily aggregates
* @param logger
* @param savedObjectsClient
*/
export async function rollDailyData(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
if (!savedObjectsClient) {
return;
}
try {
let toCreate: Map<string, ApplicationUsageDailyWithVersion>;
do {
toCreate = new Map();
const { saved_objects: rawApplicationUsageTransactional } = await savedObjectsClient.find<
ApplicationUsageTransactional
>({
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
perPage: 1000, // Process 1000 at a time as a compromise of speed and overload
});
for (const doc of rawApplicationUsageTransactional) {
const {
attributes: { appId, minutesOnScreen, numberOfClicks, timestamp },
} = doc;
const dayId = moment(timestamp).format('YYYY-MM-DD');
const dailyId = `${appId}:${dayId}`;
const existingDoc =
toCreate.get(dailyId) || (await getDailyDoc(savedObjectsClient, dailyId, appId, dayId));
toCreate.set(dailyId, {
...existingDoc,
attributes: {
...existingDoc.attributes,
minutesOnScreen: existingDoc.attributes.minutesOnScreen + minutesOnScreen,
numberOfClicks: existingDoc.attributes.numberOfClicks + numberOfClicks,
},
});
}
if (toCreate.size > 0) {
await savedObjectsClient.bulkCreate(
[...toCreate.entries()].map(([id, { attributes, version }]) => ({
type: SAVED_OBJECTS_DAILY_TYPE,
id,
attributes,
version, // Providing version to ensure via conflict matching that only 1 Kibana instance (or interval) is taking care of the updates
})),
{ overwrite: true }
);
await Promise.all(
rawApplicationUsageTransactional.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_TRANSACTIONAL_TYPE, id) // There is no bulkDelete :(
)
);
}
} while (toCreate.size > 0);
} catch (err) {
logger.warn(`Failed to rollup transactional to daily entries`);
logger.warn(err);
}
}
/**
* Gets daily doc from the SavedObjects repository. Creates a new one if not found
* @param savedObjectsClient
* @param id The ID of the document to retrieve (typically, `${appId}:${dayId}`)
* @param appId The application ID
* @param dayId The date of the document in the format YYYY-MM-DD
*/
async function getDailyDoc(
savedObjectsClient: ISavedObjectsRepository,
id: string,
appId: string,
dayId: string
): Promise<ApplicationUsageDailyWithVersion> {
try {
return await savedObjectsClient.get<ApplicationUsageDaily>(SAVED_OBJECTS_DAILY_TYPE, id);
} catch (err) {
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
return {
attributes: {
appId,
// Concatenating the day in YYYY-MM-DD form to T00:00:00Z to reduce the TZ effects
timestamp: moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString(),
minutesOnScreen: 0,
numberOfClicks: 0,
},
};
}
throw err;
}
}
/**
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
* @param logger
* @param savedObjectsClient
*/
export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObjectsRepository) {
if (!savedObjectsClient) {
return;
}
try {
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
perPage: 10000,
type: SAVED_OBJECTS_TOTAL_TYPE,
}),
savedObjectsClient.find<ApplicationUsageDaily>({
perPage: 10000,
type: SAVED_OBJECTS_DAILY_TYPE,
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
}),
]);
const existingTotals = rawApplicationUsageTotals.reduce(
(acc, { attributes: { appId, numberOfClicks, minutesOnScreen } }) => {
return {
...acc,
// No need to sum because there should be 1 document per appId only
[appId]: { appId, numberOfClicks, minutesOnScreen },
};
},
{} as Record<string, { appId: string; minutesOnScreen: number; numberOfClicks: number }>
);
const totals = rawApplicationUsageDaily.reduce((acc, { attributes }) => {
const { appId, numberOfClicks, minutesOnScreen } = attributes;
const existing = acc[appId] || { minutesOnScreen: 0, numberOfClicks: 0 };
return {
...acc,
[appId]: {
appId,
numberOfClicks: numberOfClicks + existing.numberOfClicks,
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
},
};
}, existingTotals);
await Promise.all([
Object.entries(totals).length &&
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
Object.entries(totals).map(([id, entry]) => ({
type: SAVED_OBJECTS_TOTAL_TYPE,
id,
attributes: entry,
})),
{ overwrite: true }
),
...rawApplicationUsageDaily.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
),
]);
} catch (err) {
logger.warn(`Failed to rollup daily entries to totals`);
logger.warn(err);
}
}

32
src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts
View file

@ -19,19 +19,34 @@
import { SavedObjectAttributes, SavedObjectsServiceSetup } from 'kibana/server';
/**
* Used for accumulating the totals of all the stats older than 90d
*/
export interface ApplicationUsageTotal extends SavedObjectAttributes {
appId: string;
minutesOnScreen: number;
numberOfClicks: number;
}
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
/**
* Used for storing each of the reports received from the users' browsers
*/
export interface ApplicationUsageTransactional extends ApplicationUsageTotal {
timestamp: string;
}
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
/**
* Used to aggregate the transactional events into daily summaries so we can purge the granular events
*/
export type ApplicationUsageDaily = ApplicationUsageTransactional;
export const SAVED_OBJECTS_DAILY_TYPE = 'application_usage_daily';
export function registerMappings(registerType: SavedObjectsServiceSetup['registerType']) {
// Type for storing ApplicationUsageTotal
registerType({
name: 'application_usage_totals',
name: SAVED_OBJECTS_TOTAL_TYPE,
hidden: false,
namespaceType: 'agnostic',
mappings: {
@ -42,15 +57,28 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe
},
});
// Type for storing ApplicationUsageDaily
registerType({
name: 'application_usage_transactional',
name: SAVED_OBJECTS_DAILY_TYPE,
hidden: false,
namespaceType: 'agnostic',
mappings: {
dynamic: false,
properties: {
// This type requires `timestamp` to be indexed so we can use it when rolling up totals (timestamp < now-90d)
timestamp: { type: 'date' },
},
},
});
// Type for storing ApplicationUsageTransactional (declaring empty mappings because we don't use the internal fields for query/aggregations)
registerType({
name: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
hidden: false,
namespaceType: 'agnostic',
mappings: {
dynamic: false,
properties: {},
},
});
}

213
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts Normal file
View file

@ -0,0 +1,213 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../core/server/mocks';
import {
CollectorOptions,
createUsageCollectionSetupMock,
} from '../../../../usage_collection/server/usage_collection.mock';
import {
ROLL_INDICES_START,
ROLL_TOTAL_INDICES_INTERVAL,
registerApplicationUsageCollector,
} from './telemetry_application_usage_collector';
import {
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
describe('telemetry_application_usage', () => {
jest.useFakeTimers();
const logger = loggingSystemMock.createLogger();
let collector: CollectorOptions;
const usageCollectionMock = createUsageCollectionSetupMock();
usageCollectionMock.makeUsageCollector.mockImplementation((config) => {
collector = config;
return createUsageCollectionSetupMock().makeUsageCollector(config);
});
const getUsageCollector = jest.fn();
const registerType = jest.fn();
const callCluster = jest.fn();
beforeAll(() =>
registerApplicationUsageCollector(logger, usageCollectionMock, registerType, getUsageCollector)
);
afterAll(() => jest.clearAllTimers());
test('registered collector is set', () => {
expect(collector).not.toBeUndefined();
});
test('if no savedObjectClient initialised, return undefined', async () => {
expect(collector.isReady()).toBe(false);
expect(await collector.fetch(callCluster)).toBeUndefined();
jest.runTimersToTime(ROLL_INDICES_START);
});
test('when savedObjectClient is initialised, return something', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(
async () =>
({
saved_objects: [],
total: 0,
} as any)
);
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
expect(collector.isReady()).toBe(true);
expect(await collector.fetch(callCluster)).toStrictEqual({});
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
});
test('it only gets 10k even when there are more documents (ES limitation)', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
const total = 10000;
savedObjectClient.find.mockImplementation(async (opts) => {
switch (opts.type) {
case SAVED_OBJECTS_TOTAL_TYPE:
return {
saved_objects: [
{
id: 'appId',
attributes: {
appId: 'appId',
minutesOnScreen: 10,
numberOfClicks: 10,
},
},
],
total: 1,
} as any;
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
const doc = {
id: 'test-id',
attributes: {
appId: 'appId',
timestamp: new Date().toISOString(),
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
};
const savedObjects = new Array(total).fill(doc);
return { saved_objects: savedObjects, total: total + 1 };
case SAVED_OBJECTS_DAILY_TYPE:
return {
saved_objects: [
{
id: 'appId:YYYY-MM-DD',
attributes: {
appId: 'appId',
timestamp: new Date().toISOString(),
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
total: 1,
};
}
});
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_TOTAL_INDICES_INTERVAL); // Force rollTotals to run
expect(await collector.fetch(callCluster)).toStrictEqual({
appId: {
clicks_total: total + 1 + 10,
clicks_7_days: total + 1,
clicks_30_days: total + 1,
clicks_90_days: total + 1,
minutes_on_screen_total: (total + 1) * 0.5 + 10,
minutes_on_screen_7_days: (total + 1) * 0.5,
minutes_on_screen_30_days: (total + 1) * 0.5,
minutes_on_screen_90_days: (total + 1) * 0.5,
},
});
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
id: 'appId',
type: SAVED_OBJECTS_TOTAL_TYPE,
attributes: {
appId: 'appId',
minutesOnScreen: 10.5,
numberOfClicks: 11,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId:YYYY-MM-DD'
);
});
test('old transactional data not migrated yet', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async (opts) => {
switch (opts.type) {
case SAVED_OBJECTS_TOTAL_TYPE:
case SAVED_OBJECTS_DAILY_TYPE:
return { saved_objects: [], total: 0 } as any;
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
return {
saved_objects: [
{
id: 'test-id',
attributes: {
appId: 'appId',
timestamp: new Date(0).toISOString(),
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
total: 1,
};
}
});
getUsageCollector.mockImplementation(() => savedObjectClient);
expect(await collector.fetch(callCluster)).toStrictEqual({
appId: {
clicks_total: 1,
clicks_7_days: 0,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: 0.5,
minutes_on_screen_7_days: 0,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
},
});
});
});

197
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
View file

@ -18,29 +18,42 @@
*/
import moment from 'moment';
import { ISavedObjectsRepository, SavedObjectsServiceSetup } from 'kibana/server';
import { timer } from 'rxjs';
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from 'kibana/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { findAll } from '../find_all';
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
ApplicationUsageTransactional,
registerMappings,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './saved_objects_types';
import { applicationUsageSchema } from './schema';
import { rollDailyData, rollTotals } from './rollups';
/**
* Roll indices every 24h
* Roll total indices every 24h
*/
export const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Roll daily indices every 30 minutes.
* This means that, assuming a user can visit all the 44 apps we can possibly report
* in the 3 minutes interval the browser reports to the server, up to 22 users can have the same
* behaviour and we wouldn't need to paginate in the transactional documents (less than 10k docs).
*
* Based on a more normal expected use case, the users could visit up to 5 apps in those 3 minutes,
* allowing up to 200 users before reaching the limit.
*/
export const ROLL_DAILY_INDICES_INTERVAL = 30 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up
*/
export const ROLL_INDICES_START = 5 * 60 * 1000;
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
export interface ApplicationUsageTelemetryReport {
[appId: string]: {
clicks_total: number;
@ -55,6 +68,7 @@ export interface ApplicationUsageTelemetryReport {
}
export function registerApplicationUsageCollector(
logger: Logger,
usageCollection: UsageCollectionSetup,
registerType: SavedObjectsServiceSetup['registerType'],
getSavedObjectsClient: () => ISavedObjectsRepository | undefined
@ -71,10 +85,22 @@ export function registerApplicationUsageCollector(
if (typeof savedObjectsClient === 'undefined') {
return;
}
const [rawApplicationUsageTotals, rawApplicationUsageTransactional] = await Promise.all([
findAll<ApplicationUsageTotal>(savedObjectsClient, { type: SAVED_OBJECTS_TOTAL_TYPE }),
findAll<ApplicationUsageTransactional>(savedObjectsClient, {
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
{ saved_objects: rawApplicationUsageTransactional },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
type: SAVED_OBJECTS_TOTAL_TYPE,
perPage: 10000, // We only have 44 apps for now. This limit is OK.
}),
savedObjectsClient.find<ApplicationUsageDaily>({
type: SAVED_OBJECTS_DAILY_TYPE,
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
}),
savedObjectsClient.find<ApplicationUsageTransactional>({
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
perPage: 10000, // If we have more than those, we won't report the rest (they'll be rolled up to the daily soon enough to become a problem)
}),
]);
@ -101,51 +127,51 @@ export function registerApplicationUsageCollector(
const nowMinus30 = moment().subtract(30, 'days');
const nowMinus90 = moment().subtract(90, 'days');
const applicationUsage = rawApplicationUsageTransactional.reduce(
(acc, { attributes: { appId, minutesOnScreen, numberOfClicks, timestamp } }) => {
const existing = acc[appId] || {
clicks_total: 0,
clicks_7_days: 0,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: 0,
minutes_on_screen_7_days: 0,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
};
const applicationUsage = [
...rawApplicationUsageDaily,
...rawApplicationUsageTransactional,
].reduce((acc, { attributes: { appId, minutesOnScreen, numberOfClicks, timestamp } }) => {
const existing = acc[appId] || {
clicks_total: 0,
clicks_7_days: 0,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: 0,
minutes_on_screen_7_days: 0,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
};
const timeOfEntry = moment(timestamp as string);
const isInLast7Days = timeOfEntry.isSameOrAfter(nowMinus7);
const isInLast30Days = timeOfEntry.isSameOrAfter(nowMinus30);
const isInLast90Days = timeOfEntry.isSameOrAfter(nowMinus90);
const timeOfEntry = moment(timestamp);
const isInLast7Days = timeOfEntry.isSameOrAfter(nowMinus7);
const isInLast30Days = timeOfEntry.isSameOrAfter(nowMinus30);
const isInLast90Days = timeOfEntry.isSameOrAfter(nowMinus90);
const last7Days = {
clicks_7_days: existing.clicks_7_days + numberOfClicks,
minutes_on_screen_7_days: existing.minutes_on_screen_7_days + minutesOnScreen,
};
const last30Days = {
clicks_30_days: existing.clicks_30_days + numberOfClicks,
minutes_on_screen_30_days: existing.minutes_on_screen_30_days + minutesOnScreen,
};
const last90Days = {
clicks_90_days: existing.clicks_90_days + numberOfClicks,
minutes_on_screen_90_days: existing.minutes_on_screen_90_days + minutesOnScreen,
};
const last7Days = {
clicks_7_days: existing.clicks_7_days + numberOfClicks,
minutes_on_screen_7_days: existing.minutes_on_screen_7_days + minutesOnScreen,
};
const last30Days = {
clicks_30_days: existing.clicks_30_days + numberOfClicks,
minutes_on_screen_30_days: existing.minutes_on_screen_30_days + minutesOnScreen,
};
const last90Days = {
clicks_90_days: existing.clicks_90_days + numberOfClicks,
minutes_on_screen_90_days: existing.minutes_on_screen_90_days + minutesOnScreen,
};
return {
...acc,
[appId]: {
...existing,
clicks_total: existing.clicks_total + numberOfClicks,
minutes_on_screen_total: existing.minutes_on_screen_total + minutesOnScreen,
...(isInLast7Days ? last7Days : {}),
...(isInLast30Days ? last30Days : {}),
...(isInLast90Days ? last90Days : {}),
},
};
},
applicationUsageFromTotals
);
return {
...acc,
[appId]: {
...existing,
clicks_total: existing.clicks_total + numberOfClicks,
minutes_on_screen_total: existing.minutes_on_screen_total + minutesOnScreen,
...(isInLast7Days ? last7Days : {}),
...(isInLast30Days ? last30Days : {}),
...(isInLast90Days ? last90Days : {}),
},
};
}, applicationUsageFromTotals);
return applicationUsage;
},
@ -154,65 +180,10 @@ export function registerApplicationUsageCollector(
usageCollection.registerCollector(collector);
setInterval(() => rollTotals(getSavedObjectsClient()), ROLL_INDICES_INTERVAL);
setTimeout(() => rollTotals(getSavedObjectsClient()), ROLL_INDICES_START);
}
async function rollTotals(savedObjectsClient?: ISavedObjectsRepository) {
if (!savedObjectsClient) {
return;
}
try {
const [rawApplicationUsageTotals, rawApplicationUsageTransactional] = await Promise.all([
findAll<ApplicationUsageTotal>(savedObjectsClient, { type: SAVED_OBJECTS_TOTAL_TYPE }),
findAll<ApplicationUsageTransactional>(savedObjectsClient, {
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
filter: `${SAVED_OBJECTS_TRANSACTIONAL_TYPE}.attributes.timestamp < now-90d`,
}),
]);
const existingTotals = rawApplicationUsageTotals.reduce(
(acc, { attributes: { appId, numberOfClicks, minutesOnScreen } }) => {
return {
...acc,
// No need to sum because there should be 1 document per appId only
[appId]: { appId, numberOfClicks, minutesOnScreen },
};
},
{} as Record<string, { appId: string; minutesOnScreen: number; numberOfClicks: number }>
);
const totals = rawApplicationUsageTransactional.reduce((acc, { attributes, id }) => {
const { appId, numberOfClicks, minutesOnScreen } = attributes;
const existing = acc[appId] || { minutesOnScreen: 0, numberOfClicks: 0 };
return {
...acc,
[appId]: {
appId,
numberOfClicks: numberOfClicks + existing.numberOfClicks,
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
},
};
}, existingTotals);
await Promise.all([
Object.entries(totals).length &&
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
Object.entries(totals).map(([id, entry]) => ({
type: SAVED_OBJECTS_TOTAL_TYPE,
id,
attributes: entry,
})),
{ overwrite: true }
),
...rawApplicationUsageTransactional.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_TRANSACTIONAL_TYPE, id) // There is no bulkDelete :(
),
]);
} catch (err) {
// Silent failure
}
timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(() =>
rollDailyData(logger, getSavedObjectsClient())
);
timer(ROLL_INDICES_START, ROLL_TOTAL_INDICES_INTERVAL).subscribe(() =>
rollTotals(logger, getSavedObjectsClient())
);
}

55
src/plugins/kibana_usage_collection/server/collectors/find_all.test.ts
View file

@ -1,55 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { findAll } from './find_all';
describe('telemetry_application_usage', () => {
test('when savedObjectClient is initialised, return something', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(
async () =>
({
saved_objects: [],
total: 0,
} as any)
);
expect(await findAll(savedObjectClient, { type: 'test-type' })).toStrictEqual([]);
});
test('paging in findAll works', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let total = 201;
const doc = { id: 'test-id', attributes: { test: 1 } };
savedObjectClient.find.mockImplementation(async (opts) => {
if ((opts.page || 1) > 2) {
return { saved_objects: [], total } as any;
}
const savedObjects = new Array(opts.perPage).fill(doc);
total = savedObjects.length * 2 + 1;
return { saved_objects: savedObjects, total };
});
expect(await findAll(savedObjectClient, { type: 'test-type' })).toStrictEqual(
new Array(total - 1).fill(doc)
);
});
});

41
src/plugins/kibana_usage_collection/server/collectors/find_all.ts
View file

@ -1,41 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import {
SavedObjectAttributes,
ISavedObjectsRepository,
SavedObjectsFindOptions,
SavedObject,
} from 'kibana/server';
export async function findAll<T extends SavedObjectAttributes>(
savedObjectsClient: ISavedObjectsRepository,
opts: SavedObjectsFindOptions
): Promise<Array<SavedObject<T>>> {
const { page = 1, perPage = 10000, ...options } = opts;
const { saved_objects: savedObjects, total } = await savedObjectsClient.find<T>({
...options,
page,
perPage,
});
if (page * perPage >= total) {
return savedObjects;
}
return [...savedObjects, ...(await findAll<T>(savedObjectsClient, { ...opts, page: page + 1 }))];
}

4
src/plugins/kibana_usage_collection/server/collectors/ui_metric/telemetry_ui_metric_collector.ts
View file

@ -23,7 +23,6 @@ import {
SavedObjectsServiceSetup,
} from 'kibana/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { findAll } from '../find_all';
interface UIMetricsSavedObjects extends SavedObjectAttributes {
count: number;
@ -55,9 +54,10 @@ export function registerUiMetricUsageCollector(
return;
}
const rawUiMetrics = await findAll<UIMetricsSavedObjects>(savedObjectsClient, {
const { saved_objects: rawUiMetrics } = await savedObjectsClient.find<UIMetricsSavedObjects>({
type: 'ui-metric',
fields: ['count'],
perPage: 10000,
});
const uiMetricsByAppName = rawUiMetrics.reduce((accum, rawUiMetric) => {

10
src/plugins/kibana_usage_collection/server/plugin.ts
View file

@ -30,6 +30,7 @@ import {
CoreStart,
SavedObjectsServiceSetup,
OpsMetrics,
Logger,
} from '../../../core/server';
import {
registerApplicationUsageCollector,
@ -47,12 +48,14 @@ interface KibanaUsageCollectionPluginsDepsSetup {
type SavedObjectsRegisterType = SavedObjectsServiceSetup['registerType'];
export class KibanaUsageCollectionPlugin implements Plugin {
private readonly logger: Logger;
private readonly legacyConfig$: Observable<SharedGlobalConfig>;
private savedObjectsClient?: ISavedObjectsRepository;
private uiSettingsClient?: IUiSettingsClient;
private metric$: Subject<OpsMetrics>;
constructor(initializerContext: PluginInitializerContext) {
this.logger = initializerContext.logger.get();
this.legacyConfig$ = initializerContext.config.legacy.globalConfig$;
this.metric$ = new Subject<OpsMetrics>();
}
@ -88,7 +91,12 @@ export class KibanaUsageCollectionPlugin implements Plugin {
registerKibanaUsageCollector(usageCollection, this.legacyConfig$);
registerManagementUsageCollector(usageCollection, getUiSettingsClient);
registerUiMetricUsageCollector(usageCollection, registerType, getSavedObjectsClient);
registerApplicationUsageCollector(usageCollection, registerType, getSavedObjectsClient);
registerApplicationUsageCollector(
this.logger.get('application-usage'),
usageCollection,
registerType,
getSavedObjectsClient
);
registerCspCollector(usageCollection, coreSetup.http);
}
}

108
test/api_integration/apis/telemetry/telemetry_local.js
View file

@ -154,5 +154,113 @@ export default function ({ getService }) {
expect(expected.every((m) => actual.includes(m))).to.be.ok();
});
describe('application usage limits', () => {
const timeRange = {
min: '2018-07-23T22:07:00Z',
max: '2018-07-23T22:13:00Z',
};
function createSavedObject() {
return supertest
.post('/api/saved_objects/application_usage_transactional')
.send({
attributes: {
appId: 'test-app',
minutesOnScreen: 10.99,
numberOfClicks: 10,
timestamp: new Date().toISOString(),
},
})
.expect(200)
.then((resp) => resp.body.id);
}
describe('basic behaviour', () => {
let savedObjectId;
before('create 1 entry', async () => {
return createSavedObject().then((id) => (savedObjectId = id));
});
after('cleanup', () => {
return supertest
.delete(`/api/saved_objects/application_usage_transactional/${savedObjectId}`)
.expect(200);
});
it('should return application_usage data', async () => {
const { body } = await supertest
.post('/api/telemetry/v2/clusters/_stats')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, unencrypted: true })
.expect(200);
expect(body.length).to.be(1);
const stats = body[0];
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
'test-app': {
clicks_total: 10,
clicks_7_days: 10,
clicks_30_days: 10,
clicks_90_days: 10,
minutes_on_screen_total: 10.99,
minutes_on_screen_7_days: 10.99,
minutes_on_screen_30_days: 10.99,
minutes_on_screen_90_days: 10.99,
},
});
});
});
describe('10k + 1', () => {
const savedObjectIds = [];
before('create 10k + 1 entries for application usage', async () => {
await supertest
.post('/api/saved_objects/_bulk_create')
.send(
new Array(10001).fill(0).map(() => ({
type: 'application_usage_transactional',
attributes: {
appId: 'test-app',
minutesOnScreen: 1,
numberOfClicks: 1,
timestamp: new Date().toISOString(),
},
}))
)
.expect(200)
.then((resp) => resp.body.saved_objects.forEach(({ id }) => savedObjectIds.push(id)));
});
after('clean them all', async () => {
// The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout
await es.deleteByQuery({
index: '.kibana',
body: { query: { term: { type: 'application_usage_transactional' } } },
});
});
it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => {
const { body } = await supertest
.post('/api/telemetry/v2/clusters/_stats')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, unencrypted: true })
.expect(200);
expect(body.length).to.be(1);
const stats = body[0];
expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({
'test-app': {
clicks_total: 10000,
clicks_7_days: 10000,
clicks_30_days: 10000,
clicks_90_days: 10000,
minutes_on_screen_total: 10000,
minutes_on_screen_7_days: 10000,
minutes_on_screen_30_days: 10000,
minutes_on_screen_90_days: 10000,
},
});
});
});
});
});
}