[Telemetry] Application Usage implemented in @kbn/analytics (#58401)

* [Telemetry] Report the Application Usage (time of usage + number of clicks)

* Add Unit tests to the server side

* Do not use optional chaining in JS

* Add tests on the public end

* Fix jslint errors

* jest.useFakeTimers() + jest.clearAllTimers()

* Remove Jest timer handlers from my tests (only affecting to a minimum coverage bit)

* Catch ES actions in the setup/start steps because it broke core_services tests

* Fix boolean check

* Use core's ES.adminCLient over .createClient

* Fix tests after ES.adminClient

* [Telemetry] Application Usage implemented in kbn-analytics

* Use bulkCreate in store_report

* ApplicationUsagePluginStart does not exist anymore

* Fix usage_collection mock interface

* Check there is something to store before calling the bulkCreate method

* Add unit tests

* Fix types in tests

* Unit tests for rollTotals and actual fix for the bug found

* Fix usage_collection mock after #57693 got merged

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Alejandro Fernández Haro 2020-02-28 16:52:35 +00:00 committed by GitHub
parent 5b7270541c
commit fd25ae6505
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 931 additions and 80 deletions

View file

@ -0,0 +1,56 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import moment, { Moment } from 'moment-timezone';
import { METRIC_TYPE } from './';
export interface ApplicationUsageCurrent {
type: METRIC_TYPE.APPLICATION_USAGE;
appId: string;
startTime: Moment;
numberOfClicks: number;
}
export class ApplicationUsage {
private currentUsage?: ApplicationUsageCurrent;
public start() {
// Count any clicks and assign it to the current app
if (window)
window.addEventListener(
'click',
() => this.currentUsage && this.currentUsage.numberOfClicks++
);
}
public appChanged(appId?: string) {
const currentUsage = this.currentUsage;
if (appId) {
this.currentUsage = {
type: METRIC_TYPE.APPLICATION_USAGE,
appId,
startTime: moment(),
numberOfClicks: 0,
};
} else {
this.currentUsage = void 0;
}
return currentUsage;
}
}

View file

@ -19,15 +19,18 @@
import { UiStatsMetric } from './ui_stats';
import { UserAgentMetric } from './user_agent';
import { ApplicationUsageCurrent } from './application_usage';
export { UiStatsMetric, createUiStatsMetric, UiStatsMetricType } from './ui_stats';
export { Stats } from './stats';
export { trackUsageAgent } from './user_agent';
export { ApplicationUsage, ApplicationUsageCurrent } from './application_usage';
export type Metric = UiStatsMetric | UserAgentMetric;
export type Metric = UiStatsMetric | UserAgentMetric | ApplicationUsageCurrent;
export enum METRIC_TYPE {
COUNT = 'count',
LOADED = 'loaded',
CLICK = 'click',
USER_AGENT = 'user_agent',
APPLICATION_USAGE = 'application_usage',
}

View file

@ -17,6 +17,7 @@
* under the License.
*/
import moment from 'moment-timezone';
import { UnreachableCaseError, wrapArray } from './util';
import { Metric, Stats, UiStatsMetricType, METRIC_TYPE } from './metrics';
const REPORT_VERSION = 1;
@ -42,6 +43,13 @@ export interface Report {
appName: string;
}
>;
application_usage?: Record<
string,
{
minutesOnScreen: number;
numberOfClicks: number;
}
>;
}
export class ReportManager {
@ -57,10 +65,11 @@ export class ReportManager {
this.report = ReportManager.createReport();
}
public isReportEmpty(): boolean {
const { uiStatsMetrics, userAgent } = this.report;
const { uiStatsMetrics, userAgent, application_usage: appUsage } = this.report;
const noUiStats = !uiStatsMetrics || Object.keys(uiStatsMetrics).length === 0;
const noUserAgent = !userAgent || Object.keys(userAgent).length === 0;
return noUiStats && noUserAgent;
const noAppUsage = !appUsage || Object.keys(appUsage).length === 0;
return noUiStats && noUserAgent && noAppUsage;
}
private incrementStats(count: number, stats?: Stats): Stats {
const { min = 0, max = 0, sum = 0 } = stats || {};
@ -92,6 +101,8 @@ export class ReportManager {
const { appName, eventName, type } = metric;
return `${appName}-${type}-${eventName}`;
}
case METRIC_TYPE.APPLICATION_USAGE:
return metric.appId;
default:
throw new UnreachableCaseError(metric);
}
@ -129,6 +140,20 @@ export class ReportManager {
};
return;
}
case METRIC_TYPE.APPLICATION_USAGE:
const { numberOfClicks, startTime } = metric;
const minutesOnScreen = moment().diff(startTime, 'minutes', true);
report.application_usage = report.application_usage || {};
const appExistingData = report.application_usage[key] || {
minutesOnScreen: 0,
numberOfClicks: 0,
};
report.application_usage[key] = {
minutesOnScreen: appExistingData.minutesOnScreen + minutesOnScreen,
numberOfClicks: appExistingData.numberOfClicks + numberOfClicks,
};
break;
default:
throw new UnreachableCaseError(metric);
}

View file

@ -22,6 +22,7 @@ import { Metric, createUiStatsMetric, trackUsageAgent, UiStatsMetricType } from
import { Storage, ReportStorageManager } from './storage';
import { Report, ReportManager } from './report';
import { ApplicationUsage } from './metrics';
export interface ReporterConfig {
http: ReportHTTP;
@ -35,19 +36,22 @@ export type ReportHTTP = (report: Report) => Promise<void>;
export class Reporter {
checkInterval: number;
private interval: any;
private interval?: NodeJS.Timer;
private lastAppId?: string;
private http: ReportHTTP;
private reportManager: ReportManager;
private storageManager: ReportStorageManager;
private readonly applicationUsage: ApplicationUsage;
private debug: boolean;
private retryCount = 0;
private readonly maxRetries = 3;
private started = false;
constructor(config: ReporterConfig) {
const { http, storage, debug, checkInterval = 90000, storageKey = 'analytics' } = config;
this.http = http;
this.checkInterval = checkInterval;
this.interval = null;
this.applicationUsage = new ApplicationUsage();
this.storageManager = new ReportStorageManager(storageKey, storage);
const storedReport = this.storageManager.get();
this.reportManager = new ReportManager(storedReport);
@ -68,10 +72,34 @@ export class Reporter {
public start = () => {
if (!this.interval) {
this.interval = setTimeout(() => {
this.interval = null;
this.interval = undefined;
this.sendReports();
}, this.checkInterval);
}
if (this.started) {
return;
}
if (window && document) {
// Before leaving the page, make sure we store the current usage
window.addEventListener('beforeunload', () => this.reportApplicationUsage());
// Monitoring dashboards might be open in background and we are fine with that
// but we don't want to report hours if the user goes to another tab and Kibana is not shown
document.addEventListener('visibilitychange', () => {
if (document.visibilityState === 'visible' && this.lastAppId) {
this.reportApplicationUsage(this.lastAppId);
} else if (document.visibilityState === 'hidden') {
this.reportApplicationUsage();
// We also want to send the report now because intervals and timeouts be stalled when too long in the "hidden" state
this.sendReports();
}
});
}
this.started = true;
this.applicationUsage.start();
};
private log(message: any) {
@ -102,6 +130,13 @@ export class Reporter {
this.saveToReport([report]);
};
public reportApplicationUsage(appId?: string) {
this.log(`Reporting application changed to ${appId}`);
this.lastAppId = appId || this.lastAppId;
const appChangedReport = this.applicationUsage.appChanged(appId);
if (appChangedReport) this.saveToReport([appChangedReport]);
}
public sendReports = async () => {
if (!this.reportManager.isReportEmpty()) {
try {

View file

@ -0,0 +1,31 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Legacy } from '../../../../kibana';
import { mappings } from './mappings';
// eslint-disable-next-line import/no-default-export
export default function ApplicationUsagePlugin(kibana: any) {
const config: Legacy.PluginSpecOptions = {
id: 'application_usage',
uiExports: { mappings }, // Needed to define the mappings for the SavedObjects
};
return new kibana.Plugin(config);
}

View file

@ -0,0 +1,36 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export const mappings = {
application_usage_totals: {
properties: {
appId: { type: 'keyword' },
numberOfClicks: { type: 'long' },
minutesOnScreen: { type: 'float' },
},
},
application_usage_transactional: {
properties: {
timestamp: { type: 'date' },
appId: { type: 'keyword' },
numberOfClicks: { type: 'long' },
minutesOnScreen: { type: 'float' },
},
},
};

View file

@ -0,0 +1,4 @@
{
"name": "application_usage",
"version": "kibana"
}

View file

@ -66,6 +66,11 @@ export const TELEMETRY_STATS_TYPE = 'telemetry';
*/
export const UI_METRIC_USAGE_TYPE = 'ui_metric';
/**
* Application Usage type
*/
export const APPLICATION_USAGE_TYPE = 'application_usage';
/**
* Link to Advanced Settings.
*/

View file

@ -21,7 +21,7 @@ import * as Rx from 'rxjs';
import { resolve } from 'path';
import JoiNamespace from 'joi';
import { Server } from 'hapi';
import { CoreSetup, PluginInitializerContext } from 'src/core/server';
import { PluginInitializerContext } from 'src/core/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { getConfigPath } from '../../../core/server/path';
// @ts-ignore
@ -132,11 +132,6 @@ const telemetry = (kibana: any) => {
},
} as PluginInitializerContext;
const coreSetup = ({
http: { server },
log: server.log,
} as any) as CoreSetup;
try {
await handleOldSettings(server);
} catch (err) {
@ -147,7 +142,9 @@ const telemetry = (kibana: any) => {
usageCollection,
};
telemetryPlugin(initializerContext).setup(coreSetup, pluginsSetup, server);
const npPlugin = telemetryPlugin(initializerContext);
await npPlugin.setup(server.newPlatform.setup.core, pluginsSetup, server);
await npPlugin.start(server.newPlatform.start.core);
},
});
};

View file

@ -0,0 +1,144 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
import { savedObjectsRepositoryMock } from '../../../../../../core/server/mocks';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { CollectorOptions } from '../../../../../../plugins/usage_collection/server/collector/collector';
import { registerApplicationUsageCollector } from './';
import {
ROLL_INDICES_INTERVAL,
SAVED_OBJECTS_TOTAL_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from './telemetry_application_usage_collector';
describe('telemetry_application_usage', () => {
jest.useFakeTimers();
let collector: CollectorOptions;
const usageCollectionMock: jest.Mocked<UsageCollectionSetup> = {
makeUsageCollector: jest.fn().mockImplementation(config => (collector = config)),
registerCollector: jest.fn(),
} as any;
const getUsageCollector = jest.fn();
const callCluster = jest.fn();
beforeAll(() => registerApplicationUsageCollector(usageCollectionMock, getUsageCollector));
afterAll(() => jest.clearAllTimers());
test('registered collector is set', () => {
expect(collector).not.toBeUndefined();
});
test('if no savedObjectClient initialised, return undefined', async () => {
expect(await collector.fetch(callCluster)).toBeUndefined();
jest.runTimersToTime(ROLL_INDICES_INTERVAL);
});
test('when savedObjectClient is initialised, return something', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(
async () =>
({
saved_objects: [],
total: 0,
} as any)
);
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_INDICES_INTERVAL); // Force rollTotals to run
expect(await collector.fetch(callCluster)).toStrictEqual({});
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
});
test('paging in findAll works', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let total = 201;
savedObjectClient.find.mockImplementation(async opts => {
if (opts.type === SAVED_OBJECTS_TOTAL_TYPE) {
return {
saved_objects: [
{
id: 'appId',
attributes: {
appId: 'appId',
minutesOnScreen: 10,
numberOfClicks: 10,
},
},
],
total: 1,
} as any;
}
if ((opts.page || 1) > 2) {
return { saved_objects: [], total };
}
const doc = {
id: 'test-id',
attributes: {
appId: 'appId',
timestamp: new Date().toISOString(),
minutesOnScreen: 1,
numberOfClicks: 1,
},
};
const savedObjects = new Array(opts.perPage).fill(doc);
total = savedObjects.length * 2 + 1;
return { saved_objects: savedObjects, total };
});
getUsageCollector.mockImplementation(() => savedObjectClient);
jest.runTimersToTime(ROLL_INDICES_INTERVAL); // Force rollTotals to run
expect(await collector.fetch(callCluster)).toStrictEqual({
appId: {
clicks_total: total - 1 + 10,
clicks_30_days: total - 1,
clicks_90_days: total - 1,
minutes_on_screen_total: total - 1 + 10,
minutes_on_screen_30_days: total - 1,
minutes_on_screen_90_days: total - 1,
},
});
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
id: 'appId',
type: SAVED_OBJECTS_TOTAL_TYPE,
attributes: {
appId: 'appId',
minutesOnScreen: total - 1 + 10,
numberOfClicks: total - 1 + 10,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(total - 1);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id'
);
});
});

View file

@ -0,0 +1,20 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';

View file

@ -0,0 +1,225 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import moment from 'moment';
import { APPLICATION_USAGE_TYPE } from '../../../common/constants';
import { UsageCollectionSetup } from '../../../../../../plugins/usage_collection/server';
import {
ISavedObjectsRepository,
SavedObjectAttributes,
SavedObjectsFindOptions,
SavedObject,
} from '../../../../../../core/server';
/**
* Roll indices every 24h
*/
export const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up
*/
export const ROLL_INDICES_START = 5 * 60 * 1000;
export const SAVED_OBJECTS_TOTAL_TYPE = 'application_usage_totals';
export const SAVED_OBJECTS_TRANSACTIONAL_TYPE = 'application_usage_transactional';
interface ApplicationUsageTotal extends SavedObjectAttributes {
appId: string;
minutesOnScreen: number;
numberOfClicks: number;
}
interface ApplicationUsageTransactional extends ApplicationUsageTotal {
timestamp: string;
}
interface ApplicationUsageTelemetryReport {
[appId: string]: {
clicks_total: number;
clicks_30_days: number;
clicks_90_days: number;
minutes_on_screen_total: number;
minutes_on_screen_30_days: number;
minutes_on_screen_90_days: number;
};
}
async function findAll<T extends SavedObjectAttributes>(
savedObjectsClient: ISavedObjectsRepository,
opts: SavedObjectsFindOptions
): Promise<Array<SavedObject<T>>> {
const { page = 1, perPage = 100, ...options } = opts;
const { saved_objects: savedObjects, total } = await savedObjectsClient.find<T>({
...options,
page,
perPage,
});
if (page * perPage >= total) {
return savedObjects;
}
return [...savedObjects, ...(await findAll<T>(savedObjectsClient, { ...opts, page: page + 1 }))];
}
export function registerApplicationUsageCollector(
usageCollection: UsageCollectionSetup,
getSavedObjectsClient: () => ISavedObjectsRepository | undefined
) {
const collector = usageCollection.makeUsageCollector({
type: APPLICATION_USAGE_TYPE,
isReady: () => typeof getSavedObjectsClient() !== 'undefined',
fetch: async () => {
const savedObjectsClient = getSavedObjectsClient();
if (typeof savedObjectsClient === 'undefined') {
return;
}
const [rawApplicationUsageTotals, rawApplicationUsageTransactional] = await Promise.all([
findAll<ApplicationUsageTotal>(savedObjectsClient, { type: SAVED_OBJECTS_TOTAL_TYPE }),
findAll<ApplicationUsageTransactional>(savedObjectsClient, {
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
}),
]);
const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
(acc, { attributes: { appId, minutesOnScreen, numberOfClicks } }) => {
const existing = acc[appId] || { clicks_total: 0, minutes_on_screen_total: 0 };
return {
...acc,
[appId]: {
clicks_total: numberOfClicks + existing.clicks_total,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: minutesOnScreen + existing.minutes_on_screen_total,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
},
};
},
{} as ApplicationUsageTelemetryReport
);
const nowMinus30 = moment().subtract(30, 'days');
const nowMinus90 = moment().subtract(90, 'days');
const applicationUsage = rawApplicationUsageTransactional.reduce(
(acc, { attributes: { appId, minutesOnScreen, numberOfClicks, timestamp } }) => {
const existing = acc[appId] || {
clicks_total: 0,
clicks_30_days: 0,
clicks_90_days: 0,
minutes_on_screen_total: 0,
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
};
const timeOfEntry = moment(timestamp as string);
const isInLast30Days = timeOfEntry.isSameOrAfter(nowMinus30);
const isInLast90Days = timeOfEntry.isSameOrAfter(nowMinus90);
const last30Days = {
clicks_30_days: existing.clicks_30_days + numberOfClicks,
minutes_on_screen_30_days: existing.minutes_on_screen_30_days + minutesOnScreen,
};
const last90Days = {
clicks_90_days: existing.clicks_90_days + numberOfClicks,
minutes_on_screen_90_days: existing.minutes_on_screen_90_days + minutesOnScreen,
};
return {
...acc,
[appId]: {
...existing,
clicks_total: existing.clicks_total + numberOfClicks,
minutes_on_screen_total: existing.minutes_on_screen_total + minutesOnScreen,
...(isInLast30Days ? last30Days : {}),
...(isInLast90Days ? last90Days : {}),
},
};
},
applicationUsageFromTotals
);
return applicationUsage;
},
});
usageCollection.registerCollector(collector);
setInterval(() => rollTotals(getSavedObjectsClient()), ROLL_INDICES_INTERVAL);
setTimeout(() => rollTotals(getSavedObjectsClient()), ROLL_INDICES_START);
}
async function rollTotals(savedObjectsClient?: ISavedObjectsRepository) {
if (!savedObjectsClient) {
return;
}
try {
const [rawApplicationUsageTotals, rawApplicationUsageTransactional] = await Promise.all([
findAll<ApplicationUsageTotal>(savedObjectsClient, { type: SAVED_OBJECTS_TOTAL_TYPE }),
findAll<ApplicationUsageTransactional>(savedObjectsClient, {
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
filter: `${SAVED_OBJECTS_TRANSACTIONAL_TYPE}.attributes.timestamp < now-90d`,
}),
]);
const existingTotals = rawApplicationUsageTotals.reduce(
(acc, { attributes: { appId, numberOfClicks, minutesOnScreen } }) => {
return {
...acc,
// No need to sum because there should be 1 document per appId only
[appId]: { appId, numberOfClicks, minutesOnScreen },
};
},
{} as Record<string, { appId: string; minutesOnScreen: number; numberOfClicks: number }>
);
const totals = rawApplicationUsageTransactional.reduce((acc, { attributes, id }) => {
const { appId, numberOfClicks, minutesOnScreen } = attributes;
const existing = acc[appId] || { minutesOnScreen: 0, numberOfClicks: 0 };
return {
...acc,
[appId]: {
appId,
numberOfClicks: numberOfClicks + existing.numberOfClicks,
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
},
};
}, existingTotals);
await Promise.all([
Object.entries(totals).length &&
savedObjectsClient.bulkCreate<ApplicationUsageTotal>(
Object.entries(totals).map(([id, entry]) => ({
type: SAVED_OBJECTS_TOTAL_TYPE,
id,
attributes: entry,
})),
{ overwrite: true }
),
...rawApplicationUsageTransactional.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_TRANSACTIONAL_TYPE, id) // There is no bulkDelete :(
),
]);
} catch (err) {
// Silent failure
}
}

View file

@ -23,3 +23,4 @@ export { registerUiMetricUsageCollector } from './ui_metric';
export { registerLocalizationUsageCollector } from './localization';
export { registerTelemetryPluginUsageCollector } from './telemetry_plugin';
export { registerManagementUsageCollector } from './management';
export { registerApplicationUsageCollector } from './application_usage';

View file

@ -17,7 +17,12 @@
* under the License.
*/
import { CoreSetup, PluginInitializerContext } from 'src/core/server';
import {
CoreSetup,
PluginInitializerContext,
ISavedObjectsRepository,
CoreStart,
} from 'src/core/server';
import { Server } from 'hapi';
import { registerRoutes } from './routes';
import { registerCollection } from './telemetry_collection';
@ -28,6 +33,7 @@ import {
registerLocalizationUsageCollector,
registerTelemetryPluginUsageCollector,
registerManagementUsageCollector,
registerApplicationUsageCollector,
} from './collectors';
export interface PluginsSetup {
@ -36,6 +42,7 @@ export interface PluginsSetup {
export class TelemetryPlugin {
private readonly currentKibanaVersion: string;
private savedObjectsClient?: ISavedObjectsRepository;
constructor(initializerContext: PluginInitializerContext) {
this.currentKibanaVersion = initializerContext.env.packageInfo.version;
@ -45,12 +52,19 @@ export class TelemetryPlugin {
const currentKibanaVersion = this.currentKibanaVersion;
registerCollection();
registerRoutes({ core, currentKibanaVersion });
registerRoutes({ core, currentKibanaVersion, server });
const getSavedObjectsClient = () => this.savedObjectsClient;
registerTelemetryPluginUsageCollector(usageCollection, server);
registerLocalizationUsageCollector(usageCollection, server);
registerTelemetryUsageCollector(usageCollection, server);
registerUiMetricUsageCollector(usageCollection, server);
registerManagementUsageCollector(usageCollection, server);
registerApplicationUsageCollector(usageCollection, getSavedObjectsClient);
}
public start({ savedObjects }: CoreStart) {
this.savedObjectsClient = savedObjects.createInternalRepository();
}
}

View file

@ -17,6 +17,7 @@
* under the License.
*/
import { Legacy } from 'kibana';
import { CoreSetup } from 'src/core/server';
import { registerTelemetryOptInRoutes } from './telemetry_opt_in';
import { registerTelemetryUsageStatsRoutes } from './telemetry_usage_stats';
@ -26,11 +27,12 @@ import { registerTelemetryUserHasSeenNotice } from './telemetry_user_has_seen_no
interface RegisterRoutesParams {
core: CoreSetup;
currentKibanaVersion: string;
server: Legacy.Server;
}
export function registerRoutes({ core, currentKibanaVersion }: RegisterRoutesParams) {
registerTelemetryOptInRoutes({ core, currentKibanaVersion });
registerTelemetryUsageStatsRoutes(core);
registerTelemetryOptInStatsRoutes(core);
registerTelemetryUserHasSeenNotice(core);
export function registerRoutes({ core, currentKibanaVersion, server }: RegisterRoutesParams) {
registerTelemetryOptInRoutes({ core, currentKibanaVersion, server });
registerTelemetryUsageStatsRoutes(server);
registerTelemetryOptInStatsRoutes(server);
registerTelemetryUserHasSeenNotice(server);
}

View file

@ -21,6 +21,7 @@ import Joi from 'joi';
import moment from 'moment';
import { boomify } from 'boom';
import { CoreSetup } from 'src/core/server';
import { Legacy } from 'kibana';
import { getTelemetryAllowChangingOptInStatus } from '../telemetry_config';
import { sendTelemetryOptInStatus } from './telemetry_opt_in_stats';
@ -32,14 +33,13 @@ import {
interface RegisterOptInRoutesParams {
core: CoreSetup;
currentKibanaVersion: string;
server: Legacy.Server;
}
export function registerTelemetryOptInRoutes({
core,
server,
currentKibanaVersion,
}: RegisterOptInRoutesParams) {
const { server } = core.http as any;
server.route({
method: 'POST',
path: '/api/telemetry/v2/optIn',

View file

@ -21,7 +21,7 @@
import fetch from 'node-fetch';
import Joi from 'joi';
import moment from 'moment';
import { CoreSetup } from 'src/core/server';
import { Legacy } from 'kibana';
import { telemetryCollectionManager, StatsGetterConfig } from '../collection_manager';
interface SendTelemetryOptInStatusConfig {
@ -45,9 +45,7 @@ export async function sendTelemetryOptInStatus(
});
}
export function registerTelemetryOptInStatsRoutes(core: CoreSetup) {
const { server } = core.http as any;
export function registerTelemetryOptInStatsRoutes(server: Legacy.Server) {
server.route({
method: 'POST',
path: '/api/telemetry/v2/clusters/_opt_in_stats',

View file

@ -19,16 +19,14 @@
import Joi from 'joi';
import { boomify } from 'boom';
import { CoreSetup } from 'src/core/server';
import { Legacy } from 'kibana';
import { telemetryCollectionManager } from '../collection_manager';
export function registerTelemetryUsageStatsRoutes(core: CoreSetup) {
const { server } = core.http as any;
export function registerTelemetryUsageStatsRoutes(server: Legacy.Server) {
server.route({
method: 'POST',
path: '/api/telemetry/v2/clusters/_stats',
config: {
options: {
validate: {
payload: Joi.object({
unencrypted: Joi.bool(),

View file

@ -19,7 +19,6 @@
import { Legacy } from 'kibana';
import { Request } from 'hapi';
import { CoreSetup } from 'src/core/server';
import {
TelemetrySavedObject,
TelemetrySavedObjectAttributes,
@ -34,9 +33,7 @@ const getInternalRepository = (server: Legacy.Server) => {
return internalRepository;
};
export function registerTelemetryUserHasSeenNotice(core: CoreSetup) {
const { server }: { server: Legacy.Server } = core.http as any;
export function registerTelemetryUserHasSeenNotice(server: Legacy.Server) {
server.route({
method: 'PUT',
path: '/api/telemetry/v2/userHasSeenNotice',

View file

@ -18,7 +18,6 @@
*/
import { resolve } from 'path';
import { Legacy } from '../../../../kibana';
// eslint-disable-next-line import/no-default-export
export default function(kibana: any) {
@ -29,13 +28,6 @@ export default function(kibana: any) {
uiExports: {
mappings: require('./mappings.json'),
},
init(server: Legacy.Server) {
const { getSavedObjectsRepository } = server.savedObjects;
const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
const internalRepository = getSavedObjectsRepository(callWithInternalUser);
const { usageCollection } = server.newPlatform.setup.plugins;
usageCollection.registerLegacySavedObjects(internalRepository);
},
init() {},
});
}

View file

@ -21,6 +21,7 @@ import url from 'url';
import { unhashUrl } from '../../../../../plugins/kibana_utils/public';
import { toastNotifications } from '../../notify/toasts';
import { npSetup } from '../../new_platform';
export function registerSubUrlHooks(angularModule, internals) {
angularModule.run(($rootScope, Private, $location) => {
@ -40,6 +41,7 @@ export function registerSubUrlHooks(angularModule, internals) {
function onRouteChange($event) {
if (subUrlRouteFilter($event)) {
updateUsage($event);
updateSubUrls();
}
}
@ -67,6 +69,13 @@ export function registerSubUrlHooks(angularModule, internals) {
});
}
function updateUsage($event) {
const scope = $event.targetScope;
const app = scope.chrome.getApp();
const appId = app.id === 'kibana' ? scope.getFirstPathSegment() : app.id;
if (npSetup.plugins.usageCollection) npSetup.plugins.usageCollection.__LEGACY.appChanged(appId);
}
/**
* Creates a function that will be called on each route change
* to determine if the event should be used to update the last

View file

@ -26,6 +26,9 @@ const createSetupContract = (): Setup => {
allowTrackUserAgent: jest.fn(),
reportUiStats: jest.fn(),
METRIC_TYPE,
__LEGACY: {
appChanged: jest.fn(),
},
};
return setupContract;

View file

@ -18,6 +18,7 @@
*/
import { Reporter, METRIC_TYPE } from '@kbn/analytics';
import { Subject, merge } from 'rxjs';
import { Storage } from '../../kibana_utils/public';
import { createReporter } from './services';
import {
@ -27,6 +28,7 @@ import {
CoreStart,
HttpSetup,
} from '../../../core/public';
import { reportApplicationUsage } from './services/application_usage';
interface PublicConfigType {
uiMetric: {
@ -39,6 +41,15 @@ export interface UsageCollectionSetup {
allowTrackUserAgent: (allow: boolean) => void;
reportUiStats: Reporter['reportUiStats'];
METRIC_TYPE: typeof METRIC_TYPE;
__LEGACY: {
/**
* Legacy handler so we can report the actual app being used inside "kibana#/{appId}".
* To be removed when we get rid of the legacy world
*
* @deprecated
*/
appChanged: (appId: string) => void;
};
}
export function isUnauthenticated(http: HttpSetup) {
@ -47,6 +58,7 @@ export function isUnauthenticated(http: HttpSetup) {
}
export class UsageCollectionPlugin implements Plugin<UsageCollectionSetup> {
private readonly legacyAppId$ = new Subject<string>();
private trackUserAgent: boolean = true;
private reporter?: Reporter;
private config: PublicConfigType;
@ -70,10 +82,13 @@ export class UsageCollectionPlugin implements Plugin<UsageCollectionSetup> {
},
reportUiStats: this.reporter.reportUiStats,
METRIC_TYPE,
__LEGACY: {
appChanged: appId => this.legacyAppId$.next(appId),
},
};
}
public start({ http }: CoreStart) {
public start({ http, application }: CoreStart) {
if (!this.reporter) {
return;
}
@ -85,6 +100,7 @@ export class UsageCollectionPlugin implements Plugin<UsageCollectionSetup> {
if (this.trackUserAgent) {
this.reporter.reportUserAgent('kibana');
}
reportApplicationUsage(merge(application.currentAppId$, this.legacyAppId$), this.reporter);
}
public stop() {}

View file

@ -0,0 +1,69 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Reporter } from '@kbn/analytics';
import { Subject } from 'rxjs';
import { reportApplicationUsage } from './application_usage';
describe('application_usage', () => {
test('report an appId change', () => {
const reporterMock: jest.Mocked<Reporter> = {
reportApplicationUsage: jest.fn(),
} as any;
const currentAppId$ = new Subject<string | undefined>();
reportApplicationUsage(currentAppId$, reporterMock);
currentAppId$.next('appId');
expect(reporterMock.reportApplicationUsage).toHaveBeenCalledWith('appId');
expect(reporterMock.reportApplicationUsage).toHaveBeenCalledTimes(1);
});
test('skip duplicates', () => {
const reporterMock: jest.Mocked<Reporter> = {
reportApplicationUsage: jest.fn(),
} as any;
const currentAppId$ = new Subject<string | undefined>();
reportApplicationUsage(currentAppId$, reporterMock);
currentAppId$.next('appId');
currentAppId$.next('appId');
expect(reporterMock.reportApplicationUsage).toHaveBeenCalledWith('appId');
expect(reporterMock.reportApplicationUsage).toHaveBeenCalledTimes(1);
});
test('skip if not a valid value', () => {
const reporterMock: jest.Mocked<Reporter> = {
reportApplicationUsage: jest.fn(),
} as any;
const currentAppId$ = new Subject<string | undefined>();
reportApplicationUsage(currentAppId$, reporterMock);
currentAppId$.next('');
currentAppId$.next('kibana');
currentAppId$.next(undefined);
expect(reporterMock.reportApplicationUsage).toHaveBeenCalledTimes(0);
});
});

View file

@ -0,0 +1,39 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Observable } from 'rxjs';
import { filter, distinctUntilChanged } from 'rxjs/operators';
import { Reporter } from '@kbn/analytics';
/**
* List of appIds not to report usage from (due to legacy hacks)
*/
const DO_NOT_REPORT = ['kibana'];
export function reportApplicationUsage(
currentAppId$: Observable<string | undefined>,
reporter: Reporter
) {
currentAppId$
.pipe(
filter(appId => typeof appId === 'string' && !DO_NOT_REPORT.includes(appId)),
distinctUntilChanged()
)
.subscribe(appId => appId && reporter.reportApplicationUsage(appId));
}

View file

@ -27,9 +27,6 @@ const createSetupContract = () => {
logger: loggingServiceMock.createLogger(),
maximumWaitTimeForAllCollectorsInS: 1,
}),
registerLegacySavedObjects: jest.fn() as jest.Mocked<
UsageCollectionSetup['registerLegacySavedObjects']
>,
} as UsageCollectionSetup;
};

View file

@ -18,18 +18,16 @@
*/
import { first } from 'rxjs/operators';
import { CoreStart, ISavedObjectsRepository } from 'kibana/server';
import { ConfigType } from './config';
import { PluginInitializerContext, Logger, CoreSetup } from '../../../../src/core/server';
import { CollectorSet } from './collector';
import { setupRoutes } from './routes';
export type UsageCollectionSetup = CollectorSet & {
registerLegacySavedObjects: (legacySavedObjects: any) => void;
};
export type UsageCollectionSetup = CollectorSet;
export class UsageCollectionPlugin {
logger: Logger;
private legacySavedObjects: any;
private savedObjects?: ISavedObjectsRepository;
constructor(private readonly initializerContext: PluginInitializerContext) {
this.logger = this.initializerContext.logger.get();
}
@ -46,19 +44,14 @@ export class UsageCollectionPlugin {
});
const router = core.http.createRouter();
const getLegacySavedObjects = () => this.legacySavedObjects;
setupRoutes(router, getLegacySavedObjects);
setupRoutes(router, () => this.savedObjects);
return {
...collectorSet,
registerLegacySavedObjects: (legacySavedObjects: any) => {
this.legacySavedObjects = legacySavedObjects;
},
};
return collectorSet;
}
public start() {
public start({ savedObjects }: CoreStart) {
this.logger.debug('Starting plugin');
this.savedObjects = savedObjects.createInternalRepository();
}
public stop() {

View file

@ -54,6 +54,15 @@ export const reportSchema = schema.object({
})
)
),
application_usage: schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
minutesOnScreen: schema.number(),
numberOfClicks: schema.number(),
})
)
),
});
export type ReportSchemaType = TypeOf<typeof reportSchema>;

View file

@ -0,0 +1,102 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { savedObjectsRepositoryMock } from '../../../../core/server/mocks';
import { storeReport } from './store_report';
import { ReportSchemaType } from './schema';
import { METRIC_TYPE } from '../../public';
describe('store_report', () => {
test('stores report for all types of data', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
const report: ReportSchemaType = {
reportVersion: 1,
userAgent: {
'key-user-agent': {
key: 'test-key',
type: METRIC_TYPE.USER_AGENT,
appName: 'test-app-name',
userAgent: 'test-user-agent',
},
},
uiStatsMetrics: {
any: {
key: 'test-key',
type: METRIC_TYPE.CLICK,
appName: 'test-app-name',
eventName: 'test-event-name',
stats: {
min: 1,
max: 2,
avg: 1.5,
sum: 3,
},
},
},
application_usage: {
appId: {
numberOfClicks: 3,
minutesOnScreen: 10,
},
},
};
await storeReport(savedObjectClient, report);
expect(savedObjectClient.create).toHaveBeenCalledWith(
'ui-metric',
{ count: 1 },
{
id: 'key-user-agent:test-user-agent',
overwrite: true,
}
);
expect(savedObjectClient.incrementCounter).toHaveBeenCalledWith(
'ui-metric',
'test-app-name:test-event-name',
'count'
);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith([
{
type: 'application_usage_transactional',
attributes: {
numberOfClicks: 3,
minutesOnScreen: 10,
appId: 'appId',
timestamp: expect.any(Date),
},
},
]);
});
test('it should not fail if nothing to store', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
const report: ReportSchemaType = {
reportVersion: 1,
userAgent: void 0,
uiStatsMetrics: void 0,
application_usage: void 0,
};
await storeReport(savedObjectClient, report);
expect(savedObjectClient.bulkCreate).not.toHaveBeenCalled();
expect(savedObjectClient.incrementCounter).not.toHaveBeenCalled();
expect(savedObjectClient.create).not.toHaveBeenCalled();
expect(savedObjectClient.create).not.toHaveBeenCalled();
});
});

View file

@ -17,28 +17,50 @@
* under the License.
*/
import { ISavedObjectsRepository, SavedObject } from 'kibana/server';
import { ReportSchemaType } from './schema';
export async function storeReport(internalRepository: any, report: ReportSchemaType) {
export async function storeReport(
internalRepository: ISavedObjectsRepository,
report: ReportSchemaType
) {
const uiStatsMetrics = report.uiStatsMetrics ? Object.entries(report.uiStatsMetrics) : [];
const userAgents = report.userAgent ? Object.entries(report.userAgent) : [];
return Promise.all([
const appUsage = report.application_usage ? Object.entries(report.application_usage) : [];
const timestamp = new Date();
return Promise.all<{ saved_objects: Array<SavedObject<any>> }>([
...userAgents.map(async ([key, metric]) => {
const { userAgent } = metric;
const savedObjectId = `${key}:${userAgent}`;
return await internalRepository.create(
'ui-metric',
{ count: 1 },
{
id: savedObjectId,
overwrite: true,
}
);
return {
saved_objects: [
await internalRepository.create(
'ui-metric',
{ count: 1 },
{
id: savedObjectId,
overwrite: true,
}
),
],
};
}),
...uiStatsMetrics.map(async ([key, metric]) => {
const { appName, eventName } = metric;
const savedObjectId = `${appName}:${eventName}`;
return await internalRepository.incrementCounter('ui-metric', savedObjectId, 'count');
return {
saved_objects: [
await internalRepository.incrementCounter('ui-metric', savedObjectId, 'count'),
],
};
}),
appUsage.length
? internalRepository.bulkCreate(
appUsage.map(([appId, metric]) => ({
type: 'application_usage_transactional',
attributes: { ...metric, appId, timestamp },
}))
)
: { saved_objects: [] },
]);
}

View file

@ -17,9 +17,12 @@
* under the License.
*/
import { IRouter } from '../../../../../src/core/server';
import { IRouter, ISavedObjectsRepository } from 'kibana/server';
import { registerUiMetricRoute } from './report_metrics';
export function setupRoutes(router: IRouter, getLegacySavedObjects: any) {
registerUiMetricRoute(router, getLegacySavedObjects);
export function setupRoutes(
router: IRouter,
getSavedObjects: () => ISavedObjectsRepository | undefined
) {
registerUiMetricRoute(router, getSavedObjects);
}

View file

@ -18,10 +18,13 @@
*/
import { schema } from '@kbn/config-schema';
import { IRouter } from '../../../../../src/core/server';
import { IRouter, ISavedObjectsRepository } from 'kibana/server';
import { storeReport, reportSchema } from '../report';
export function registerUiMetricRoute(router: IRouter, getLegacySavedObjects: () => any) {
export function registerUiMetricRoute(
router: IRouter,
getSavedObjects: () => ISavedObjectsRepository | undefined
) {
router.post(
{
path: '/api/ui_metric/report',
@ -34,7 +37,10 @@ export function registerUiMetricRoute(router: IRouter, getLegacySavedObjects: ()
async (context, req, res) => {
const { report } = req.body;
try {
const internalRepository = getLegacySavedObjects();
const internalRepository = getSavedObjects();
if (!internalRepository) {
throw Error(`The saved objects client hasn't been initialised yet`);
}
await storeReport(internalRepository, report);
return res.ok({ body: { status: 'ok' } });
} catch (error) {