maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Uptime][Synthetics] fleet managed monitor telemetry (#97502)

Browse source 
* add fleet monitor telemetry

* add changes to telemetry json file

* add functional tests

* document telemetry fields

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Dominique Clarke 2021-05-04 07:48:43 -04:00 committed by GitHub
parent 6d22f6f552
commit 16b4f7176d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 1840 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -5405,13 +5405,22 @@
"monitor_name_stats": {
"properties": {
"avg_length": {
"type": "float"
"type": "float",
"_meta": {
"description": "This field represents the average length of monitor names"
}
},
"max_length": {
"type": "long"
"type": "long",
"_meta": {
"description": "This field represents the max length of monitor names"
}
},
"min_length": {
"type": "long"
"type": "long",
"_meta": {
"description": "This field represents the min length of monitor names"
}
}
}
},
@ -5419,21 +5428,36 @@
"type": "long"
},
"no_of_unique_monitors": {
"type": "long"
"type": "long",
"_meta": {
"description": "This field represents the number of unique configured monitors"
}
},
"no_of_unique_observer_locations": {
"type": "long"
"type": "long",
"_meta": {
"description": "This field represents the number of unique monitor observer locations"
}
},
"observer_location_name_stats": {
"properties": {
"avg_length": {
"type": "float"
"type": "float",
"_meta": {
"description": "This field represents the average length of monitor observer location names"
}
},
"max_length": {
"type": "long"
"type": "long",
"_meta": {
"description": "This field represents the max length of monitor observer location names"
}
},
"min_length": {
"type": "long"
"type": "long",
"_meta": {
"description": "This field represents the min length of monitor observer location names"
}
}
}
},
@ -5442,6 +5466,43 @@
},
"settings_page": {
"type": "long"
},
"fleet_monitor_name_stats": {
"properties": {
"avg_length": {
"type": "float",
"_meta": {
"description": "This field represents the average length of fleet managed monitor names"
}
},
"max_length": {
"type": "long",
"_meta": {
"description": "This field represents the max length of fleet managed monitor names"
}
},
"min_length": {
"type": "long",
"_meta": {
"description": "This field represents the min length of fleet managed monitor names"
}
}
}
},
"fleet_monitor_frequency": {
"type": "array",
"items": {
"type": "long",
"_meta": {
"description": "This field represents the average the monitor frequency of fleet managed monitors"
}
}
},
"fleet_no_of_unique_monitors": {
"type": "long",
"_meta": {
"description": "This field represents the number of unique configured fleet managed monitors"
}
}
}
}

14
x-pack/plugins/uptime/server/lib/adapters/telemetry/__snapshots__/kibana_telemetry_adapter.test.ts.snap generated
View file

@ -14,6 +14,13 @@ Object {
"dateRangeStart": Array [
"now-15",
],
"fleet_monitor_frequency": Array [],
"fleet_monitor_name_stats": Object {
"avg_length": 0,
"max_length": 0,
"min_length": 0,
},
"fleet_no_of_unique_monitors": 0,
"monitor_frequency": Array [],
"monitor_name_stats": Object {
"avg_length": 0,
@ -49,6 +56,13 @@ Object {
"dateRangeStart": Array [
"now-15",
],
"fleet_monitor_frequency": Array [],
"fleet_monitor_name_stats": Object {
"avg_length": 0,
"max_length": 0,
"min_length": 0,
},
"fleet_no_of_unique_monitors": 0,
"monitor_frequency": Array [],
"monitor_name_stats": Object {
"avg_length": 0,

4
x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.test.ts
View file

@ -27,7 +27,9 @@ describe('KibanaTelemetryAdapter', () => {
},
};
getSavedObjectsClient = () => {
return {};
return {
get: () => {},
};
};
collectorFetchContext = createCollectorFetchContextMock();
});

182
x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts
View file

@ -52,20 +52,104 @@ export class KibanaTelemetryAdapter {
dateRangeStart: { type: 'array', items: { type: 'date' } },
monitor_frequency: { type: 'array', items: { type: 'long' } },
monitor_name_stats: {
avg_length: { type: 'float' },
max_length: { type: 'long' },
min_length: { type: 'long' },
avg_length: {
type: 'float',
_meta: {
description: 'This field represents the average length of monitor names',
},
},
max_length: {
type: 'long',
_meta: {
description: 'This field represents the max length of monitor names',
},
},
min_length: {
type: 'long',
_meta: {
description: 'This field represents the min length of monitor names',
},
},
},
monitor_page: { type: 'long' },
no_of_unique_monitors: { type: 'long' },
no_of_unique_observer_locations: { type: 'long' },
no_of_unique_monitors: {
type: 'long',
_meta: {
description: 'This field represents the number of unique configured monitors',
},
},
no_of_unique_observer_locations: {
type: 'long',
_meta: {
description:
'This field represents the number of unique monitor observer locations',
},
},
observer_location_name_stats: {
avg_length: { type: 'float' },
max_length: { type: 'long' },
min_length: { type: 'long' },
avg_length: {
type: 'float',
_meta: {
description:
'This field represents the average length of monitor observer location names',
},
},
max_length: {
type: 'long',
_meta: {
description:
'This field represents the max length of monitor observer location names',
},
},
min_length: {
type: 'long',
_meta: {
description:
'This field represents the min length of monitor observer location names',
},
},
},
overview_page: { type: 'long' },
settings_page: { type: 'long' },
fleet_monitor_name_stats: {
avg_length: {
type: 'float',
_meta: {
description:
'This field represents the average length of fleet managed monitor names',
},
},
max_length: {
type: 'long',
_meta: {
description:
'This field represents the max length of fleet managed monitor names',
},
},
min_length: {
type: 'long',
_meta: {
description:
'This field represents the min length of fleet managed monitor names',
},
},
},
fleet_monitor_frequency: {
type: 'array',
items: {
type: 'long',
_meta: {
description:
'This field represents the average the monitor frequency of fleet managed monitors',
},
},
},
fleet_no_of_unique_monitors: {
type: 'long',
_meta: {
description:
'This field represents the number of unique configured fleet managed monitors',
},
},
},
},
},
@ -74,6 +158,7 @@ export class KibanaTelemetryAdapter {
if (savedObjectsClient) {
const uptimeEsClient = createUptimeESClient({ esClient, savedObjectsClient });
await this.countNoOfUniqueMonitorAndLocations(uptimeEsClient, savedObjectsClient);
await this.countNoOfUniqueFleetManagedMonitors(uptimeEsClient);
}
const report = this.getReport();
return { last_24_hours: { hits: { ...report } } };
@ -218,6 +303,79 @@ export class KibanaTelemetryAdapter {
return bucket;
}
public static async countNoOfUniqueFleetManagedMonitors(callCluster: UptimeESClient) {
const params = {
index: 'synthetics-*',
body: {
query: {
bool: {
must: [
{
range: {
'@timestamp': {
gte: 'now-1d/d',
lt: 'now',
},
},
},
{
term: {
'monitor.fleet_managed': true,
},
},
],
},
},
size: 0,
aggs: {
unique_monitors: {
cardinality: {
field: 'monitor.id',
},
},
monitor_name: {
string_stats: {
field: 'monitor.name',
},
},
monitors: {
terms: {
field: 'monitor.id',
size: 1000,
},
aggs: {
docs: {
top_hits: {
size: 1,
_source: ['monitor.timespan'],
},
},
},
},
},
},
};
const { body: result } = await callCluster.search(params);
const numberOfUniqueMonitors: number = result?.aggregations?.unique_monitors?.value ?? 0;
const monitorNameStats = result?.aggregations?.monitor_name;
const uniqueMonitors: any = result?.aggregations?.monitors.buckets;
const bucketId = this.getBucketToIncrement();
const bucket = this.collector[bucketId];
bucket.fleet_no_of_unique_monitors = numberOfUniqueMonitors;
bucket.fleet_monitor_name_stats = {
min_length: monitorNameStats?.min_length ?? 0,
max_length: monitorNameStats?.max_length ?? 0,
avg_length: +(monitorNameStats?.avg_length?.toFixed(2) ?? 0),
};
bucket.fleet_monitor_frequency = this.getMonitorsFrequency(uniqueMonitors);
return bucket;
}
private static getMonitorsFrequency(uniqueMonitors = []) {
const frequencies: number[] = [];
uniqueMonitors
@ -285,6 +443,14 @@ export class KibanaTelemetryAdapter {
dateRangeEnd: [],
autoRefreshEnabled: false,
autorefreshInterval: [],
fleet_no_of_unique_monitors: 0,
fleet_monitor_frequency: [],
fleet_monitor_name_stats: {
min_length: 0,
max_length: 0,
avg_length: 0,
},
};
}
return bucketId;

4
x-pack/plugins/uptime/server/lib/adapters/telemetry/types.ts
View file

@ -40,4 +40,8 @@ export interface UptimeTelemetry {
dateRangeEnd: string[];
autorefreshInterval: number[];
autoRefreshEnabled: boolean;
fleet_no_of_unique_monitors: number;
fleet_monitor_frequency: number[];
fleet_monitor_name_stats: Stats;
}

1
x-pack/plugins/uptime/server/rest_api/telemetry/log_page_view.ts
View file

@ -34,6 +34,7 @@ export const createLogPageViewRoute: UMRestApiRouteFactory = () => ({
uptimeEsClient,
savedObjectsClient
);
await KibanaTelemetryAdapter.countNoOfUniqueFleetManagedMonitors(uptimeEsClient);
return KibanaTelemetryAdapter.countPageView(pageView as PageViewParams);
},
});

31
x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts
View file

@ -18,6 +18,7 @@ interface CheckProps {
mogrify?: (doc: any) => any;
refresh?: boolean;
tls?: boolean | TlsProps;
isFleetManaged?: boolean;
}
const getRandomMonitorId = () => {
@ -31,6 +32,7 @@ export const makeCheck = async ({
mogrify = (d) => d,
refresh = true,
tls = false,
isFleetManaged = false,
}: CheckProps): Promise<{ monitorId: string; docs: any }> => {
const cgFields = {
monitor: {
@ -52,7 +54,15 @@ export const makeCheck = async ({
if (i === numIps - 1) {
pingFields.summary = summary;
}
const doc = await makePing(es, monitorId, pingFields, mogrify, false, tls as any);
const doc = await makePing(
es,
monitorId,
pingFields,
mogrify,
false,
tls as any,
isFleetManaged
);
docs.push(doc);
// @ts-ignore
summary[doc.monitor.status]++;
@ -73,7 +83,8 @@ export const makeChecks = async (
every: number = 10000, // number of millis between checks
fields: { [key: string]: any } = {},
mogrify: (doc: any) => any = (d) => d,
refresh: boolean = true
refresh: boolean = true,
isFleetManaged: boolean = false
) => {
const checks = [];
const oldestTime = new Date().getTime() - numChecks * every;
@ -90,7 +101,15 @@ export const makeChecks = async (
},
},
});
const { docs } = await makeCheck({ es, monitorId, numIps, fields, mogrify, refresh: false });
const { docs } = await makeCheck({
es,
monitorId,
numIps,
fields,
mogrify,
refresh: false,
isFleetManaged,
});
checks.push(docs);
}
@ -110,7 +129,8 @@ export const makeChecksWithStatus = async (
fields: { [key: string]: any } = {},
status: 'up' | 'down',
mogrify: (doc: any) => any = (d) => d,
refresh: boolean = true
refresh: boolean = true,
isFleetManaged: boolean = false
) => {
const oppositeStatus = status === 'up' ? 'down' : 'up';
@ -130,7 +150,8 @@ export const makeChecksWithStatus = async (
return mogrify(d);
},
refresh
refresh,
isFleetManaged
);
};

8
x-pack/test/api_integration/apis/uptime/rest/helper/make_ping.ts
View file

@ -9,7 +9,8 @@ import uuid from 'uuid';
import { merge } from 'lodash';
import { makeTls, TlsProps } from './make_tls';
const INDEX_NAME = 'heartbeat-8-generated-test';
const DEFAULT_INDEX_NAME = 'heartbeat-8-generated-test';
const DATA_STREAM_INDEX_NAME = 'synthetics-http-default';
export const makePing = async (
es: any,
@ -17,7 +18,8 @@ export const makePing = async (
fields: { [key: string]: any },
mogrify: (doc: any) => any,
refresh: boolean = true,
tls: boolean | TlsProps = false
tls: boolean | TlsProps = false,
isFleetManaged: boolean | undefined = false
) => {
const timestamp = new Date();
const baseDoc: any = {
@ -115,7 +117,7 @@ export const makePing = async (
const doc = mogrify(merge(baseDoc, fields));
await es.index({
index: INDEX_NAME,
index: isFleetManaged ? DATA_STREAM_INDEX_NAME : DEFAULT_INDEX_NAME,
refresh,
body: doc,
});

11
x-pack/test/api_integration/apis/uptime/rest/index.ts
View file

@ -38,14 +38,21 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
});
describe('with generated data', () => {
beforeEach('load heartbeat data', async () => await esArchiver.loadIfNeeded('uptime/blank'));
after('unload', async () => await esArchiver.unload('uptime/blank'));
beforeEach('load heartbeat data', async () => {
await esArchiver.loadIfNeeded('uptime/blank');
await esArchiver.loadIfNeeded('uptime/blank_data_stream');
});
after('unload', async () => {
await esArchiver.unload('uptime/blank');
await esArchiver.unload('uptime/blank_data_stream');
});
loadTestFile(require.resolve('./certs'));
loadTestFile(require.resolve('./dynamic_settings'));
loadTestFile(require.resolve('./snapshot'));
loadTestFile(require.resolve('./monitor_states_generated'));
loadTestFile(require.resolve('./telemetry_collectors'));
loadTestFile(require.resolve('./telemetry_collectors_fleet'));
});
describe('with real-world data', () => {

13
x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts
View file

@ -14,7 +14,7 @@ export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const es = getService('legacyEs');
describe('telemetry collectors', () => {
describe('telemetry collectors heartbeat', () => {
before('generating data', async () => {
await getService('esArchiver').load('uptime/blank');
@ -82,7 +82,9 @@ export default function ({ getService }: FtrProviderContext) {
await es.indices.refresh();
});
after('unload heartbeat index', () => getService('esArchiver').unload('uptime/blank'));
after('unload heartbeat index', () => {
getService('esArchiver').unload('uptime/blank');
});
beforeEach(async () => {
await es.indices.refresh();
@ -116,6 +118,13 @@ export default function ({ getService }: FtrProviderContext) {
dateRangeEnd: ['now/d'],
autoRefreshEnabled: true,
autorefreshInterval: [100],
fleet_monitor_frequency: [],
fleet_monitor_name_stats: {
avg_length: 0,
max_length: 0,
min_length: 0,
},
fleet_no_of_unique_monitors: 0,
});
});

191
x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors_fleet.ts Normal file
View file

@ -0,0 +1,191 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { API_URLS } from '../../../../../plugins/uptime/common/constants';
import { makeChecksWithStatus } from './helper/make_checks';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const es = getService('legacyEs');
const client = getService('es');
describe('telemetry collectors fleet', () => {
before('generating data', async () => {
await getService('esArchiver').load('uptime/blank_data_stream');
const observer = {
geo: {
name: 'US-East',
location: '40.7128, -74.0060',
},
};
const observer2 = {
geo: {
name: 'US',
location: '40.7128, -74.0060',
},
};
await makeChecksWithStatus(
es,
'upMonitorId',
1,
1,
60 * 1000,
{
observer: {},
monitor: {
name: 'Elastic',
fleet_managed: true,
},
},
'up',
undefined,
undefined,
true
);
await makeChecksWithStatus(
es,
'downMonitorId',
1,
1,
120 * 1000,
{
observer,
monitor: {
name: 'Long Name with 22 Char',
fleet_managed: true,
},
},
'down',
undefined,
undefined,
true
);
await makeChecksWithStatus(
es,
'noGeoNameMonitor',
1,
1,
60 * 1000,
{
observer: {},
monitor: {
fleet_managed: true,
},
},
'down',
undefined,
undefined,
true
);
await makeChecksWithStatus(
es,
'downMonitorId',
1,
1,
1,
{
observer,
monitor: {
name: 'Elastic',
fleet_managed: true,
},
},
'down',
undefined,
undefined,
true
);
await makeChecksWithStatus(
es,
'mixMonitorId',
1,
1,
1,
{ observer: observer2, monitor: { fleet_managed: true } },
'down',
undefined,
undefined,
true
);
await es.indices.refresh();
});
after('unload heartbeat index', () => {
getService('esArchiver').unload('uptime/blank_data_stream');
/**
* Data streams aren't included in the javascript elasticsearch client in kibana yet so we
* need to do raw requests here. Delete a data stream is slightly different than that of a regular index which
* is why we're using _data_stream here.
*/
client.transport.request({
method: 'DELETE',
path: `_data_stream/synthetics-http-default`,
});
});
beforeEach(async () => {
await es.indices.refresh();
});
it('should receive expected results for fleet managed monitors after calling monitor logging', async () => {
// call monitor page
const { body: result } = await supertest
.post(API_URLS.LOG_PAGE_VIEW)
.set('kbn-xsrf', 'true')
.send({
page: 'Monitor',
autorefreshInterval: 100,
dateStart: 'now/d',
dateEnd: 'now/d',
autoRefreshEnabled: true,
refreshTelemetryHistory: true,
})
.expect(200);
expect(result).to.eql({
overview_page: 0,
monitor_page: 1,
no_of_unique_monitors: 4,
settings_page: 0,
monitor_frequency: [120, 0.001, 60, 60],
monitor_name_stats: { min_length: 7, max_length: 22, avg_length: 12 },
no_of_unique_observer_locations: 3,
observer_location_name_stats: { min_length: 2, max_length: 7, avg_length: 4.8 },
dateRangeStart: ['now/d'],
dateRangeEnd: ['now/d'],
autoRefreshEnabled: true,
autorefreshInterval: [100],
fleet_monitor_frequency: [120, 0.001, 60, 60],
fleet_monitor_name_stats: { min_length: 7, max_length: 22, avg_length: 12 },
fleet_no_of_unique_monitors: 4,
});
});
it('should receive 200 status after overview logging', async () => {
// call overview page
await supertest
.post(API_URLS.LOG_PAGE_VIEW)
.set('kbn-xsrf', 'true')
.send({
page: 'Overview',
autorefreshInterval: 60,
dateStart: 'now/d',
dateEnd: 'now-30',
autoRefreshEnabled: true,
})
.expect(200);
});
});
}

0
x-pack/test/functional/es_archives/uptime/blank_data_stream/data.json Normal file
View file

1333
x-pack/test/functional/es_archives/uptime/blank_data_stream/mappings.json Normal file
View file

File diff suppressed because it is too large Load diff