maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Telemetry] Have Telemetry automatically get all the Kibana usage stats (#22336)

Browse source 
* Have Telemetry automatically get all the Kibana usage stats

* simplify

* remove reporting module

* remove unused helper function

* fix tests

* fix integration tests

* --wip-- [skip ci]

* getKibanaStats flattens nested xpack plugin stats into a single level

* fix integration test
This commit is contained in:
Tim Sullivan 2018-10-22 11:28:02 -07:00 committed by GitHub
parent a239ca2d17
commit c4cb8203b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 173 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_kibana_stats.js
View file

@ -46,7 +46,8 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 0 },
graph_workspace: { total: 1 },
timelion_sheet: { total: 1 },
indices: 1
indices: 1,
plugins: {}
}
};
@ -83,12 +84,57 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 1 },
graph_workspace: { total: 1 },
timelion_sheet: { total: 1 },
indices: 1
indices: 1,
plugins: {}
}
};
expect(getUsageStats(rawStats)).to.eql(expected);
});
it('flattens x-pack stats', () => {
const rawStats = {
hits: {
hits: [{
_source: {
cluster_uuid: 'clusterone',
kibana_stats: {
kibana: { version: '7.0.0-alpha1-test02' },
usage: {
dashboard: { total: 1 },
visualization: { total: 3 },
search: { total: 1 },
index_pattern: { total: 1 },
graph_workspace: { total: 1 },
timelion_sheet: { total: 1 },
index: '.kibana-test-01',
foo: { total: 5 },
xpack: {
fancy: {
available: true,
total: 15
}
}
}
}
}
}]
}
};
expect(getUsageStats(rawStats)).to.eql({
clusterone: {
dashboard: { total: 1 },
visualization: { total: 3 },
search: { total: 1 },
index_pattern: { total: 1 },
graph_workspace: { total: 1 },
timelion_sheet: { total: 1 },
indices: 1,
plugins: { foo: { total: 5 }, fancy: { available: true, total: 15 } }
}
});
});
});
describe('separate indices', () => {
@ -154,7 +200,8 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 1 },
graph_workspace: { total: 2 },
timelion_sheet: { total: 2 },
indices: 2
indices: 2,
plugins: {}
}
};
@ -223,7 +270,8 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 4 },
graph_workspace: { total: 2 },
timelion_sheet: { total: 2 },
indices: 2
indices: 2,
plugins: {}
}
};
@ -312,7 +360,8 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 4 },
graph_workspace: { total: 6 },
timelion_sheet: { total: 8 },
indices: 2
indices: 2,
plugins: {}
},
clustertwo: {
dashboard: { total: 300 },
@ -321,7 +370,8 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 300 },
graph_workspace: { total: 3 },
timelion_sheet: { total: 4 },
indices: 1
indices: 1,
plugins: {}
}
};
@ -353,7 +403,10 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 3 },
indices: 2,
search: { total: 1 },
visualization: { total: 7 }
visualization: { total: 7 },
plugins: {
foo: { available: true }
}
}
};
@ -365,7 +418,10 @@ describe('Get Kibana Stats', () => {
indices: 2,
search: { total: 1 },
versions: [ { count: 2, version: '7.0.0-alpha1-test12' } ],
visualization: { total: 7 }
visualization: { total: 7 },
plugins: {
foo: { available: true }
}
}
});
});
@ -387,14 +443,20 @@ describe('Get Kibana Stats', () => {
index_pattern: { total: 3 },
indices: 2,
search: { total: 1 },
visualization: { total: 7 }
visualization: { total: 7 },
plugins: {
bar: { available: false }
}
},
clustertwo: {
dashboard: { total: 3 },
index_pattern: { total: 5 },
indices: 1,
search: { total: 3 },
visualization: { total: 15 }
visualization: { total: 15 },
plugins: {
bear: { enabled: true }
}
}
};
@ -406,7 +468,10 @@ describe('Get Kibana Stats', () => {
indices: 2,
search: { total: 1 },
versions: [ { count: 2, version: '7.0.0-alpha1-test13' } ],
visualization: { total: 7 }
visualization: { total: 7 },
plugins: {
bar: { available: false }
}
},
clustertwo: {
count: 1,
@ -415,7 +480,10 @@ describe('Get Kibana Stats', () => {
indices: 1,
search: { total: 3 },
versions: [ { count: 1, version: '7.0.0-alpha1-test14' } ],
visualization: { total: 15 }
visualization: { total: 15 },
plugins: {
bear: { enabled: true }
}
}
});
});

20
x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_all_stats.js
View file

@ -8,13 +8,11 @@ import { get, set, merge } from 'lodash';
import {
KIBANA_SYSTEM_ID,
LOGSTASH_SYSTEM_ID,
REPORTING_SYSTEM_ID,
BEATS_SYSTEM_ID,
} from '../../../../common/constants';
import { getClusterUuids } from './get_cluster_uuids';
import { getElasticsearchStats } from './get_es_stats';
import { getKibanaStats } from './get_kibana_stats';
import { getReportingStats } from './get_reporting_stats';
import { getBeatsStats } from './get_beats_stats';
import { getHighLevelStats } from './get_high_level_stats';
@ -69,10 +67,9 @@ function getAllStatsWithCaller(server, callCluster, start, end) {
getElasticsearchStats(server, callCluster, clusterUuids), // cluster_stats, stack_stats.xpack, cluster_name/uuid, license, version
getKibanaStats(server, callCluster, clusterUuids, start, end), // stack_stats.kibana
getHighLevelStats(server, callCluster, clusterUuids, start, end, LOGSTASH_SYSTEM_ID), // stack_stats.logstash
getReportingStats(server, callCluster, clusterUuids, start, end), // stack_stats.xpack.reporting
getBeatsStats(server, callCluster, clusterUuids, start, end), // stack_stats.beats
])
.then(([esClusters, kibana, logstash, reporting, beats]) => handleAllStats(esClusters, { kibana, logstash, reporting, beats }));
.then(([esClusters, kibana, logstash, beats]) => handleAllStats(esClusters, { kibana, logstash, beats }));
});
}
@ -85,13 +82,12 @@ function getAllStatsWithCaller(server, callCluster, start, end) {
* @param {Object} logstash The Logstash nodes keyed by Cluster UUID
* @return {Array} The clusters joined with the Kibana and Logstash instances under each cluster's {@code stack_stats}.
*/
export function handleAllStats(clusters, { kibana, logstash, reporting, beats }) {
export function handleAllStats(clusters, { kibana, logstash, beats }) {
return clusters.map(cluster => {
// if they are using Kibana or Logstash, then add it to the cluster details under cluster.stack_stats
addStackStats(cluster, kibana, KIBANA_SYSTEM_ID);
addStackStats(cluster, logstash, LOGSTASH_SYSTEM_ID);
addStackStats(cluster, beats, BEATS_SYSTEM_ID);
addXPackStats(cluster, reporting, REPORTING_SYSTEM_ID);
mergeXPackStats(cluster, kibana, 'graph_workspace', 'graph'); // copy graph_workspace info out of kibana, merge it into stack_stats.xpack.graph
return cluster;
@ -118,18 +114,6 @@ export function addStackStats(cluster, allProductStats, product) {
}
}
export function addXPackStats(cluster, allProductStats, product) {
const productStats = get(allProductStats, cluster.cluster_uuid);
if (productStats) {
if (!get(cluster, 'stack_stats.xpack')) {
set(cluster, 'stack_stats.xpack', {});
}
set(cluster, `stack_stats.xpack[${product}]`, productStats);
}
}
export function mergeXPackStats(cluster, allProductStats, path, product) {
const productStats = get(allProductStats, cluster.cluster_uuid + '.' + path);

78
x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_kibana_stats.js
View file

@ -4,66 +4,76 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { get, isEmpty } from 'lodash';
import { get, isEmpty, omit } from 'lodash';
import { KIBANA_SYSTEM_ID } from '../../../../common/constants';
import { fetchHighLevelStats, handleHighLevelStatsResponse } from './get_high_level_stats';
export function rollUpTotals(rolledUp, addOn, field) {
return { total: rolledUp[field].total + addOn[field].total };
const rolledUpTotal = get(rolledUp, [field, 'total'], 0);
const addOnTotal = get(addOn, [field, 'total'], 0);
return { total: rolledUpTotal + addOnTotal };
}
export function rollUpIndices(rolledUp) {
return rolledUp.indices + 1;
}
/*
* @param {Object} rawStats
*/
export function getUsageStats(rawStats) {
const clusterIndexCache = [];
const clusterIndexCache = new Set();
const rawStatsHits = get(rawStats, 'hits.hits', []);
const usageStatsByCluster = rawStatsHits.reduce((accum, currInstance) => {
// get usage stats per cluster / .kibana index
return rawStatsHits.reduce((accum, currInstance) => {
const clusterUuid = get(currInstance, '_source.cluster_uuid');
const currUsage = get(currInstance, '_source.kibana_stats.usage', {});
const clusterIndexCombination = clusterUuid + currUsage.index;
// add usage data to the result if this cluster/index has not been processed yet
if (isEmpty(currUsage) || clusterIndexCache.includes(clusterIndexCombination)) {
// return early if usage data is empty or if this cluster/index has already been processed
if (isEmpty(currUsage) || clusterIndexCache.has(clusterIndexCombination)) {
return accum;
}
clusterIndexCache.add(clusterIndexCombination);
const rolledUpStats = get(accum, clusterUuid);
if (rolledUpStats) {
// this cluster has been seen, but this index hasn't
// process the usage stats for the unique index of this cluster
return {
...accum,
[clusterUuid]: {
dashboard: rollUpTotals(rolledUpStats, currUsage, 'dashboard'),
visualization: rollUpTotals(rolledUpStats, currUsage, 'visualization'),
search: rollUpTotals(rolledUpStats, currUsage, 'search'),
index_pattern: rollUpTotals(rolledUpStats, currUsage, 'index_pattern'),
graph_workspace: rollUpTotals(rolledUpStats, currUsage, 'graph_workspace'),
timelion_sheet: rollUpTotals(rolledUpStats, currUsage, 'timelion_sheet'),
indices: ++rolledUpStats.indices
}
};
}
// Get the stats that were read from any number of different .kibana indices in the cluster,
// roll them up into cluster-wide totals
const rolledUpStats = get(accum, clusterUuid, { indices: 0 });
const stats = {
dashboard: rollUpTotals(rolledUpStats, currUsage, 'dashboard'),
visualization: rollUpTotals(rolledUpStats, currUsage, 'visualization'),
search: rollUpTotals(rolledUpStats, currUsage, 'search'),
index_pattern: rollUpTotals(rolledUpStats, currUsage, 'index_pattern'),
graph_workspace: rollUpTotals(rolledUpStats, currUsage, 'graph_workspace'),
timelion_sheet: rollUpTotals(rolledUpStats, currUsage, 'timelion_sheet'),
indices: rollUpIndices(rolledUpStats)
};
clusterIndexCache.push(clusterIndexCombination);
// Get the stats provided by telemetry collectors.
const pluginsNested = omit(currUsage, [
'index',
'dashboard',
'visualization',
'search',
'index_pattern',
'graph_workspace',
'timelion_sheet',
]);
// Stats filtered by telemetry collectors need to be flattened since they're pulled in a generic way.
// A plugin might not provide flat stats if it implements formatForBulkUpload in its collector.
// e.g: we want `xpack.reporting` to just be `reporting`
const top = omit(pluginsNested, 'xpack');
const plugins = { ...top, ...pluginsNested.xpack };
// add the usage stats for the unique cluster
return {
...accum,
[clusterUuid]: {
dashboard: currUsage.dashboard,
visualization: currUsage.visualization,
search: currUsage.search,
index_pattern: currUsage.index_pattern,
graph_workspace: currUsage.graph_workspace,
timelion_sheet: currUsage.timelion_sheet,
indices: 1
...stats,
plugins
}
};
}, {});
return usageStatsByCluster;
}
export function combineStats(highLevelStats, usageStats = {}) {

61
x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_reporting_stats.js
View file

@ -1,61 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { get } from 'lodash';
import { createQuery } from './create_query';
import { KIBANA_SYSTEM_ID, REPORTING_SYSTEM_ID } from '../../../../common/constants';
const reportingStatsPath = `${KIBANA_SYSTEM_ID}_stats.usage.xpack.${REPORTING_SYSTEM_ID}`;
export function fetchHighLevelReportingStats(server, callCluster, clusterUuids, start, end) {
const config = server.config();
const params = {
index: config.get(`xpack.monitoring.${KIBANA_SYSTEM_ID}.index_pattern`),
size: config.get('xpack.monitoring.max_bucket_size'),
ignoreUnavailable: true,
body: {
query: createQuery({
start,
end,
type: `${KIBANA_SYSTEM_ID}_stats`, // reporting stats are in kibana_stats.xpack.reporting
filters: [ { terms: { cluster_uuid: clusterUuids } } ]
}),
collapse: {
field: 'cluster_uuid'
},
sort: [
{ 'timestamp': 'desc' }
],
_source: {
includes: [
'cluster_uuid',
reportingStatsPath,
]
},
}
};
return callCluster('search', params);
}
export function handleHighLevelReportingStatsResponse(response) {
const hits = get(response, 'hits.hits', []);
return hits.reduce((accum, curr) => {
const clusterUuid = get(curr, '_source.cluster_uuid');
const stats = get(curr, `_source.${reportingStatsPath}`);
return {
...accum,
[clusterUuid]: stats
};
}, {});
}
export async function getReportingStats(server, callCluster, clusterUuids, start, end) {
const rawStats = await fetchHighLevelReportingStats(server, callCluster, clusterUuids, start, end, REPORTING_SYSTEM_ID);
const stats = handleHighLevelReportingStatsResponse(rawStats);
return stats;
}

60
x-pack/test/api_integration/apis/xpack_main/telemetry/fixtures/basiccluster.json
View file

@ -201,6 +201,37 @@
],
"visualization": {
"total": 0
},
"plugins": {
"reporting": {
"available": true,
"enabled": false,
"csv": {
"available": true
},
"printable_pdf": {
"available": false
},
"status": {},
"lastDay": {
"csv": {
"available": true
},
"printable_pdf": {
"available": false
},
"status": {}
},
"last7Days": {
"csv": {
"available": true
},
"printable_pdf": {
"available": false
},
"status": {}
}
}
}
},
"xpack": {
@ -249,35 +280,6 @@
"local": 1
}
},
"reporting": {
"available": true,
"csv": {
"available": true
},
"enabled": false,
"last7Days": {
"csv": {
"available": true
},
"printable_pdf": {
"available": false
},
"status": {}
},
"lastDay": {
"csv": {
"available": true
},
"printable_pdf": {
"available": false
},
"status": {}
},
"printable_pdf": {
"available": false
},
"status": {}
},
"rollup": {
"available": true,
"enabled": true

14
x-pack/test/api_integration/apis/xpack_main/telemetry/fixtures/multicluster.json
View file

@ -743,6 +743,12 @@
"index_pattern": {
"total": 0
},
"graph_workspace": {
"total": 0
},
"timelion_sheet": {
"total": 0
},
"indices": 1,
"os": {
"distros": [],
@ -761,7 +767,8 @@
],
"visualization": {
"total": 0
}
},
"plugins": {}
},
"logstash": {
"count": 1,
@ -781,7 +788,10 @@
"xpack": {
"graph": {
"available": true,
"enabled": true
"enabled": true,
"graph_workspace": {
"total": 0
}
},
"logstash": {
"available": true,

8
x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry.js
View file

@ -5,8 +5,8 @@
*/
import expect from 'expect.js';
import multiclusterFixture from './fixtures/multicluster';
import basicclusterFixture from './fixtures/basiccluster';
import multiClusterFixture from './fixtures/multicluster';
import basicClusterFixture from './fixtures/basiccluster';
export default function ({ getService }) {
const supertest = getService('supertest');
@ -27,7 +27,7 @@ export default function ({ getService }) {
.set('kbn-xsrf', 'xxx')
.send({ timeRange })
.expect(200);
expect(body).to.eql(multiclusterFixture);
expect(body).to.eql(multiClusterFixture);
await esArchiver.unload(archive);
});
@ -47,7 +47,7 @@ export default function ({ getService }) {
.set('kbn-xsrf', 'xxx')
.send({ timeRange })
.expect(200);
expect(body).to.eql(basicclusterFixture);
expect(body).to.eql(basicClusterFixture);
await esArchiver.unload(archive);
});