[Telemetry] move x-pack/monitoring collector classes to src/server/usage (#20248)

* [Monitoring/Telemetry] Move Usage service from Monitoring to Kibana core

* fix tests

* fix reporting integration

* roll back more diffs

* roll logger into bulk uploader to remove file duplication

* fix xpack usage api

* subclass constructor is not needed

* collectorSet has factory methods for collector object creation

* fix reporting usage jest test
This commit is contained in:
Tim Sullivan 2018-07-09 09:32:02 -07:00 committed by GitHub
parent d4d5fe1368
commit c043788c97
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 237 additions and 169 deletions

View file

@ -28,6 +28,7 @@ import httpMixin from './http';
import { loggingMixin } from './logging';
import warningsMixin from './warnings';
import { statusMixin } from './status';
import { usageMixin } from './usage';
import pidMixin from './pid';
import { configDeprecationWarningsMixin } from './config/deprecation_warnings';
import configCompleteMixin from './config/complete';
@ -64,6 +65,7 @@ export default class KbnServer {
configDeprecationWarningsMixin,
warningsMixin,
statusMixin,
usageMixin,
// writes pid file
pidMixin,

View file

@ -1,7 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { noop } from 'lodash';
@ -48,7 +61,7 @@ describe('CollectorSet', () => {
const calls = server.log.getCalls();
expect(calls.length).to.be(1);
expect(calls[0].args).to.eql([
['debug', 'monitoring-ui', 'kibana-monitoring'],
['debug', 'stats-collection'],
'Fetching data from MY_TEST_COLLECTOR collector',
]);
expect(result).to.eql([{

View file

@ -1,7 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { getCollectorLogger } from '../lib';

View file

@ -1,7 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Promise from 'bluebird';
@ -25,6 +38,13 @@ export class CollectorSet {
constructor(server) {
this._log = getCollectorLogger(server);
this._collectors = [];
/*
* Helper Factory methods
* Define as instance properties to allow enclosing the server object
*/
this.makeStatsCollector = options => new Collector(server, options);
this.makeUsageCollector = options => new UsageCollector(server, options);
}
/*

View file

@ -0,0 +1,22 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { CollectorSet } from './collector_set';
export { Collector } from './collector';
export { UsageCollector } from './usage_collector';

View file

@ -0,0 +1,22 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { Collector } from './collector';
export class UsageCollector extends Collector {}

31
src/server/usage/index.js Normal file
View file

@ -0,0 +1,31 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { CollectorSet } from './classes';
export function usageMixin(kbnServer, server) {
const collectorSet = new CollectorSet(server);
/*
* expose the collector set object on the server
* provides factory methods for feature owners to create their own collector objects
* use collectorSet.register(collector) to register your feature's collector object(s)
*/
server.decorate('server', 'usage', { collectorSet });
}

View file

@ -0,0 +1,31 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
const LOGGING_TAGS = ['stats-collection'];
/*
* @param {Object} server
* @return {Object} helpful logger object
*/
export function getCollectorLogger(server) {
return {
debug: message => server.log(['debug', ...LOGGING_TAGS], message),
info: message => server.log(['info', ...LOGGING_TAGS], message),
warn: message => server.log(['warning', ...LOGGING_TAGS], message)
};
}

View file

@ -0,0 +1,20 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export { getCollectorLogger } from './get_collector_logger';

View file

@ -8,7 +8,6 @@ import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG, } from './common/constants'
import { requireUIRoutes } from './server/routes';
import { instantiateClient } from './server/es_client/instantiate_client';
import { initMonitoringXpackInfo } from './server/init_monitoring_xpack_info';
import { CollectorSet } from './server/kibana_monitoring/classes';
import { initBulkUploader } from './server/kibana_monitoring';
import {
getKibanaUsageCollector,
@ -28,9 +27,7 @@ import {
*/
export const init = (monitoringPlugin, server) => {
const config = server.config();
const collectorSet = new CollectorSet(server);
server.expose('collectorSet', collectorSet); // expose the collectorSet service
const { collectorSet } = server.usage;
/*
* Register collector objects for stats to show up in the APIs
*/
@ -77,7 +74,7 @@ export const init = (monitoringPlugin, server) => {
} else if (!kibanaCollectionEnabled) {
server.log(
['info', LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG],
'Internal collection for Kibana monitoring will is disabled per configuration.'
'Internal collection for Kibana monitoring is disabled per configuration.'
);
}

View file

@ -7,12 +7,20 @@
import { noop } from 'lodash';
import sinon from 'sinon';
import expect from 'expect.js';
import { Collector, CollectorSet } from '../classes';
import { BulkUploader } from '../bulk_uploader';
const FETCH_INTERVAL = 300;
const CHECK_DELAY = 500;
class MockCollectorSet {
constructor(_mockServer, mockCollectors) {
this.mockCollectors = mockCollectors;
}
async bulkFetch() {
return this.mockCollectors.map(({ fetch }) => fetch());
}
}
describe('BulkUploader', () => {
describe('registers a collector set and runs lifecycle events', () => {
let server;
@ -31,17 +39,17 @@ describe('BulkUploader', () => {
}),
},
},
usage: {},
};
});
it('should skip bulk upload if payload is empty', done => {
const collectors = new CollectorSet(server);
collectors.register(
new Collector(server, {
const collectors = new MockCollectorSet(server, [
{
type: 'type_collector_test',
fetch: noop, // empty payloads
})
);
}
]);
const uploader = new BulkUploader(server, {
interval: FETCH_INTERVAL,
@ -61,10 +69,6 @@ describe('BulkUploader', () => {
'Starting monitoring stats collection',
]);
expect(loggingCalls[1].args).to.eql([
['debug', 'monitoring-ui', 'kibana-monitoring'],
'Fetching data from type_collector_test collector',
]);
expect(loggingCalls[2].args).to.eql([
['debug', 'monitoring-ui', 'kibana-monitoring'],
'Skipping bulk uploading of an empty stats payload',
]);
@ -82,13 +86,9 @@ describe('BulkUploader', () => {
return [data[0][0], { ...data[0][1], combined: true }];
});
const collectors = new CollectorSet(server);
collectors.register(
new Collector(server, {
type: 'type_collector_test',
fetch: () => ({ testData: 12345 }),
})
);
const collectors = new MockCollectorSet(server, [
{ fetch: () => ({ type: 'type_collector_test', result: { testData: 12345 } }) }
]);
const uploader = new BulkUploader(server, {
interval: FETCH_INTERVAL,
combineTypes,
@ -107,10 +107,6 @@ describe('BulkUploader', () => {
'Starting monitoring stats collection',
]);
expect(loggingCalls[1].args).to.eql([
['debug', 'monitoring-ui', 'kibana-monitoring'],
'Fetching data from type_collector_test collector',
]);
expect(loggingCalls[2].args).to.eql([
['debug', 'monitoring-ui', 'kibana-monitoring'],
'Uploading bulk stats payload to the local cluster',
]);

View file

@ -6,12 +6,14 @@
import { isEmpty, flatten } from 'lodash';
import { callClusterFactory } from '../../../xpack_main';
import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG } from '../../common/constants';
import {
getCollectorLogger,
sendBulkPayload,
monitoringBulk,
} from './lib';
const LOGGING_TAGS = [LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG];
/*
* Handles internal Kibana stats collection and uploading data to Monitoring
* bulk endpoint.
@ -40,7 +42,11 @@ export class BulkUploader {
this._timer = null;
this._interval = interval;
this._combineTypes = combineTypes;
this._log = getCollectorLogger(server);
this._log = {
debug: message => server.log(['debug', ...LOGGING_TAGS], message),
info: message => server.log(['info', ...LOGGING_TAGS], message),
warn: message => server.log(['warning', ...LOGGING_TAGS], message)
};
this._client = server.plugins.elasticsearch.getCluster('admin').createClient({
plugins: [monitoringBulk],

View file

@ -1,9 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { CollectorSet } from './collector_set';
export { Collector } from './collector';
export { UsageCollector } from './usage_collector';

View file

@ -1,13 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Collector } from './collector';
export class UsageCollector extends Collector {
constructor(server, properties) {
super(server, properties);
}
}

View file

@ -1,70 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from 'expect.js';
import sinon from 'sinon';
import { getKibanaUsageCollector } from '../get_kibana_usage_collector';
import { callClusterFactory } from '../../../../../xpack_main';
describe('getKibanaUsageCollector', () => {
let clusterStub;
let serverStub;
let callClusterStub;
beforeEach(() => {
clusterStub = { callWithInternalUser: sinon.stub().returns(Promise.resolve({})) };
serverStub = {
plugins: {
elasticsearch: {
getCluster: sinon.stub()
}
},
config: () => ({ get: sinon.stub() }),
log: sinon.stub(),
};
serverStub.plugins.elasticsearch.getCluster.withArgs('admin').returns(clusterStub);
callClusterStub = callClusterFactory(serverStub).getCallClusterInternal();
});
it('correctly defines usage collector.', () => {
const usageCollector = getKibanaUsageCollector(serverStub);
expect(usageCollector.type).to.be('kibana');
expect(usageCollector.fetch).to.be.a(Function);
});
it('calls callWithInternalUser with the `search` method', async () => {
callClusterStub.returns({
aggregations: {
types: {
buckets: []
}
}
});
const usageCollector = getKibanaUsageCollector(serverStub);
await usageCollector.fetch(callClusterStub);
sinon.assert.calledOnce(clusterStub.callWithInternalUser);
sinon.assert.calledWithExactly(clusterStub.callWithInternalUser, 'search', sinon.match({
body: {
query: {
terms: {
type: sinon.match.array
},
},
aggs: {
types: {
terms: {
field: 'type',
size: sinon.match.number
}
}
}
}
}));
});
});

View file

@ -6,7 +6,6 @@
import { get, snakeCase } from 'lodash';
import { KIBANA_USAGE_TYPE } from '../../../common/constants';
import { UsageCollector } from '../classes';
const TYPES = [
'dashboard',
@ -21,7 +20,8 @@ const TYPES = [
* Fetches saved object client counts by querying the saved object index
*/
export function getKibanaUsageCollector(server) {
return new UsageCollector(server, {
const { collectorSet } = server.usage;
return collectorSet.makeUsageCollector({
type: KIBANA_USAGE_TYPE,
async fetch(callCluster) {
const index = server.config().get('kibana.index');

View file

@ -6,7 +6,6 @@
import { KIBANA_STATS_TYPE } from '../../../common/constants';
import { opsBuffer } from './ops_buffer';
import { Collector } from '../classes';
/*
* Initialize a collector for Kibana Ops Stats
@ -43,7 +42,8 @@ export function getOpsStatsCollector(server) {
}, 5 * 1000); // wait 5 seconds to avoid race condition with reloading logging configuration
});
return new Collector(server, {
const { collectorSet } = server.usage;
return collectorSet.makeStatsCollector({
type: KIBANA_STATS_TYPE,
init: start,
fetch: buffer.flush

View file

@ -7,7 +7,6 @@
import { get } from 'lodash';
import { XPACK_DEFAULT_ADMIN_EMAIL_UI_SETTING } from '../../../../../server/lib/constants';
import { KIBANA_SETTINGS_TYPE } from '../../../common/constants';
import { Collector } from '../classes';
/*
* Check if Cluster Alert email notifications is enabled in config
@ -57,7 +56,8 @@ export async function checkForEmailValue(
export function getSettingsCollector(server) {
const config = server.config();
return new Collector(server, {
const { collectorSet } = server.usage;
return collectorSet.makeStatsCollector({
type: KIBANA_SETTINGS_TYPE,
async fetch(callCluster) {
let kibanaSettingsData;

View file

@ -1,21 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG } from '../../../common/constants';
const LOGGING_TAGS = [LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG];
/*
* @param {Object} server
* @return {Object} helpful logger object
*/
export function getCollectorLogger(server) {
return {
debug: message => server.log(['debug', ...LOGGING_TAGS], message),
info: message => server.log(['info', ...LOGGING_TAGS], message),
warn: message => server.log(['warning', ...LOGGING_TAGS], message)
};
}

View file

@ -4,7 +4,6 @@
* you may not use this file except in compliance with the Elastic License.
*/
export { getCollectorLogger } from './get_collector_logger';
export { getCollectorTypesCombiner } from './get_collector_types_combiner';
export { sendBulkPayload } from './send_bulk_payload';
export { monitoringBulk } from './monitoring_bulk';

View file

@ -146,7 +146,7 @@ export const reporting = (kibana) => {
validateConfig(config, message => server.log(['reporting', 'warning'], message));
logConfiguration(config, message => server.log(['reporting', 'debug'], message));
const { xpack_main: xpackMainPlugin, monitoring: monitoringPlugin } = server.plugins;
const { xpack_main: xpackMainPlugin } = server.plugins;
mirrorPluginStatus(xpackMainPlugin, this);
const checkLicense = checkLicenseFactory(exportTypesRegistry);
@ -156,12 +156,8 @@ export const reporting = (kibana) => {
xpackMainPlugin.info.feature(this.id).registerLicenseCheckResultsGenerator(checkLicense);
});
// Register a function to with Monitoring to manage the collection of usage stats
monitoringPlugin && monitoringPlugin.status.once('green', () => {
if (monitoringPlugin.collectorSet) {
monitoringPlugin.collectorSet.register(getReportingUsageCollector(server));
}
});
// Register a function with server to manage the collection of usage stats
server.usage.collectorSet.register(getReportingUsageCollector(server));
server.expose('browserDriverFactory', await createBrowserDriverFactory(server));
server.expose('queue', createQueueFactory(server));

View file

@ -8,7 +8,6 @@ import { uniq } from 'lodash';
import { getExportTypesHandler } from './get_export_type_handler';
import { getReportCountsByParameter } from './get_reporting_type_counts';
import { KIBANA_REPORTING_TYPE } from '../../common/constants';
import { UsageCollector } from '../../../monitoring/server/kibana_monitoring/classes';
/**
* @typedef {Object} ReportingUsageStats Almost all of these stats are optional.
@ -115,7 +114,8 @@ async function getReportingUsageWithinRange(callCluster, server, reportingAvaila
* @return {Object} kibana usage stats type collection object
*/
export function getReportingUsageCollector(server) {
return new UsageCollector(server, {
const { collectorSet } = server.usage;
return collectorSet.makeUsageCollector({
type: KIBANA_REPORTING_TYPE,
fetch: async callCluster => {
const xpackInfo = server.plugins.xpack_main.info;

View file

@ -7,6 +7,12 @@ import sinon from 'sinon';
import { getReportingUsageCollector } from './get_reporting_usage_collector';
function getServerMock(customization) {
class MockUsageCollector {
constructor(_server, { fetch }) {
this.fetch = fetch;
}
}
const getLicenseCheckResults = sinon.stub().returns({});
const defaultServerMock = {
plugins: {
@ -37,7 +43,14 @@ function getServerMock(customization) {
return '.reporting-index';
}
}
})
}),
usage: {
collectorSet: {
makeUsageCollector: options => {
return new MockUsageCollector(this, options);
}
}
}
};
return Object.assign(defaultServerMock, customization);
}

View file

@ -16,7 +16,7 @@ const getClusterUuid = async callCluster => {
* @throws {Error} if the Monitoring CollectorSet is not ready
*/
const getUsage = (callCluster, server) => {
const { collectorSet } = server.plugins.monitoring;
const { collectorSet } = server.usage;
return collectorSet.bulkFetchUsage(callCluster);
};