[Monitoring] Usage collection (#75878)

* First stab at some internal telemetry

* Add missing files

* mbCount telemetry

* Include more data

* Remove unused field

* This file isn't used

* Mock in tests

* Add schema

* Store schema

* Use sample cluster instead

* Fix telemetry schema

* Fix type issues

* Updates

* Fix schema and tests

* Add tests

* Add tests

* Go back to using an array

* Fix schema

* Add page view data

* Remove debug

* Handle loading scenario here

* Add delay tracking too

* Add clicks for setup mode

* Add clicks for setup mode

* Fix beats/apm page views

* Fix typings
This commit is contained in:
Chris Roberson 2020-09-25 10:19:53 -04:00 committed by GitHub
parent f2fc48dec8
commit a88c27258e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 1288 additions and 120 deletions

View file

@ -273,3 +273,10 @@ export const ALERT_ACTION_TYPE_EMAIL = '.email';
export const ALERT_ACTION_TYPE_LOG = '.server-log';
export const ALERT_EMAIL_SERVICES = ['gmail', 'hotmail', 'icloud', 'outlook365', 'ses', 'yahoo'];
/**
* The saved object type for various monitoring data
*/
export const SAVED_OBJECT_TELEMETRY = 'monitoring-telemetry';
export const TELEMETRY_METRIC_BUTTON_CLICK = 'btnclick__';

View file

@ -12,7 +12,8 @@
"triggers_actions_ui",
"alerts",
"actions",
"encryptedSavedObjects"
"encryptedSavedObjects",
"observability"
],
"optionalPlugins": ["infra", "telemetryCollectionManager", "usageCollection", "home", "cloud"],
"server": true,

View file

@ -26,6 +26,7 @@ export class AngularApp {
pluginInitializerContext,
externalConfig,
triggersActionsUi,
usageCollection,
kibanaLegacy,
} = deps;
const app: IModule = localAppModule(deps);
@ -42,6 +43,7 @@ export class AngularApp {
externalConfig,
kibanaLegacy,
triggersActionsUi,
usageCollection,
},
this.injector
);

View file

@ -15,8 +15,9 @@ import {
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import './page_loading.scss';
import { useTrackPageview } from '../../../../observability/public';
export function PageLoading() {
function PageLoadingUI() {
return (
<EuiPage style={{ height: 'calc(100vh - 50px)' }}>
<EuiPageBody>
@ -45,3 +46,18 @@ export function PageLoading() {
</EuiPage>
);
}
function PageLoadingTracking({ pageViewTitle }) {
const path = pageViewTitle.toLowerCase().replace(/-/g, '').replace(/\s+/g, '_');
useTrackPageview({ app: 'stack_monitoring', path });
useTrackPageview({ app: 'stack_monitoring', path, delay: 15000 });
return <PageLoadingUI />;
}
export function PageLoading({ pageViewTitle }) {
if (pageViewTitle) {
return <PageLoadingTracking pageViewTitle={pageViewTitle} />;
}
return <PageLoadingUI />;
}

View file

@ -8,6 +8,8 @@ import React from 'react';
import { EuiButton } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import './enter_button.scss';
import { METRIC_TYPE, useUiTracker } from '../../../../observability/public';
import { TELEMETRY_METRIC_BUTTON_CLICK } from '../../../common/constants';
export interface SetupModeEnterButtonProps {
enabled: boolean;
@ -18,6 +20,7 @@ export const SetupModeEnterButton: React.FC<SetupModeEnterButtonProps> = (
props: SetupModeEnterButtonProps
) => {
const [isLoading, setIsLoading] = React.useState(false);
const trackStat = useUiTracker({ app: 'stack_monitoring' });
if (!props.enabled) {
return null;
@ -26,6 +29,10 @@ export const SetupModeEnterButton: React.FC<SetupModeEnterButtonProps> = (
async function enterSetupMode() {
setIsLoading(true);
await props.toggleSetupMode(true);
trackStat({
metric: `${TELEMETRY_METRIC_BUTTON_CLICK}setupmode_enter`,
metricType: METRIC_TYPE.CLICK,
});
setIsLoading(false);
}

View file

@ -14,6 +14,7 @@ import { TriggersAndActionsUIPublicPluginSetup } from '../../triggers_actions_ui
import { TypeRegistry } from '../../triggers_actions_ui/public/application/type_registry';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { ActionTypeModel, AlertTypeModel } from '../../triggers_actions_ui/public/types';
import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
interface BreadcrumbItem {
['data-test-subj']?: string;
@ -59,13 +60,14 @@ export interface IShims {
) => Promise<any>;
isCloud: boolean;
triggersActionsUi: TriggersAndActionsUIPublicPluginSetup;
usageCollection: UsageCollectionSetup;
}
export class Legacy {
private static _shims: IShims;
public static init(
{ core, data, isCloud, triggersActionsUi }: MonitoringStartPluginDependencies,
{ core, data, isCloud, triggersActionsUi, usageCollection }: MonitoringStartPluginDependencies,
ngInjector: angular.auto.IInjectorService
) {
this._shims = {
@ -119,6 +121,7 @@ export class Legacy {
}),
isCloud,
triggersActionsUi,
usageCollection,
};
}

View file

@ -8,6 +8,7 @@ import React from 'react';
import { render } from 'react-dom';
import { get, includes } from 'lodash';
import { i18n } from '@kbn/i18n';
import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public';
import { Legacy } from '../legacy_shims';
import { ajaxErrorHandlersProvider } from './ajax_error_handler';
import { SetupModeEnterButton } from '../components/setup_mode/enter_button';
@ -179,8 +180,17 @@ export const setSetupModeMenuItem = () => {
const globalState = angularState.injector.get('globalState');
const enabled = !globalState.inSetupMode;
const services = {
usageCollection: Legacy.shims.usageCollection,
};
const I18nContext = Legacy.shims.I18nContext;
render(
<SetupModeEnterButton enabled={enabled} toggleSetupMode={toggleSetupMode} />,
<KibanaContextProvider services={services}>
<I18nContext>
<SetupModeEnterButton enabled={enabled} toggleSetupMode={toggleSetupMode} />
</I18nContext>
</KibanaContextProvider>,
document.getElementById('setupModeNav')
);
};

View file

@ -13,6 +13,7 @@ import {
Plugin,
PluginInitializerContext,
} from 'kibana/public';
import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
import {
FeatureCatalogueCategory,
HomePublicPluginSetup,
@ -28,6 +29,7 @@ interface MonitoringSetupPluginDependencies {
home?: HomePublicPluginSetup;
cloud?: { isCloudEnabled: boolean };
triggers_actions_ui: TriggersAndActionsUIPublicPluginSetup;
usageCollection: UsageCollectionSetup;
}
export class MonitoringPlugin
@ -93,6 +95,7 @@ export class MonitoringPlugin
pluginInitializerContext: this.initializerContext,
externalConfig: this.getExternalConfig(),
triggersActionsUi: plugins.triggers_actions_ui,
usageCollection: plugins.usageCollection,
};
pluginsStart.kibanaLegacy.loadFontAwesome();

View file

@ -9,6 +9,7 @@ import { NavigationPublicPluginStart as NavigationStart } from '../../../../src/
import { DataPublicPluginStart } from '../../../../src/plugins/data/public';
import { TriggersAndActionsUIPublicPluginSetup } from '../../triggers_actions_ui/public';
import { KibanaLegacyStart } from '../../../../src/plugins/kibana_legacy/public';
import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
export { MonitoringConfig } from '../server';
@ -23,4 +24,5 @@ export interface MonitoringStartPluginDependencies {
pluginInitializerContext: PluginInitializerContext;
externalConfig: Array<Array<string | number> | Array<string | boolean>>;
triggersActionsUi: TriggersAndActionsUIPublicPluginSetup;
usageCollection: UsageCollectionSetup;
}

View file

@ -44,6 +44,7 @@ uiRoutes.when('/apm/instances/:uuid', {
apm: 'APM server',
},
}),
telemetryPageViewTitle: 'apm_server_instance',
api: `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/apm/${$route.current.params.uuid}`,
defaultData: {},
reactNodeId: 'apmInstanceReact',
@ -63,21 +64,16 @@ uiRoutes.when('/apm/instances/:uuid', {
})
);
title($scope.cluster, `APM server - ${get(data, 'apmSummary.name')}`);
this.renderReact(data);
this.renderReact(
<ApmServerInstance
summary={data.apmSummary || {}}
metrics={data.metrics || {}}
onBrush={this.onBrush}
zoomInfo={this.zoomInfo}
/>
);
}
);
}
renderReact(data) {
const component = (
<ApmServerInstance
summary={data.apmSummary || {}}
metrics={data.metrics || {}}
onBrush={this.onBrush}
zoomInfo={this.zoomInfo}
/>
);
super.renderReact(component);
}
},
});

View file

@ -55,37 +55,33 @@ uiRoutes.when('/apm/instances', {
$scope.$watch(
() => this.data,
(data) => {
this.renderReact(data);
const { pagination, sorting, onTableChange } = this;
const component = (
<SetupModeRenderer
scope={this.scope}
injector={this.injector}
productName={APM_SYSTEM_ID}
render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
<Fragment>
{flyoutComponent}
<ApmServerInstances
setupMode={setupMode}
apms={{
pagination,
sorting,
onTableChange,
data,
}}
/>
{bottomBarComponent}
</Fragment>
)}
/>
);
this.renderReact(component);
}
);
}
renderReact(data) {
const { pagination, sorting, onTableChange } = this;
const component = (
<SetupModeRenderer
scope={this.scope}
injector={this.injector}
productName={APM_SYSTEM_ID}
render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
<Fragment>
{flyoutComponent}
<ApmServerInstances
setupMode={setupMode}
apms={{
pagination,
sorting,
onTableChange,
data,
}}
/>
{bottomBarComponent}
</Fragment>
)}
/>
);
super.renderReact(component);
}
},
});

View file

@ -47,14 +47,11 @@ uiRoutes.when('/apm', {
$scope.$watch(
() => this.data,
(data) => {
this.renderReact(data);
this.renderReact(
<ApmOverview {...data} onBrush={this.onBrush} zoomInfo={this.zoomInfo} />
);
}
);
}
renderReact(data) {
const component = <ApmOverview {...data} onBrush={this.onBrush} zoomInfo={this.zoomInfo} />;
super.renderReact(component);
}
},
});

View file

@ -13,6 +13,7 @@ import { Legacy } from '../legacy_shims';
import { PromiseWithCancel } from '../../common/cancel_promise';
import { SetupModeFeature } from '../../common/enums';
import { updateSetupModeData, isSetupModeFeatureEnabled } from '../lib/setup_mode';
import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public';
/**
* Given a timezone, this function will calculate the offset in milliseconds
@ -89,6 +90,7 @@ export class MonitoringViewBaseController {
options = {},
alerts = { shouldFetch: false, options: {} },
fetchDataImmediately = true,
telemetryPageViewTitle = '',
}) {
const titleService = $injector.get('title');
const $executor = $injector.get('$executor');
@ -102,6 +104,7 @@ export class MonitoringViewBaseController {
$scope.pageData = this.data = { ...defaultData };
this._isDataInitialized = false;
this.reactNodeId = reactNodeId;
this.telemetryPageViewTitle = telemetryPageViewTitle || title;
let deferTimer;
let zoomInLevel = 0;
@ -207,6 +210,8 @@ export class MonitoringViewBaseController {
deferTimer = setTimeout(() => addPopstateHandler(), 10);
};
// Render loading state
this.renderReact(null, true);
fetchDataImmediately && this.updateData();
});
@ -228,15 +233,26 @@ export class MonitoringViewBaseController {
this.setTitle = (title) => titleService($scope.cluster, title);
}
renderReact(component) {
renderReact(component, trackPageView = false) {
const renderElement = document.getElementById(this.reactNodeId);
if (!renderElement) {
console.warn(`"#${this.reactNodeId}" element has not been added to the DOM yet`);
return;
}
const services = {
usageCollection: Legacy.shims.usageCollection,
};
const I18nContext = Legacy.shims.I18nContext;
const wrappedComponent = (
<I18nContext>{!this._isDataInitialized ? <PageLoading /> : component}</I18nContext>
<KibanaContextProvider services={services}>
<I18nContext>
{!this._isDataInitialized ? (
<PageLoading pageViewTitle={trackPageView ? this.telemetryPageViewTitle : null} />
) : (
component
)}
</I18nContext>
</KibanaContextProvider>
);
render(wrappedComponent, renderElement);
}

View file

@ -47,6 +47,7 @@ uiRoutes.when('/beats/beat/:beatUuid', {
beatName: pageData.summary.name,
},
}),
telemetryPageViewTitle: 'beats_instance',
getPageData,
$scope,
$injector,

View file

@ -40,6 +40,7 @@ uiRoutes.when('/beats/beats', {
pageTitle: i18n.translate('xpack.monitoring.beats.listing.pageTitle', {
defaultMessage: 'Beats listing',
}),
telemetryPageViewTitle: 'beats_listing',
storageKey: 'beats.beats',
getPageData,
reactNodeId: 'monitoringBeatsInstancesApp',
@ -51,9 +52,6 @@ uiRoutes.when('/beats/beats', {
this.scope = $scope;
this.injector = $injector;
//Bypassing super.updateData, since this controller loads its own data
this._isDataInitialized = true;
$scope.$watch(
() => this.data,
() => this.renderComponent()

View file

@ -52,6 +52,7 @@ uiRoutes
$scope,
$injector,
reactNodeId: 'monitoringClusterListingApp',
telemetryPageViewTitle: 'cluster_listing',
});
const $route = $injector.get('$route');

View file

@ -54,6 +54,7 @@ uiRoutes.when('/overview', {
alerts: {
shouldFetch: true,
},
telemetryPageViewTitle: 'cluster_overview',
});
this.init = () => this.renderReact(null);

View file

@ -42,13 +42,12 @@ uiRoutes.when('/elasticsearch/ccr', {
$scope.$watch(
() => this.data,
(data) => {
this.renderReact(data);
if (!data) {
return;
}
this.renderReact(<Ccr data={data.data} />);
}
);
this.renderReact = ({ data }) => {
super.renderReact(<Ccr data={data} />);
};
}
},
});

View file

@ -48,6 +48,10 @@ uiRoutes.when('/elasticsearch/ccr/:index/shard/:shardId', {
$scope.$watch(
() => this.data,
(data) => {
if (!data) {
return;
}
this.setPageTitle(
i18n.translate('xpack.monitoring.elasticsearch.ccr.shard.pageTitle', {
defaultMessage: 'Elasticsearch Ccr Shard - Index: {followerIndex} Shard: {shardId}',
@ -57,13 +61,10 @@ uiRoutes.when('/elasticsearch/ccr/:index/shard/:shardId', {
},
})
);
this.renderReact(data);
this.renderReact(<CcrShard {...data} />);
}
);
this.renderReact = (props) => {
super.renderReact(<CcrShard {...props} />);
};
}
},
});

View file

@ -64,6 +64,7 @@ uiRoutes.when('/elasticsearch/indices/:index/advanced', {
indexName,
},
}),
telemetryPageViewTitle: 'elasticsearch_index_advanced',
defaultData: {},
getPageData,
reactNodeId: 'monitoringElasticsearchAdvancedIndexApp',

View file

@ -66,6 +66,7 @@ uiRoutes.when('/elasticsearch/indices/:index', {
indexName,
},
}),
telemetryPageViewTitle: 'elasticsearch_index',
pageTitle: i18n.translate('xpack.monitoring.elasticsearch.indices.overview.pageTitle', {
defaultMessage: 'Index: {indexName}',
values: {

View file

@ -67,23 +67,24 @@ uiRoutes.when('/elasticsearch/indices', {
$scope.$watch(
() => this.data,
(data) => {
this.renderReact(data);
if (!data) {
return;
}
const { clusterStatus, indices } = data;
this.renderReact(
<ElasticsearchIndices
clusterStatus={clusterStatus}
indices={indices}
showSystemIndices={showSystemIndices}
toggleShowSystemIndices={toggleShowSystemIndices}
sorting={this.sorting}
pagination={this.pagination}
onTableChange={this.onTableChange}
/>
);
}
);
this.renderReact = ({ clusterStatus, indices }) => {
super.renderReact(
<ElasticsearchIndices
clusterStatus={clusterStatus}
indices={indices}
showSystemIndices={showSystemIndices}
toggleShowSystemIndices={toggleShowSystemIndices}
sorting={this.sorting}
pagination={this.pagination}
onTableChange={this.onTableChange}
/>
);
};
}
},
});

View file

@ -61,6 +61,7 @@ uiRoutes.when('/elasticsearch/nodes/:node/advanced', {
defaultData: {},
getPageData,
reactNodeId: 'monitoringElasticsearchAdvancedNodeApp',
telemetryPageViewTitle: 'elasticsearch_node_advanced',
$scope,
$injector,
alerts: {

View file

@ -36,6 +36,13 @@ uiRoutes.when('/elasticsearch/nodes/:node', {
const nodeName = $route.current.params.node;
super({
title: i18n.translate('xpack.monitoring.elasticsearch.node.overview.routeTitle', {
defaultMessage: 'Elasticsearch - Nodes - {nodeName} - Overview',
values: {
nodeName,
},
}),
telemetryPageViewTitle: 'elasticsearch_node',
defaultData: {},
getPageData,
reactNodeId: 'monitoringElasticsearchNodeApp',

View file

@ -95,38 +95,41 @@ uiRoutes.when('/elasticsearch/nodes', {
$scope.$watch(
() => this.data,
() => this.renderReact(this.data || {})
(data) => {
if (!data) {
return;
}
const { clusterStatus, nodes, totalNodeCount } = data;
const pagination = {
...this.pagination,
totalItemCount: totalNodeCount,
};
this.renderReact(
<SetupModeRenderer
scope={$scope}
injector={$injector}
productName={ELASTICSEARCH_SYSTEM_ID}
render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
<Fragment>
{flyoutComponent}
<ElasticsearchNodes
clusterStatus={clusterStatus}
clusterUuid={globalState.cluster_uuid}
setupMode={setupMode}
nodes={nodes}
alerts={this.alerts}
showCgroupMetricsElasticsearch={showCgroupMetricsElasticsearch}
{...this.getPaginationTableProps(pagination)}
/>
{bottomBarComponent}
</Fragment>
)}
/>
);
}
);
this.renderReact = ({ clusterStatus, nodes, totalNodeCount }) => {
const pagination = {
...this.pagination,
totalItemCount: totalNodeCount,
};
super.renderReact(
<SetupModeRenderer
scope={$scope}
injector={$injector}
productName={ELASTICSEARCH_SYSTEM_ID}
render={({ setupMode, flyoutComponent, bottomBarComponent }) => (
<Fragment>
{flyoutComponent}
<ElasticsearchNodes
clusterStatus={clusterStatus}
clusterUuid={globalState.cluster_uuid}
setupMode={setupMode}
nodes={nodes}
alerts={this.alerts}
showCgroupMetricsElasticsearch={showCgroupMetricsElasticsearch}
{...this.getPaginationTableProps(pagination)}
/>
{bottomBarComponent}
</Fragment>
)}
/>
);
};
}
},
});

View file

@ -78,7 +78,7 @@ export class ElasticsearchOverviewController extends MonitoringViewBaseControlle
renderReact(data, cluster) {
// All data needs to originate in this view, and get passed as a prop to the components, for statelessness
const { clusterStatus, metrics, shardActivity, logs } = data;
const { clusterStatus, metrics, shardActivity, logs } = data || {};
const shardActivityData = shardActivity && this.filterShardActivityData(shardActivity); // no filter on data = null
const component = (
<ElasticsearchOverview

View file

@ -67,6 +67,7 @@ uiRoutes.when('/kibana/instances/:uuid', {
constructor($injector, $scope) {
super({
title: `Kibana - ${get($scope.pageData, 'kibanaSummary.name')}`,
telemetryPageViewTitle: 'kibana_instance',
defaultData: {},
getPageData,
reactNodeId: 'monitoringKibanaInstanceApp',
@ -86,7 +87,6 @@ uiRoutes.when('/kibana/instances/:uuid', {
if (!data || !data.metrics) {
return;
}
this.setTitle(`Kibana - ${get(data, 'kibanaSummary.name')}`);
this.setPageTitle(
i18n.translate('xpack.monitoring.kibana.instance.pageTitle', {

View file

@ -69,6 +69,7 @@ uiRoutes.when('/logstash/node/:uuid/advanced', {
reactNodeId: 'monitoringLogstashNodeAdvancedApp',
$scope,
$injector,
telemetryPageViewTitle: 'logstash_node_advanced',
});
$scope.$watch(

View file

@ -76,6 +76,7 @@ uiRoutes.when('/logstash/node/:uuid', {
alertTypeIds: [ALERT_LOGSTASH_VERSION_MISMATCH],
},
},
telemetryPageViewTitle: 'logstash_node',
});
$scope.$watch(

View file

@ -81,6 +81,7 @@ uiRoutes.when('/logstash/node/:uuid/pipelines', {
$scope,
$injector,
fetchDataImmediately: false, // We want to apply pagination before sending the first request
telemetryPageViewTitle: 'logstash_node_pipelines',
});
$scope.$watch(

View file

@ -0,0 +1,161 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { getMonitoringUsageCollector } from './get_usage_collector';
import { fetchClusters } from '../../lib/alerts/fetch_clusters';
jest.mock('../../lib/alerts/fetch_clusters', () => ({
fetchClusters: jest.fn().mockImplementation(() => {
return [
{
clusterUuid: '1abc',
clusterName: 'unitTesting',
},
];
}),
}));
jest.mock('./lib/get_stack_products_usage', () => ({
getStackProductsUsage: jest.fn().mockImplementation(() => {
return {
elasticsearch: {
count: 5,
enabled: true,
metricbeatUsed: true,
},
kibana: {
count: 2,
enabled: true,
metricbeatUsed: false,
},
logstash: {
count: 0,
enabled: false,
metricbeatUsed: false,
},
beats: {
count: 1,
enabled: true,
metricbeatUsed: false,
},
apm: {
count: 1,
enabled: true,
metricbeatUsed: true,
},
};
}),
}));
jest.mock('./lib/fetch_license_type', () => ({
fetchLicenseType: jest.fn().mockImplementation(() => {
return 'trial';
}),
}));
describe('getMonitoringUsageCollector', () => {
const callCluster = jest.fn();
const config: any = {
ui: {
ccs: {
enabled: true,
},
},
};
it('should be configured correctly', async () => {
const usageCollection: any = {
makeUsageCollector: jest.fn(),
};
await getMonitoringUsageCollector(usageCollection, config, callCluster);
const mock = (usageCollection.makeUsageCollector as jest.Mock).mock;
const args = mock.calls[0];
expect(args[0].type).toBe('monitoring');
expect(typeof args[0].isReady).toBe('function');
expect(args[0].schema).toStrictEqual({
hasMonitoringData: { type: 'boolean' },
clusters: {
license: { type: 'keyword' },
clusterUuid: { type: 'keyword' },
metricbeatUsed: { type: 'boolean' },
elasticsearch: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
kibana: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
logstash: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
beats: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
apm: {
enabled: { type: 'boolean' },
count: { type: 'long' },
metricbeatUsed: { type: 'boolean' },
},
},
});
});
it('should fetch usage data', async () => {
const usageCollection: any = {
makeUsageCollector: jest.fn(),
};
await getMonitoringUsageCollector(usageCollection, config, callCluster);
const mock = (usageCollection.makeUsageCollector as jest.Mock).mock;
const args = mock.calls[0];
const result = await args[0].fetch();
expect(result).toStrictEqual({
hasMonitoringData: true,
clusters: [
{
clusterUuid: '1abc',
license: 'trial',
elasticsearch: { count: 5, enabled: true, metricbeatUsed: true },
kibana: { count: 2, enabled: true, metricbeatUsed: false },
logstash: { count: 0, enabled: false, metricbeatUsed: false },
beats: { count: 1, enabled: true, metricbeatUsed: false },
apm: { count: 1, enabled: true, metricbeatUsed: true },
metricbeatUsed: true,
},
],
});
});
it('should handle no monitoring data', async () => {
const usageCollection: any = {
makeUsageCollector: jest.fn(),
};
await getMonitoringUsageCollector(usageCollection, config, callCluster);
const mock = (usageCollection.makeUsageCollector as jest.Mock).mock;
const args = mock.calls[0];
(fetchClusters as jest.Mock).mockImplementation(() => {
return [];
});
const result = await args[0].fetch();
expect(result).toStrictEqual({
hasMonitoringData: false,
clusters: [],
});
});
});

View file

@ -0,0 +1,128 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { LegacyAPICaller } from 'src/core/server';
import { MonitoringConfig } from '../../config';
import { fetchAvailableCcs } from '../../lib/alerts/fetch_available_ccs';
import { getStackProductsUsage } from './lib/get_stack_products_usage';
import { fetchLicenseType } from './lib/fetch_license_type';
import { MonitoringUsage, StackProductUsage, MonitoringClusterStackProductUsage } from './types';
import { INDEX_PATTERN_ELASTICSEARCH } from '../../../common/constants';
import { getCcsIndexPattern } from '../../lib/alerts/get_ccs_index_pattern';
import { fetchClusters } from '../../lib/alerts/fetch_clusters';
export function getMonitoringUsageCollector(
usageCollection: UsageCollectionSetup,
config: MonitoringConfig,
callCluster: LegacyAPICaller
) {
return usageCollection.makeUsageCollector<MonitoringUsage>({
type: 'monitoring',
isReady: () => true,
schema: {
hasMonitoringData: {
type: 'boolean',
},
clusters: {
license: {
type: 'keyword',
},
clusterUuid: {
type: 'keyword',
},
metricbeatUsed: {
type: 'boolean',
},
elasticsearch: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
kibana: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
logstash: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
beats: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
apm: {
enabled: {
type: 'boolean',
},
count: {
type: 'long',
},
metricbeatUsed: {
type: 'boolean',
},
},
},
},
fetch: async () => {
const usageClusters: MonitoringClusterStackProductUsage[] = [];
const availableCcs = config.ui.ccs.enabled ? await fetchAvailableCcs(callCluster) : [];
const elasticsearchIndex = getCcsIndexPattern(INDEX_PATTERN_ELASTICSEARCH, availableCcs);
const clusters = await fetchClusters(callCluster, elasticsearchIndex);
for (const cluster of clusters) {
const license = await fetchLicenseType(callCluster, availableCcs, cluster.clusterUuid);
const stackProducts = await getStackProductsUsage(
config,
callCluster,
availableCcs,
cluster.clusterUuid
);
usageClusters.push({
clusterUuid: cluster.clusterUuid,
license,
metricbeatUsed: Object.values(stackProducts).some(
(_usage: StackProductUsage) => _usage.metricbeatUsed
),
...stackProducts,
});
}
const usage = {
hasMonitoringData: usageClusters.length > 0,
clusters: usageClusters,
};
return usage;
},
});
}

View file

@ -4,15 +4,21 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { LegacyAPICaller } from 'src/core/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { getSettingsCollector } from './get_settings_collector';
import { getMonitoringUsageCollector } from './get_usage_collector';
import { MonitoringConfig } from '../../config';
export { KibanaSettingsCollector } from './get_settings_collector';
export function registerCollectors(
usageCollection: UsageCollectionSetup,
config: MonitoringConfig
config: MonitoringConfig,
callCluster: LegacyAPICaller
) {
usageCollection.registerCollector(getSettingsCollector(usageCollection, config));
usageCollection.registerCollector(
getMonitoringUsageCollector(usageCollection, config, callCluster)
);
}

View file

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { fetchESUsage } from './fetch_es_usage';
describe('fetchESUsage', () => {
const clusterUuid = '1abcde2';
const index = '.monitoring-es-*';
const callCluster = jest.fn().mockImplementation(() => ({
hits: {
hits: [
{
_source: {
cluster_stats: {
nodes: {
count: {
total: 10,
},
},
},
},
},
],
},
aggregations: {
indices: {
buckets: [
{
key: '.monitoring-es-2',
},
],
},
},
}));
const config: any = {};
it('should return usage data for Elasticsearch', async () => {
const result = await fetchESUsage(config, callCluster, clusterUuid, index);
expect(result).toStrictEqual({
count: 10,
enabled: true,
metricbeatUsed: false,
});
});
it('should handle some indices coming from Metricbeat', async () => {
const customCallCluster = jest.fn().mockImplementation(() => ({
hits: {
hits: [
{
_source: {
cluster_stats: {
nodes: {
count: {
total: 10,
},
},
},
},
},
],
},
aggregations: {
indices: {
buckets: [
{
key: '.monitoring-es-mb-2',
},
],
},
},
}));
const result = await fetchESUsage(config, customCallCluster, clusterUuid, index);
expect(result).toStrictEqual({
count: 10,
enabled: true,
metricbeatUsed: true,
});
});
it('should handle no monitoring data', async () => {
const customCallCluster = jest.fn().mockImplementation(() => ({
hits: {
hits: [],
},
}));
const result = await fetchESUsage(config, customCallCluster, clusterUuid, index);
expect(result).toStrictEqual({
count: 0,
enabled: false,
metricbeatUsed: false,
});
});
});

View file

@ -0,0 +1,122 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { LegacyAPICaller } from 'src/core/server';
import { get } from 'lodash';
import { MonitoringConfig } from '../../../config';
import { StackProductUsage } from '../types';
interface ESResponse {
hits: {
hits: ESResponseHits[];
};
aggregations: {
indices: {
buckets: ESIndicesBucket;
};
};
}
interface ESIndicesBucket {
key: string;
}
interface ESResponseHits {
_source: ClusterStats;
}
interface ClusterStats {
cluster_stats: {
nodes: {
count: {
total: number;
};
};
};
version: string;
}
export async function fetchESUsage(
config: MonitoringConfig,
callCluster: LegacyAPICaller,
clusterUuid: string,
index: string
): Promise<StackProductUsage> {
const params = {
index,
size: 1,
ignoreUnavailable: true,
filterPath: [
'hits.hits._source.cluster_stats.nodes.count.total',
'aggregations.indices.buckets',
],
body: {
sort: [
{
timestamp: {
order: 'desc',
},
},
],
query: {
bool: {
must: [
{
term: {
type: {
value: 'cluster_stats',
},
},
},
{
term: {
cluster_uuid: {
value: clusterUuid,
},
},
},
{
range: {
timestamp: {
gte: 'now-1h',
},
},
},
],
},
},
aggs: {
indices: {
terms: {
field: '_index',
size: 2,
},
},
},
},
};
const response = await callCluster('search', params);
const esResponse = response as ESResponse;
if (esResponse.hits.hits.length === 0) {
return {
count: 0,
enabled: false,
metricbeatUsed: false,
};
}
const hit = esResponse.hits.hits[0]._source;
const count = hit.cluster_stats.nodes.count.total;
const buckets = get(esResponse, 'aggregations.indices.buckets', []) as ESIndicesBucket[];
const metricbeatUsed = Boolean(buckets.find((indexBucket) => indexBucket.key.includes('-mb-')));
return {
count,
enabled: true,
metricbeatUsed,
};
}

View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { fetchLicenseType } from './fetch_license_type';
describe('fetchLicenseType', () => {
const clusterUuid = '1abcde2';
const availableCcs: string[] = [];
const callCluster = jest.fn().mockImplementation(() => ({
hits: {
hits: [
{
_source: {
license: {
type: 'trial',
},
},
},
],
},
}));
it('should get the license type', async () => {
const result = await fetchLicenseType(callCluster, availableCcs, clusterUuid);
expect(result).toStrictEqual('trial');
});
it('should handle no license data', async () => {
const customCallCluster = jest.fn().mockImplementation(() => ({
hits: {
hits: [],
},
}));
const result = await fetchLicenseType(customCallCluster, availableCcs, clusterUuid);
expect(result).toStrictEqual(null);
});
});

View file

@ -0,0 +1,57 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { get } from 'lodash';
import { LegacyAPICaller } from 'src/core/server';
import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../common/constants';
import { getCcsIndexPattern } from '../../../lib/alerts/get_ccs_index_pattern';
export async function fetchLicenseType(
callCluster: LegacyAPICaller,
availableCcs: string[],
clusterUuid: string
) {
let index = INDEX_PATTERN_ELASTICSEARCH;
if (availableCcs) {
index = getCcsIndexPattern(index, availableCcs);
}
const params = {
index,
filterPath: ['hits.hits._source.license'],
body: {
size: 1,
sort: [
{
timestamp: {
order: 'desc',
},
},
],
query: {
bool: {
must: [
{
term: {
cluster_uuid: {
value: clusterUuid,
},
},
},
{
term: {
type: {
value: 'cluster_stats',
},
},
},
],
},
},
},
};
const response = await callCluster('search', params);
return get(response, 'hits.hits[0]._source.license.type', null);
}

View file

@ -0,0 +1,117 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { fetchStackProductUsage } from './fetch_stack_product_usage';
describe('fetchStackProductUsage', () => {
const clusterUuid = '1abcde2';
const config: any = {
ui: {
max_bucket_size: 10000,
},
};
it('should use appropiate query parameters', async () => {
const callCluster = jest.fn();
await fetchStackProductUsage(
config,
callCluster,
clusterUuid,
'.monitoring-kibana-*',
'kibana_stats',
'kibana_stats.kibana.uuid',
[
{
term: {
type: {
value: 'foo',
},
},
},
]
);
const params = callCluster.mock.calls[0][1];
expect(params.body.query.bool.must[0].term.type.value).toBe('kibana_stats');
expect(params.body.query.bool.must[1].term.cluster_uuid.value).toBe(clusterUuid);
expect(params.body.query.bool.must[2].range.timestamp.gte).toBe('now-1h');
expect(params.body.query.bool.must[3].term.type.value).toBe('foo');
});
it('should get the usage data', async () => {
const callCluster = jest.fn().mockImplementation(() => ({
aggregations: {
uuids: {
buckets: [
{
key: 'sadfsdf',
indices: {
buckets: [
{
key: '.monitoring-kibana-8',
},
],
},
},
],
},
},
}));
const result = await fetchStackProductUsage(
config,
callCluster,
clusterUuid,
'.monitoring-kibana-*',
'kibana_stats',
'kibana_stats.kibana.uuid'
);
expect(result).toStrictEqual({
count: 1,
enabled: true,
metricbeatUsed: false,
});
});
it('should handle both collection types', async () => {
const callCluster = jest.fn().mockImplementation(() => ({
aggregations: {
uuids: {
buckets: [
{
key: 'sadfsdf',
indices: {
buckets: [
{
key: '.monitoring-kibana-8',
},
{
key: '.monitoring-kibana-mb-8',
},
],
},
},
],
},
},
}));
const result = await fetchStackProductUsage(
config,
callCluster,
clusterUuid,
'.monitoring-kibana-*',
'kibana_stats',
'kibana_stats.kibana.uuid'
);
expect(result).toStrictEqual({
count: 1,
enabled: true,
metricbeatUsed: true,
});
});
});

View file

@ -0,0 +1,111 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { get } from 'lodash';
import { LegacyAPICaller } from 'src/core/server';
import { MonitoringConfig } from '../../../config';
// @ts-ignore
import { prefixIndexPattern } from '../../../lib/ccs_utils';
import { StackProductUsage } from '../types';
interface ESResponse {
aggregations?: {
uuids: {
buckets: UuidBucket[];
};
};
}
interface UuidBucket {
key: string;
indices: {
buckets: KeyBucket[];
};
}
interface KeyBucket {
key: string;
}
export async function fetchStackProductUsage(
config: MonitoringConfig,
callCluster: LegacyAPICaller,
clusterUuid: string,
index: string,
type: string,
uuidPath: string,
filters: any[] = []
): Promise<StackProductUsage> {
const size = config.ui.max_bucket_size;
const params = {
index,
size: 0,
ignoreUnavailable: true,
filterPath: ['aggregations.uuids.buckets'],
body: {
query: {
bool: {
must: [
{
term: {
type: {
value: type,
},
},
},
{
term: {
cluster_uuid: {
value: clusterUuid,
},
},
},
{
range: {
timestamp: {
gte: 'now-1h',
},
},
},
...filters,
],
},
},
aggs: {
uuids: {
terms: {
field: uuidPath,
size,
},
aggs: {
indices: {
terms: {
field: '_index',
size: 2,
},
},
},
},
},
},
};
const response = (await callCluster('search', params)) as ESResponse;
const uuidBuckets = get(response, 'aggregations.uuids.buckets', []) as UuidBucket[];
const count = uuidBuckets.length;
const metricbeatUsed = Boolean(
uuidBuckets.find((uuidBucket) =>
(get(uuidBucket, 'indices.buckets', []) as KeyBucket[]).find((indexBucket) =>
indexBucket.key.includes('-mb-')
)
)
);
return {
count,
enabled: count > 0,
metricbeatUsed,
};
}

View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { getStackProductsUsage } from './get_stack_products_usage';
describe('getStackProductsUsage', () => {
const config: any = {
ui: {
max_bucket_size: 10000,
},
};
const clusterUuid = '1abcde2';
const availableCcs: string[] = [];
const callCluster = jest.fn().mockImplementation(() => ({
hits: {
hits: [],
},
}));
it('should get all stack products', async () => {
const result = await getStackProductsUsage(config, callCluster, availableCcs, clusterUuid);
expect(result.elasticsearch).toBeDefined();
expect(result.kibana).toBeDefined();
expect(result.logstash).toBeDefined();
expect(result.beats).toBeDefined();
expect(result.apm).toBeDefined();
});
});

View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { LegacyAPICaller } from 'src/core/server';
import { MonitoringClusterStackProductUsage } from '../types';
import { fetchESUsage } from './fetch_es_usage';
import { MonitoringConfig } from '../../../config';
// @ts-ignore
import { getIndexPatterns } from '../../../lib/cluster/get_index_patterns';
// @ts-ignore
import { prefixIndexPattern } from '../../../lib/ccs_utils';
import {
INDEX_PATTERN_ELASTICSEARCH,
INDEX_PATTERN_KIBANA,
INDEX_PATTERN_LOGSTASH,
INDEX_PATTERN_BEATS,
} from '../../../../common/constants';
import { fetchStackProductUsage } from './fetch_stack_product_usage';
import { getCcsIndexPattern } from '../../../lib/alerts/get_ccs_index_pattern';
export const getStackProductsUsage = async (
config: MonitoringConfig,
callCluster: LegacyAPICaller,
availableCcs: string[],
clusterUuid: string
): Promise<
Pick<
MonitoringClusterStackProductUsage,
'elasticsearch' | 'kibana' | 'logstash' | 'beats' | 'apm'
>
> => {
const elasticsearchIndex = getCcsIndexPattern(INDEX_PATTERN_ELASTICSEARCH, availableCcs);
const kibanaIndex = getCcsIndexPattern(INDEX_PATTERN_KIBANA, availableCcs);
const logstashIndex = getCcsIndexPattern(INDEX_PATTERN_LOGSTASH, availableCcs);
const beatsIndex = getCcsIndexPattern(INDEX_PATTERN_BEATS, availableCcs);
const [elasticsearch, kibana, logstash, beats, apm] = await Promise.all([
fetchESUsage(config, callCluster, clusterUuid, elasticsearchIndex),
fetchStackProductUsage(
config,
callCluster,
clusterUuid,
kibanaIndex,
'kibana_stats',
'kibana_stats.kibana.uuid'
),
fetchStackProductUsage(
config,
callCluster,
clusterUuid,
logstashIndex,
'logstash_stats',
'logstash_stats.logstash.uuid'
),
fetchStackProductUsage(
config,
callCluster,
clusterUuid,
beatsIndex,
'beats_stats',
'beats_stats.beat.uuid'
),
fetchStackProductUsage(
config,
callCluster,
clusterUuid,
beatsIndex,
'beats_stats',
'beats_stats.beat.uuid',
[{ term: { 'beats_stats.beat.type': 'apm-server' } }]
),
]);
return { elasticsearch, kibana, logstash, beats, apm };
};

View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export interface MonitoringUsage {
hasMonitoringData: boolean;
clusters: MonitoringClusterStackProductUsage[];
}
export interface MonitoringClusterStackProductUsage {
clusterUuid: string;
license: string;
metricbeatUsed: boolean;
elasticsearch: StackProductUsage;
logstash: StackProductUsage;
kibana: StackProductUsage;
beats: StackProductUsage;
apm: StackProductUsage;
}
export interface StackProductUsage {
count: number;
enabled: boolean;
metricbeatUsed: boolean;
}

View file

@ -4,6 +4,9 @@
* you may not use this file except in compliance with the Elastic License.
*/
export function getCcsIndexPattern(indexPattern: string, remotes: string[]): string {
if (remotes.length === 0) {
return indexPattern;
}
const patternsToAdd = [];
for (const index of indexPattern.split(',')) {
for (const remote of remotes) {

View file

@ -14,7 +14,9 @@ jest.mock('rxjs', () => ({
}));
jest.mock('./es_client/instantiate_client', () => ({
instantiateClient: jest.fn(),
instantiateClient: jest.fn().mockImplementation(() => ({
cluster: {},
})),
}));
jest.mock('./license_service', () => ({
@ -25,6 +27,10 @@ jest.mock('./license_service', () => ({
})),
}));
jest.mock('./kibana_monitoring/collectors', () => ({
registerCollectors: jest.fn(),
}));
describe('Monitoring plugin', () => {
const initializerContext = {
logger: {
@ -70,6 +76,9 @@ describe('Monitoring plugin', () => {
subscribe: jest.fn(),
},
},
savedObjects: {
registerType: jest.fn(),
},
};
const setupPlugins = {

View file

@ -28,6 +28,7 @@ import {
KIBANA_MONITORING_LOGGING_TAG,
KIBANA_STATS_TYPE_MONITORING,
ALERTS,
SAVED_OBJECT_TELEMETRY,
} from '../common/constants';
import { MonitoringConfig, createConfig, configSchema } from './config';
// @ts-ignore
@ -157,7 +158,20 @@ export class Plugin {
// Register collector objects for stats to show up in the APIs
if (plugins.usageCollection) {
registerCollectors(plugins.usageCollection, config);
core.savedObjects.registerType({
name: SAVED_OBJECT_TELEMETRY,
hidden: true,
namespaceType: 'agnostic',
mappings: {
properties: {
reportedClusterUuids: {
type: 'keyword',
},
},
},
});
registerCollectors(plugins.usageCollection, config, cluster.callAsInternalUser);
}
// Always create the bulk uploader

View file

@ -32,7 +32,10 @@ export type FetchData<T extends FetchDataResponse = FetchDataResponse> = (
export type HasData = () => Promise<boolean>;
export type ObservabilityFetchDataPlugins = Exclude<ObservabilityApp, 'observability'>;
export type ObservabilityFetchDataPlugins = Exclude<
ObservabilityApp,
'observability' | 'stack_monitoring'
>;
export interface DataHandler<
T extends ObservabilityFetchDataPlugins = ObservabilityFetchDataPlugins

View file

@ -4,7 +4,13 @@
* you may not use this file except in compliance with the Elastic License.
*/
export type ObservabilityApp = 'infra_metrics' | 'infra_logs' | 'apm' | 'uptime' | 'observability';
export type ObservabilityApp =
| 'infra_metrics'
| 'infra_logs'
| 'apm'
| 'uptime'
| 'observability'
| 'stack_monitoring';
export type PromiseReturnType<Func> = Func extends (...args: any[]) => Promise<infer Value>
? Value

View file

@ -540,6 +540,91 @@
}
}
},
"monitoring": {
"properties": {
"hasMonitoringData": {
"type": "boolean"
},
"clusters": {
"properties": {
"license": {
"type": "keyword"
},
"clusterUuid": {
"type": "keyword"
},
"metricbeatUsed": {
"type": "boolean"
},
"elasticsearch": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
},
"kibana": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
},
"logstash": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
},
"beats": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
},
"apm": {
"properties": {
"enabled": {
"type": "boolean"
},
"count": {
"type": "long"
},
"metricbeatUsed": {
"type": "boolean"
}
}
}
}
}
}
},
"rollups": {
"properties": {
"index_patterns": {