[Metrics UI] Add overrides to Snapshot API to support alert previews (#68125)
* [Metrics UI] Add overrides to Snapshot API to support alert previews * Renaming tests from infra to metrics_ui; renaming waffle to snapshot; adding tests for overrides * removing limit and afterKey; removing extra interval * Setting the minimum value for lookbackSize
This commit is contained in:
parent
0279b18a46
commit
0b8e0c054f
|
@ -6,6 +6,7 @@
|
||||||
|
|
||||||
import * as rt from 'io-ts';
|
import * as rt from 'io-ts';
|
||||||
import { SnapshotMetricTypeRT, ItemTypeRT } from '../inventory_models/types';
|
import { SnapshotMetricTypeRT, ItemTypeRT } from '../inventory_models/types';
|
||||||
|
import { metricsExplorerSeriesRT } from './metrics_explorer';
|
||||||
|
|
||||||
export const SnapshotNodePathRT = rt.intersection([
|
export const SnapshotNodePathRT = rt.intersection([
|
||||||
rt.type({
|
rt.type({
|
||||||
|
@ -21,6 +22,7 @@ const SnapshotNodeMetricOptionalRT = rt.partial({
|
||||||
value: rt.union([rt.number, rt.null]),
|
value: rt.union([rt.number, rt.null]),
|
||||||
avg: rt.union([rt.number, rt.null]),
|
avg: rt.union([rt.number, rt.null]),
|
||||||
max: rt.union([rt.number, rt.null]),
|
max: rt.union([rt.number, rt.null]),
|
||||||
|
timeseries: metricsExplorerSeriesRT,
|
||||||
});
|
});
|
||||||
|
|
||||||
const SnapshotNodeMetricRequiredRT = rt.type({
|
const SnapshotNodeMetricRequiredRT = rt.type({
|
||||||
|
@ -41,11 +43,18 @@ export const SnapshotNodeResponseRT = rt.type({
|
||||||
interval: rt.string,
|
interval: rt.string,
|
||||||
});
|
});
|
||||||
|
|
||||||
export const InfraTimerangeInputRT = rt.type({
|
export const InfraTimerangeInputRT = rt.intersection([
|
||||||
interval: rt.string,
|
rt.type({
|
||||||
to: rt.number,
|
interval: rt.string,
|
||||||
from: rt.number,
|
to: rt.number,
|
||||||
});
|
from: rt.number,
|
||||||
|
}),
|
||||||
|
rt.partial({
|
||||||
|
lookbackSize: rt.number,
|
||||||
|
ignoreLookback: rt.boolean,
|
||||||
|
forceInterval: rt.boolean,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
export const SnapshotGroupByRT = rt.array(
|
export const SnapshotGroupByRT = rt.array(
|
||||||
rt.partial({
|
rt.partial({
|
||||||
|
@ -97,6 +106,7 @@ export const SnapshotRequestRT = rt.intersection([
|
||||||
accountId: rt.string,
|
accountId: rt.string,
|
||||||
region: rt.string,
|
region: rt.string,
|
||||||
filterQuery: rt.union([rt.string, rt.null]),
|
filterQuery: rt.union([rt.string, rt.null]),
|
||||||
|
includeTimeseries: rt.boolean,
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
|
@ -14,6 +14,8 @@ import {
|
||||||
SnapshotNodeResponseRT,
|
SnapshotNodeResponseRT,
|
||||||
SnapshotNodeResponse,
|
SnapshotNodeResponse,
|
||||||
SnapshotGroupBy,
|
SnapshotGroupBy,
|
||||||
|
SnapshotRequest,
|
||||||
|
InfraTimerangeInput,
|
||||||
} from '../../../../../common/http_api/snapshot_api';
|
} from '../../../../../common/http_api/snapshot_api';
|
||||||
import {
|
import {
|
||||||
InventoryItemType,
|
InventoryItemType,
|
||||||
|
@ -37,10 +39,11 @@ export function useSnapshot(
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
const timerange = {
|
const timerange: InfraTimerangeInput = {
|
||||||
interval: '1m',
|
interval: '1m',
|
||||||
to: currentTime,
|
to: currentTime,
|
||||||
from: currentTime - 360 * 1000,
|
from: currentTime - 360 * 1000,
|
||||||
|
lookbackSize: 20,
|
||||||
};
|
};
|
||||||
|
|
||||||
const { error, loading, response, makeRequest } = useHTTPRequest<SnapshotNodeResponse>(
|
const { error, loading, response, makeRequest } = useHTTPRequest<SnapshotNodeResponse>(
|
||||||
|
@ -55,7 +58,8 @@ export function useSnapshot(
|
||||||
sourceId,
|
sourceId,
|
||||||
accountId,
|
accountId,
|
||||||
region,
|
region,
|
||||||
}),
|
includeTimeseries: true,
|
||||||
|
} as SnapshotRequest),
|
||||||
decodeResponse
|
decodeResponse
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
|
@ -12,30 +12,48 @@ import { SnapshotModel, SnapshotModelMetricAggRT } from '../../../common/invento
|
||||||
import { getDatasetForField } from '../../routes/metrics_explorer/lib/get_dataset_for_field';
|
import { getDatasetForField } from '../../routes/metrics_explorer/lib/get_dataset_for_field';
|
||||||
import { InfraTimerangeInput } from '../../../common/http_api/snapshot_api';
|
import { InfraTimerangeInput } from '../../../common/http_api/snapshot_api';
|
||||||
import { ESSearchClient } from '.';
|
import { ESSearchClient } from '.';
|
||||||
|
import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds';
|
||||||
|
|
||||||
export const createTimeRangeWithInterval = async (
|
const createInterval = async (client: ESSearchClient, options: InfraSnapshotRequestOptions) => {
|
||||||
client: ESSearchClient,
|
const { timerange } = options;
|
||||||
options: InfraSnapshotRequestOptions
|
if (timerange.forceInterval && timerange.interval) {
|
||||||
): Promise<InfraTimerangeInput> => {
|
return getIntervalInSeconds(timerange.interval);
|
||||||
|
}
|
||||||
const aggregations = getMetricsAggregations(options);
|
const aggregations = getMetricsAggregations(options);
|
||||||
const modules = await aggregationsToModules(client, aggregations, options);
|
const modules = await aggregationsToModules(client, aggregations, options);
|
||||||
const interval = Math.max(
|
return Math.max(
|
||||||
(await calculateMetricInterval(
|
(await calculateMetricInterval(
|
||||||
client,
|
client,
|
||||||
{
|
{
|
||||||
indexPattern: options.sourceConfiguration.metricAlias,
|
indexPattern: options.sourceConfiguration.metricAlias,
|
||||||
timestampField: options.sourceConfiguration.fields.timestamp,
|
timestampField: options.sourceConfiguration.fields.timestamp,
|
||||||
timerange: { from: options.timerange.from, to: options.timerange.to },
|
timerange: { from: timerange.from, to: timerange.to },
|
||||||
},
|
},
|
||||||
modules,
|
modules,
|
||||||
options.nodeType
|
options.nodeType
|
||||||
)) || 60,
|
)) || 60,
|
||||||
60
|
60
|
||||||
);
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const createTimeRangeWithInterval = async (
|
||||||
|
client: ESSearchClient,
|
||||||
|
options: InfraSnapshotRequestOptions
|
||||||
|
): Promise<InfraTimerangeInput> => {
|
||||||
|
const { timerange } = options;
|
||||||
|
const calculatedInterval = await createInterval(client, options);
|
||||||
|
if (timerange.ignoreLookback) {
|
||||||
|
return {
|
||||||
|
interval: `${calculatedInterval}s`,
|
||||||
|
from: timerange.from,
|
||||||
|
to: timerange.to,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const lookbackSize = Math.max(timerange.lookbackSize || 5, 5);
|
||||||
return {
|
return {
|
||||||
interval: `${interval}s`,
|
interval: `${calculatedInterval}s`,
|
||||||
from: options.timerange.to - interval * 5000, // We need at least 5 buckets worth of data
|
from: timerange.to - calculatedInterval * lookbackSize * 1000, // We need at least 5 buckets worth of data
|
||||||
to: options.timerange.to,
|
to: timerange.to,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
import { isNumber, last, max, sum, get } from 'lodash';
|
import { isNumber, last, max, sum, get } from 'lodash';
|
||||||
import moment from 'moment';
|
import moment from 'moment';
|
||||||
|
|
||||||
|
import { MetricsExplorerSeries } from '../../../common/http_api/metrics_explorer';
|
||||||
import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds';
|
import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds';
|
||||||
import { InfraSnapshotRequestOptions } from './types';
|
import { InfraSnapshotRequestOptions } from './types';
|
||||||
import { findInventoryModel } from '../../../common/inventory_models';
|
import { findInventoryModel } from '../../../common/inventory_models';
|
||||||
|
@ -127,12 +128,15 @@ export const getNodeMetrics = (
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const lastBucket = findLastFullBucket(nodeBuckets, options);
|
const lastBucket = findLastFullBucket(nodeBuckets, options);
|
||||||
const result = {
|
const result: SnapshotNodeMetric = {
|
||||||
name: options.metric.type,
|
name: options.metric.type,
|
||||||
value: getMetricValueFromBucket(options.metric.type, lastBucket),
|
value: getMetricValueFromBucket(options.metric.type, lastBucket),
|
||||||
max: calculateMax(nodeBuckets, options.metric.type),
|
max: calculateMax(nodeBuckets, options.metric.type),
|
||||||
avg: calculateAvg(nodeBuckets, options.metric.type),
|
avg: calculateAvg(nodeBuckets, options.metric.type),
|
||||||
};
|
};
|
||||||
|
if (options.includeTimeseries) {
|
||||||
|
result.timeseries = getTimeseriesData(nodeBuckets, options.metric.type);
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -164,3 +168,20 @@ function calculateMax(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetri
|
||||||
function calculateAvg(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetricType) {
|
function calculateAvg(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetricType) {
|
||||||
return sum(buckets.map((bucket) => getMetricValueFromBucket(type, bucket))) / buckets.length || 0;
|
return sum(buckets.map((bucket) => getMetricValueFromBucket(type, bucket))) / buckets.length || 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getTimeseriesData(
|
||||||
|
buckets: InfraSnapshotMetricsBucket[],
|
||||||
|
type: SnapshotMetricType
|
||||||
|
): MetricsExplorerSeries {
|
||||||
|
return {
|
||||||
|
id: type,
|
||||||
|
columns: [
|
||||||
|
{ name: 'timestamp', type: 'date' },
|
||||||
|
{ name: 'metric_0', type: 'number' },
|
||||||
|
],
|
||||||
|
rows: buckets.map((bucket) => ({
|
||||||
|
timestamp: bucket.key as number,
|
||||||
|
metric_0: getMetricValueFromBucket(type, bucket),
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
|
@ -39,6 +39,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => {
|
||||||
timerange,
|
timerange,
|
||||||
accountId,
|
accountId,
|
||||||
region,
|
region,
|
||||||
|
includeTimeseries,
|
||||||
} = pipe(
|
} = pipe(
|
||||||
SnapshotRequestRT.decode(request.body),
|
SnapshotRequestRT.decode(request.body),
|
||||||
fold(throwErrors(Boom.badRequest), identity)
|
fold(throwErrors(Boom.badRequest), identity)
|
||||||
|
@ -57,6 +58,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => {
|
||||||
sourceConfiguration: source.configuration,
|
sourceConfiguration: source.configuration,
|
||||||
metric,
|
metric,
|
||||||
timerange,
|
timerange,
|
||||||
|
includeTimeseries,
|
||||||
};
|
};
|
||||||
|
|
||||||
const searchES = <Hit = {}, Aggregation = undefined>(
|
const searchES = <Hit = {}, Aggregation = undefined>(
|
||||||
|
|
|
@ -17,7 +17,7 @@ export default function ({ loadTestFile }) {
|
||||||
loadTestFile(require.resolve('./telemetry'));
|
loadTestFile(require.resolve('./telemetry'));
|
||||||
loadTestFile(require.resolve('./logstash'));
|
loadTestFile(require.resolve('./logstash'));
|
||||||
loadTestFile(require.resolve('./kibana'));
|
loadTestFile(require.resolve('./kibana'));
|
||||||
loadTestFile(require.resolve('./infra'));
|
loadTestFile(require.resolve('./metrics_ui'));
|
||||||
loadTestFile(require.resolve('./beats'));
|
loadTestFile(require.resolve('./beats'));
|
||||||
loadTestFile(require.resolve('./console'));
|
loadTestFile(require.resolve('./console'));
|
||||||
loadTestFile(require.resolve('./management'));
|
loadTestFile(require.resolve('./management'));
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export default function ({ loadTestFile }) {
|
export default function ({ loadTestFile }) {
|
||||||
describe('InfraOps Endpoints', () => {
|
describe('MetricsUI Endpoints', () => {
|
||||||
loadTestFile(require.resolve('./metadata'));
|
loadTestFile(require.resolve('./metadata'));
|
||||||
loadTestFile(require.resolve('./log_analysis'));
|
loadTestFile(require.resolve('./log_analysis'));
|
||||||
loadTestFile(require.resolve('./log_entries'));
|
loadTestFile(require.resolve('./log_entries'));
|
||||||
|
@ -15,7 +15,7 @@ export default function ({ loadTestFile }) {
|
||||||
loadTestFile(require.resolve('./log_summary'));
|
loadTestFile(require.resolve('./log_summary'));
|
||||||
loadTestFile(require.resolve('./metrics'));
|
loadTestFile(require.resolve('./metrics'));
|
||||||
loadTestFile(require.resolve('./sources'));
|
loadTestFile(require.resolve('./sources'));
|
||||||
loadTestFile(require.resolve('./waffle'));
|
loadTestFile(require.resolve('./snapshot'));
|
||||||
loadTestFile(require.resolve('./log_item'));
|
loadTestFile(require.resolve('./log_item'));
|
||||||
loadTestFile(require.resolve('./metrics_alerting'));
|
loadTestFile(require.resolve('./metrics_alerting'));
|
||||||
loadTestFile(require.resolve('./metrics_explorer'));
|
loadTestFile(require.resolve('./metrics_explorer'));
|
|
@ -10,25 +10,15 @@ import { first, last } from 'lodash';
|
||||||
import {
|
import {
|
||||||
InfraSnapshotMetricInput,
|
InfraSnapshotMetricInput,
|
||||||
InfraNodeType,
|
InfraNodeType,
|
||||||
InfraTimerangeInput,
|
|
||||||
InfraSnapshotGroupbyInput,
|
|
||||||
} from '../../../../plugins/infra/server/graphql/types';
|
} from '../../../../plugins/infra/server/graphql/types';
|
||||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||||
import {
|
import {
|
||||||
SnapshotNodeResponse,
|
SnapshotNodeResponse,
|
||||||
SnapshotMetricInput,
|
SnapshotMetricInput,
|
||||||
|
SnapshotRequest,
|
||||||
} from '../../../../plugins/infra/common/http_api/snapshot_api';
|
} from '../../../../plugins/infra/common/http_api/snapshot_api';
|
||||||
import { DATES } from './constants';
|
import { DATES } from './constants';
|
||||||
|
|
||||||
interface SnapshotRequest {
|
|
||||||
filterQuery?: string | null;
|
|
||||||
metric: SnapshotMetricInput;
|
|
||||||
groupBy: InfraSnapshotGroupbyInput[];
|
|
||||||
nodeType: InfraNodeType;
|
|
||||||
sourceId: string;
|
|
||||||
timerange: InfraTimerangeInput;
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function ({ getService }: FtrProviderContext) {
|
export default function ({ getService }: FtrProviderContext) {
|
||||||
const esArchiver = getService('esArchiver');
|
const esArchiver = getService('esArchiver');
|
||||||
const supertest = getService('supertest');
|
const supertest = getService('supertest');
|
||||||
|
@ -200,6 +190,74 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should allow for overrides for interval and ignoring lookback', () => {
|
||||||
|
const resp = fetchSnapshot({
|
||||||
|
sourceId: 'default',
|
||||||
|
timerange: {
|
||||||
|
to: max,
|
||||||
|
from: min,
|
||||||
|
interval: '10s',
|
||||||
|
forceInterval: true,
|
||||||
|
ignoreLookback: true,
|
||||||
|
},
|
||||||
|
metric: { type: 'cpu' } as InfraSnapshotMetricInput,
|
||||||
|
nodeType: 'host' as InfraNodeType,
|
||||||
|
groupBy: [],
|
||||||
|
includeTimeseries: true,
|
||||||
|
});
|
||||||
|
return resp.then((data) => {
|
||||||
|
const snapshot = data;
|
||||||
|
expect(snapshot).to.have.property('nodes');
|
||||||
|
if (snapshot) {
|
||||||
|
const { nodes } = snapshot;
|
||||||
|
expect(nodes.length).to.equal(1);
|
||||||
|
const firstNode = first(nodes);
|
||||||
|
expect(firstNode).to.have.property('path');
|
||||||
|
expect(firstNode.path.length).to.equal(1);
|
||||||
|
expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
|
||||||
|
expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01');
|
||||||
|
expect(firstNode).to.have.property('metric');
|
||||||
|
expect(firstNode.metric).to.have.property('timeseries');
|
||||||
|
expect(firstNode.metric.timeseries?.rows.length).to.equal(58);
|
||||||
|
const rows = firstNode.metric.timeseries?.rows;
|
||||||
|
const rowInterval = (rows?.[1]?.timestamp || 0) - (rows?.[0]?.timestamp || 0);
|
||||||
|
expect(rowInterval).to.equal(10000);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should allow for overrides for lookback', () => {
|
||||||
|
const resp = fetchSnapshot({
|
||||||
|
sourceId: 'default',
|
||||||
|
timerange: {
|
||||||
|
to: max,
|
||||||
|
from: min,
|
||||||
|
interval: '1m',
|
||||||
|
lookbackSize: 6,
|
||||||
|
},
|
||||||
|
metric: { type: 'cpu' } as InfraSnapshotMetricInput,
|
||||||
|
nodeType: 'host' as InfraNodeType,
|
||||||
|
groupBy: [],
|
||||||
|
includeTimeseries: true,
|
||||||
|
});
|
||||||
|
return resp.then((data) => {
|
||||||
|
const snapshot = data;
|
||||||
|
expect(snapshot).to.have.property('nodes');
|
||||||
|
if (snapshot) {
|
||||||
|
const { nodes } = snapshot;
|
||||||
|
expect(nodes.length).to.equal(1);
|
||||||
|
const firstNode = first(nodes);
|
||||||
|
expect(firstNode).to.have.property('path');
|
||||||
|
expect(firstNode.path.length).to.equal(1);
|
||||||
|
expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
|
||||||
|
expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01');
|
||||||
|
expect(firstNode).to.have.property('metric');
|
||||||
|
expect(firstNode.metric).to.have.property('timeseries');
|
||||||
|
expect(firstNode.metric.timeseries?.rows.length).to.equal(7);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('should work with custom metrics', async () => {
|
it('should work with custom metrics', async () => {
|
||||||
const data = await fetchSnapshot({
|
const data = await fetchSnapshot({
|
||||||
sourceId: 'default',
|
sourceId: 'default',
|
Loading…
Reference in a new issue