From 0b8e0c054f9edf3569bf4fd711614d777ed5b6db Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 3 Jun 2020 15:25:39 -0700 Subject: [PATCH] [Metrics UI] Add overrides to Snapshot API to support alert previews (#68125) * [Metrics UI] Add overrides to Snapshot API to support alert previews * Renaming tests from infra to metrics_ui; renaming waffle to snapshot; adding tests for overrides * removing limit and afterKey; removing extra interval * Setting the minimum value for lookbackSize --- .../infra/common/http_api/snapshot_api.ts | 20 +++-- .../inventory_view/hooks/use_snaphot.ts | 8 +- .../create_timerange_with_interval.ts | 36 ++++++--- .../server/lib/snapshot/response_helpers.ts | 23 +++++- .../infra/server/routes/snapshot/index.ts | 2 + x-pack/test/api_integration/apis/index.js | 2 +- .../apis/{infra => metrics_ui}/constants.ts | 0 .../{infra => metrics_ui}/feature_controls.ts | 0 .../apis/{infra => metrics_ui}/index.js | 4 +- .../{infra => metrics_ui}/ip_to_hostname.ts | 0 .../{infra => metrics_ui}/log_analysis.ts | 0 .../apis/{infra => metrics_ui}/log_entries.ts | 0 .../log_entry_highlights.ts | 0 .../apis/{infra => metrics_ui}/log_item.ts | 0 .../apis/{infra => metrics_ui}/log_sources.ts | 0 .../apis/{infra => metrics_ui}/log_summary.ts | 0 .../logs_without_millis.ts | 0 .../apis/{infra => metrics_ui}/metadata.ts | 0 .../apis/{infra => metrics_ui}/metrics.ts | 0 .../{infra => metrics_ui}/metrics_alerting.ts | 0 .../{infra => metrics_ui}/metrics_explorer.ts | 0 .../waffle.ts => metrics_ui/snapshot.ts} | 80 ++++++++++++++++--- .../apis/{infra => metrics_ui}/sources.ts | 0 23 files changed, 144 insertions(+), 31 deletions(-) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/constants.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/feature_controls.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/index.js (91%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/ip_to_hostname.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/log_analysis.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/log_entries.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/log_entry_highlights.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/log_item.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/log_sources.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/log_summary.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/logs_without_millis.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/metadata.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/metrics.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/metrics_alerting.ts (100%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/metrics_explorer.ts (100%) rename x-pack/test/api_integration/apis/{infra/waffle.ts => metrics_ui/snapshot.ts} (81%) rename x-pack/test/api_integration/apis/{infra => metrics_ui}/sources.ts (100%) diff --git a/x-pack/plugins/infra/common/http_api/snapshot_api.ts b/x-pack/plugins/infra/common/http_api/snapshot_api.ts index 6b666b39b009..1c7dfed82783 100644 --- a/x-pack/plugins/infra/common/http_api/snapshot_api.ts +++ b/x-pack/plugins/infra/common/http_api/snapshot_api.ts @@ -6,6 +6,7 @@ import * as rt from 'io-ts'; import { SnapshotMetricTypeRT, ItemTypeRT } from '../inventory_models/types'; +import { metricsExplorerSeriesRT } from './metrics_explorer'; export const SnapshotNodePathRT = rt.intersection([ rt.type({ @@ -21,6 +22,7 @@ const SnapshotNodeMetricOptionalRT = rt.partial({ value: rt.union([rt.number, rt.null]), avg: rt.union([rt.number, rt.null]), max: rt.union([rt.number, rt.null]), + timeseries: metricsExplorerSeriesRT, }); const SnapshotNodeMetricRequiredRT = rt.type({ @@ -41,11 +43,18 @@ export const SnapshotNodeResponseRT = rt.type({ interval: rt.string, }); -export const InfraTimerangeInputRT = rt.type({ - interval: rt.string, - to: rt.number, - from: rt.number, -}); +export const InfraTimerangeInputRT = rt.intersection([ + rt.type({ + interval: rt.string, + to: rt.number, + from: rt.number, + }), + rt.partial({ + lookbackSize: rt.number, + ignoreLookback: rt.boolean, + forceInterval: rt.boolean, + }), +]); export const SnapshotGroupByRT = rt.array( rt.partial({ @@ -97,6 +106,7 @@ export const SnapshotRequestRT = rt.intersection([ accountId: rt.string, region: rt.string, filterQuery: rt.union([rt.string, rt.null]), + includeTimeseries: rt.boolean, }), ]); diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts index 3ec63d7b2de2..721a2d5792dc 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/hooks/use_snaphot.ts @@ -14,6 +14,8 @@ import { SnapshotNodeResponseRT, SnapshotNodeResponse, SnapshotGroupBy, + SnapshotRequest, + InfraTimerangeInput, } from '../../../../../common/http_api/snapshot_api'; import { InventoryItemType, @@ -37,10 +39,11 @@ export function useSnapshot( ); }; - const timerange = { + const timerange: InfraTimerangeInput = { interval: '1m', to: currentTime, from: currentTime - 360 * 1000, + lookbackSize: 20, }; const { error, loading, response, makeRequest } = useHTTPRequest( @@ -55,7 +58,8 @@ export function useSnapshot( sourceId, accountId, region, - }), + includeTimeseries: true, + } as SnapshotRequest), decodeResponse ); diff --git a/x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts b/x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts index 924d12bec0c5..d1a4ed431a2b 100644 --- a/x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts +++ b/x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts @@ -12,30 +12,48 @@ import { SnapshotModel, SnapshotModelMetricAggRT } from '../../../common/invento import { getDatasetForField } from '../../routes/metrics_explorer/lib/get_dataset_for_field'; import { InfraTimerangeInput } from '../../../common/http_api/snapshot_api'; import { ESSearchClient } from '.'; +import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds'; -export const createTimeRangeWithInterval = async ( - client: ESSearchClient, - options: InfraSnapshotRequestOptions -): Promise => { +const createInterval = async (client: ESSearchClient, options: InfraSnapshotRequestOptions) => { + const { timerange } = options; + if (timerange.forceInterval && timerange.interval) { + return getIntervalInSeconds(timerange.interval); + } const aggregations = getMetricsAggregations(options); const modules = await aggregationsToModules(client, aggregations, options); - const interval = Math.max( + return Math.max( (await calculateMetricInterval( client, { indexPattern: options.sourceConfiguration.metricAlias, timestampField: options.sourceConfiguration.fields.timestamp, - timerange: { from: options.timerange.from, to: options.timerange.to }, + timerange: { from: timerange.from, to: timerange.to }, }, modules, options.nodeType )) || 60, 60 ); +}; + +export const createTimeRangeWithInterval = async ( + client: ESSearchClient, + options: InfraSnapshotRequestOptions +): Promise => { + const { timerange } = options; + const calculatedInterval = await createInterval(client, options); + if (timerange.ignoreLookback) { + return { + interval: `${calculatedInterval}s`, + from: timerange.from, + to: timerange.to, + }; + } + const lookbackSize = Math.max(timerange.lookbackSize || 5, 5); return { - interval: `${interval}s`, - from: options.timerange.to - interval * 5000, // We need at least 5 buckets worth of data - to: options.timerange.to, + interval: `${calculatedInterval}s`, + from: timerange.to - calculatedInterval * lookbackSize * 1000, // We need at least 5 buckets worth of data + to: timerange.to, }; }; diff --git a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts b/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts index 031eb881c91a..6cb415d8e7ac 100644 --- a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts +++ b/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts @@ -7,6 +7,7 @@ import { isNumber, last, max, sum, get } from 'lodash'; import moment from 'moment'; +import { MetricsExplorerSeries } from '../../../common/http_api/metrics_explorer'; import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds'; import { InfraSnapshotRequestOptions } from './types'; import { findInventoryModel } from '../../../common/inventory_models'; @@ -127,12 +128,15 @@ export const getNodeMetrics = ( }; } const lastBucket = findLastFullBucket(nodeBuckets, options); - const result = { + const result: SnapshotNodeMetric = { name: options.metric.type, value: getMetricValueFromBucket(options.metric.type, lastBucket), max: calculateMax(nodeBuckets, options.metric.type), avg: calculateAvg(nodeBuckets, options.metric.type), }; + if (options.includeTimeseries) { + result.timeseries = getTimeseriesData(nodeBuckets, options.metric.type); + } return result; }; @@ -164,3 +168,20 @@ function calculateMax(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetri function calculateAvg(buckets: InfraSnapshotMetricsBucket[], type: SnapshotMetricType) { return sum(buckets.map((bucket) => getMetricValueFromBucket(type, bucket))) / buckets.length || 0; } + +function getTimeseriesData( + buckets: InfraSnapshotMetricsBucket[], + type: SnapshotMetricType +): MetricsExplorerSeries { + return { + id: type, + columns: [ + { name: 'timestamp', type: 'date' }, + { name: 'metric_0', type: 'number' }, + ], + rows: buckets.map((bucket) => ({ + timestamp: bucket.key as number, + metric_0: getMetricValueFromBucket(type, bucket), + })), + }; +} diff --git a/x-pack/plugins/infra/server/routes/snapshot/index.ts b/x-pack/plugins/infra/server/routes/snapshot/index.ts index 2d951d426b03..3a0326fb6ae8 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/index.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/index.ts @@ -39,6 +39,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => { timerange, accountId, region, + includeTimeseries, } = pipe( SnapshotRequestRT.decode(request.body), fold(throwErrors(Boom.badRequest), identity) @@ -57,6 +58,7 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => { sourceConfiguration: source.configuration, metric, timerange, + includeTimeseries, }; const searchES = ( diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index f3a2d2fa3cd8..b76cc00d842f 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -17,7 +17,7 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./telemetry')); loadTestFile(require.resolve('./logstash')); loadTestFile(require.resolve('./kibana')); - loadTestFile(require.resolve('./infra')); + loadTestFile(require.resolve('./metrics_ui')); loadTestFile(require.resolve('./beats')); loadTestFile(require.resolve('./console')); loadTestFile(require.resolve('./management')); diff --git a/x-pack/test/api_integration/apis/infra/constants.ts b/x-pack/test/api_integration/apis/metrics_ui/constants.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/constants.ts rename to x-pack/test/api_integration/apis/metrics_ui/constants.ts diff --git a/x-pack/test/api_integration/apis/infra/feature_controls.ts b/x-pack/test/api_integration/apis/metrics_ui/feature_controls.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/feature_controls.ts rename to x-pack/test/api_integration/apis/metrics_ui/feature_controls.ts diff --git a/x-pack/test/api_integration/apis/infra/index.js b/x-pack/test/api_integration/apis/metrics_ui/index.js similarity index 91% rename from x-pack/test/api_integration/apis/infra/index.js rename to x-pack/test/api_integration/apis/metrics_ui/index.js index d88dcec7b050..eb8ee77da582 100644 --- a/x-pack/test/api_integration/apis/infra/index.js +++ b/x-pack/test/api_integration/apis/metrics_ui/index.js @@ -5,7 +5,7 @@ */ export default function ({ loadTestFile }) { - describe('InfraOps Endpoints', () => { + describe('MetricsUI Endpoints', () => { loadTestFile(require.resolve('./metadata')); loadTestFile(require.resolve('./log_analysis')); loadTestFile(require.resolve('./log_entries')); @@ -15,7 +15,7 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./log_summary')); loadTestFile(require.resolve('./metrics')); loadTestFile(require.resolve('./sources')); - loadTestFile(require.resolve('./waffle')); + loadTestFile(require.resolve('./snapshot')); loadTestFile(require.resolve('./log_item')); loadTestFile(require.resolve('./metrics_alerting')); loadTestFile(require.resolve('./metrics_explorer')); diff --git a/x-pack/test/api_integration/apis/infra/ip_to_hostname.ts b/x-pack/test/api_integration/apis/metrics_ui/ip_to_hostname.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/ip_to_hostname.ts rename to x-pack/test/api_integration/apis/metrics_ui/ip_to_hostname.ts diff --git a/x-pack/test/api_integration/apis/infra/log_analysis.ts b/x-pack/test/api_integration/apis/metrics_ui/log_analysis.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/log_analysis.ts rename to x-pack/test/api_integration/apis/metrics_ui/log_analysis.ts diff --git a/x-pack/test/api_integration/apis/infra/log_entries.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/log_entries.ts rename to x-pack/test/api_integration/apis/metrics_ui/log_entries.ts diff --git a/x-pack/test/api_integration/apis/infra/log_entry_highlights.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/log_entry_highlights.ts rename to x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts diff --git a/x-pack/test/api_integration/apis/infra/log_item.ts b/x-pack/test/api_integration/apis/metrics_ui/log_item.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/log_item.ts rename to x-pack/test/api_integration/apis/metrics_ui/log_item.ts diff --git a/x-pack/test/api_integration/apis/infra/log_sources.ts b/x-pack/test/api_integration/apis/metrics_ui/log_sources.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/log_sources.ts rename to x-pack/test/api_integration/apis/metrics_ui/log_sources.ts diff --git a/x-pack/test/api_integration/apis/infra/log_summary.ts b/x-pack/test/api_integration/apis/metrics_ui/log_summary.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/log_summary.ts rename to x-pack/test/api_integration/apis/metrics_ui/log_summary.ts diff --git a/x-pack/test/api_integration/apis/infra/logs_without_millis.ts b/x-pack/test/api_integration/apis/metrics_ui/logs_without_millis.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/logs_without_millis.ts rename to x-pack/test/api_integration/apis/metrics_ui/logs_without_millis.ts diff --git a/x-pack/test/api_integration/apis/infra/metadata.ts b/x-pack/test/api_integration/apis/metrics_ui/metadata.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/metadata.ts rename to x-pack/test/api_integration/apis/metrics_ui/metadata.ts diff --git a/x-pack/test/api_integration/apis/infra/metrics.ts b/x-pack/test/api_integration/apis/metrics_ui/metrics.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/metrics.ts rename to x-pack/test/api_integration/apis/metrics_ui/metrics.ts diff --git a/x-pack/test/api_integration/apis/infra/metrics_alerting.ts b/x-pack/test/api_integration/apis/metrics_ui/metrics_alerting.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/metrics_alerting.ts rename to x-pack/test/api_integration/apis/metrics_ui/metrics_alerting.ts diff --git a/x-pack/test/api_integration/apis/infra/metrics_explorer.ts b/x-pack/test/api_integration/apis/metrics_ui/metrics_explorer.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/metrics_explorer.ts rename to x-pack/test/api_integration/apis/metrics_ui/metrics_explorer.ts diff --git a/x-pack/test/api_integration/apis/infra/waffle.ts b/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts similarity index 81% rename from x-pack/test/api_integration/apis/infra/waffle.ts rename to x-pack/test/api_integration/apis/metrics_ui/snapshot.ts index 80f86e91378e..09c7866bf3a1 100644 --- a/x-pack/test/api_integration/apis/infra/waffle.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts @@ -10,25 +10,15 @@ import { first, last } from 'lodash'; import { InfraSnapshotMetricInput, InfraNodeType, - InfraTimerangeInput, - InfraSnapshotGroupbyInput, } from '../../../../plugins/infra/server/graphql/types'; import { FtrProviderContext } from '../../ftr_provider_context'; import { SnapshotNodeResponse, SnapshotMetricInput, + SnapshotRequest, } from '../../../../plugins/infra/common/http_api/snapshot_api'; import { DATES } from './constants'; -interface SnapshotRequest { - filterQuery?: string | null; - metric: SnapshotMetricInput; - groupBy: InfraSnapshotGroupbyInput[]; - nodeType: InfraNodeType; - sourceId: string; - timerange: InfraTimerangeInput; -} - export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); @@ -200,6 +190,74 @@ export default function ({ getService }: FtrProviderContext) { }); }); + it('should allow for overrides for interval and ignoring lookback', () => { + const resp = fetchSnapshot({ + sourceId: 'default', + timerange: { + to: max, + from: min, + interval: '10s', + forceInterval: true, + ignoreLookback: true, + }, + metric: { type: 'cpu' } as InfraSnapshotMetricInput, + nodeType: 'host' as InfraNodeType, + groupBy: [], + includeTimeseries: true, + }); + return resp.then((data) => { + const snapshot = data; + expect(snapshot).to.have.property('nodes'); + if (snapshot) { + const { nodes } = snapshot; + expect(nodes.length).to.equal(1); + const firstNode = first(nodes); + expect(firstNode).to.have.property('path'); + expect(firstNode.path.length).to.equal(1); + expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01'); + expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01'); + expect(firstNode).to.have.property('metric'); + expect(firstNode.metric).to.have.property('timeseries'); + expect(firstNode.metric.timeseries?.rows.length).to.equal(58); + const rows = firstNode.metric.timeseries?.rows; + const rowInterval = (rows?.[1]?.timestamp || 0) - (rows?.[0]?.timestamp || 0); + expect(rowInterval).to.equal(10000); + } + }); + }); + + it('should allow for overrides for lookback', () => { + const resp = fetchSnapshot({ + sourceId: 'default', + timerange: { + to: max, + from: min, + interval: '1m', + lookbackSize: 6, + }, + metric: { type: 'cpu' } as InfraSnapshotMetricInput, + nodeType: 'host' as InfraNodeType, + groupBy: [], + includeTimeseries: true, + }); + return resp.then((data) => { + const snapshot = data; + expect(snapshot).to.have.property('nodes'); + if (snapshot) { + const { nodes } = snapshot; + expect(nodes.length).to.equal(1); + const firstNode = first(nodes); + expect(firstNode).to.have.property('path'); + expect(firstNode.path.length).to.equal(1); + expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01'); + expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01'); + expect(firstNode).to.have.property('metric'); + expect(firstNode.metric).to.have.property('timeseries'); + expect(firstNode.metric.timeseries?.rows.length).to.equal(7); + } + }); + }); + it('should work with custom metrics', async () => { const data = await fetchSnapshot({ sourceId: 'default', diff --git a/x-pack/test/api_integration/apis/infra/sources.ts b/x-pack/test/api_integration/apis/metrics_ui/sources.ts similarity index 100% rename from x-pack/test/api_integration/apis/infra/sources.ts rename to x-pack/test/api_integration/apis/metrics_ui/sources.ts