From a08895dbfc20ea883afb526a8597014ded4571b7 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Fri, 29 Jan 2021 10:42:35 -0500 Subject: [PATCH] [ML] Anomaly Detection: add anomalies map to explorer for jobs with 'lat_long' function (#88416) * wip: create embedded map component for explorer * add embeddedMap component to explorer * use geo_results * remove charts callout when map is shown * add translation, round geo coordinates * create GEO_MAP chart type and move embedded map to charts area * remove embedded map that is no longer used * fix type and fail silently if plugin not available * fix multiple type of jobs charts view * fix tooltip function and remove single viewer link for latlong * ensure diff types of jobs show correct charts. fix jest test * show errorCallout if maps not enabled and is lat_long job * use shared MlEmbeddedMapComponent in explorer * ensure latLong jobs not viewable in single metric viewer * update jest test --- x-pack/plugins/ml/common/util/job_utils.ts | 12 ++ .../ml_embedded_map/ml_embedded_map.tsx | 11 +- .../explorer_chart_embedded_map.tsx | 33 ++++ .../explorer_charts_container.js | 76 +++++++-- .../explorer_charts_container_service.js | 111 +++++++++--- .../explorer/explorer_charts/map_config.ts | 161 ++++++++++++++++++ .../explorer/explorer_constants.ts | 1 + .../application/explorer/explorer_utils.js | 1 + .../advanced_detector_modal/descriptions.tsx | 2 +- .../results_service/result_service_rx.ts | 3 +- .../application/util/chart_config_builder.js | 5 +- .../ml/public/application/util/chart_utils.js | 5 + 12 files changed, 364 insertions(+), 57 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_embedded_map.tsx create mode 100644 x-pack/plugins/ml/public/application/explorer/explorer_charts/map_config.ts diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index 4f4d9851c495..d20ad4a36894 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -78,6 +78,18 @@ export function isTimeSeriesViewDetector(job: CombinedJob, detectorIndex: number ); } +// Returns a flag to indicate whether the specified job is suitable for embedded map viewing. +export function isMappableJob(job: CombinedJob, detectorIndex: number): boolean { + let isMappable = false; + const { detectors } = job.analysis_config; + if (detectorIndex >= 0 && detectorIndex < detectors.length) { + const dtr = detectors[detectorIndex]; + const functionName = dtr.function; + isMappable = functionName === ML_JOB_AGGREGATION.LAT_LONG; + } + return isMappable; +} + // Returns a flag to indicate whether the source data can be plotted in a time // series chart for the specified detector. export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex: number): boolean { diff --git a/x-pack/plugins/ml/public/application/components/ml_embedded_map/ml_embedded_map.tsx b/x-pack/plugins/ml/public/application/components/ml_embedded_map/ml_embedded_map.tsx index d5fdc9d52a10..12c7d6ac69bb 100644 --- a/x-pack/plugins/ml/public/application/components/ml_embedded_map/ml_embedded_map.tsx +++ b/x-pack/plugins/ml/public/application/components/ml_embedded_map/ml_embedded_map.tsx @@ -8,6 +8,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { htmlIdGenerator } from '@elastic/eui'; import { LayerDescriptor } from '../../../../../maps/common/descriptor_types'; +import { INITIAL_LOCATION } from '../../../../../maps/common/constants'; import { MapEmbeddable, MapEmbeddableInput, @@ -81,21 +82,13 @@ export function MlEmbeddedMapComponent({ viewMode: ViewMode.VIEW, isLayerTOCOpen: false, hideFilterActions: true, - // Zoom Lat/Lon values are set to make sure map is in center in the panel - // It will also omit Greenland/Antarctica etc. NOTE: Can be removed when initialLocation is set - mapCenter: { - lon: 11, - lat: 20, - zoom: 1, - }, // can use mapSettings to center map on anomalies mapSettings: { disableInteractive: false, hideToolbarOverlay: false, hideLayerControl: false, hideViewControl: false, - // Doesn't currently work with GEO_JSON. Will uncomment when https://github.com/elastic/kibana/pull/88294 is in - // initialLocation: INITIAL_LOCATION.AUTO_FIT_TO_BOUNDS, // this will startup based on data-extent + initialLocation: INITIAL_LOCATION.AUTO_FIT_TO_BOUNDS, // this will startup based on data-extent autoFitToDataBounds: true, // this will auto-fit when there are changes to the filter and/or query }, }; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_embedded_map.tsx b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_embedded_map.tsx new file mode 100644 index 000000000000..fc1621e962f3 --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_embedded_map.tsx @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState, useEffect } from 'react'; +import { Dictionary } from '../../../../common/types/common'; +import { LayerDescriptor } from '../../../../../maps/common/descriptor_types'; +import { getMLAnomaliesActualLayer, getMLAnomaliesTypicalLayer } from './map_config'; +import { MlEmbeddedMapComponent } from '../../components/ml_embedded_map'; +interface Props { + seriesConfig: Dictionary; +} + +export function EmbeddedMapComponentWrapper({ seriesConfig }: Props) { + const [layerList, setLayerList] = useState([]); + + useEffect(() => { + if (seriesConfig.mapData && seriesConfig.mapData.length > 0) { + setLayerList([ + getMLAnomaliesActualLayer(seriesConfig.mapData), + getMLAnomaliesTypicalLayer(seriesConfig.mapData), + ]); + } + }, [seriesConfig]); + + return ( +
+ +
+ ); +} diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js index 774372f678c9..9921b5f99184 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js @@ -22,6 +22,7 @@ import { } from '../../util/chart_utils'; import { ExplorerChartDistribution } from './explorer_chart_distribution'; import { ExplorerChartSingleMetric } from './explorer_chart_single_metric'; +import { EmbeddedMapComponentWrapper } from './explorer_chart_embedded_map'; import { ExplorerChartLabel } from './components/explorer_chart_label'; import { CHART_TYPE } from '../explorer_constants'; @@ -30,6 +31,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { MlTooltipComponent } from '../../components/chart_tooltip'; import { withKibana } from '../../../../../../../src/plugins/kibana_react/public'; import { ML_APP_URL_GENERATOR } from '../../../../common/constants/ml_url_generator'; +import { ML_JOB_AGGREGATION } from '../../../../common/constants/aggregation_types'; import { addItemToRecentlyAccessed } from '../../util/recently_accessed'; import { ExplorerChartsErrorCallOuts } from './explorer_charts_error_callouts'; @@ -43,6 +45,9 @@ const textViewButton = i18n.translate( defaultMessage: 'Open in Single Metric Viewer', } ); +const mapsPluginMessage = i18n.translate('xpack.ml.explorer.charts.mapsPluginMissingMessage', { + defaultMessage: 'maps or embeddable start plugin not found', +}); // create a somewhat unique ID // from charts metadata for React's key attribute @@ -67,8 +72,8 @@ function ExplorerChartContainer({ useEffect(() => { let isCancelled = false; const generateLink = async () => { - const singleMetricViewerLink = await getExploreSeriesLink(mlUrlGenerator, series); - if (!isCancelled) { + if (!isCancelled && series.functionDescription !== ML_JOB_AGGREGATION.LAT_LONG) { + const singleMetricViewerLink = await getExploreSeriesLink(mlUrlGenerator, series); setExplorerSeriesLink(singleMetricViewerLink); } }; @@ -150,6 +155,18 @@ function ExplorerChartContainer({ {(() => { + if (chartType === CHART_TYPE.GEO_MAP) { + return ( + + {(tooltipService) => ( + + )} + + ); + } if ( chartType === CHART_TYPE.EVENT_DISTRIBUTION || chartType === CHART_TYPE.POPULATION_DISTRIBUTION @@ -167,18 +184,20 @@ function ExplorerChartContainer({ ); } - return ( - - {(tooltipService) => ( - - )} - - ); + if (chartType === CHART_TYPE.SINGLE_METRIC) { + return ( + + {(tooltipService) => ( + + )} + + ); + } })()} ); @@ -199,8 +218,31 @@ export const ExplorerChartsContainerUI = ({ share: { urlGenerators: { getUrlGenerator }, }, + embeddable: embeddablePlugin, + maps: mapsPlugin, }, } = kibana; + + let seriesToPlotFiltered; + + if (!embeddablePlugin || !mapsPlugin) { + seriesToPlotFiltered = []; + // Show missing plugin callout + seriesToPlot.forEach((series) => { + if (series.functionDescription === 'lat_long') { + if (errorMessages[mapsPluginMessage] === undefined) { + errorMessages[mapsPluginMessage] = new Set([series.jobId]); + } else { + errorMessages[mapsPluginMessage].add(series.jobId); + } + } else { + seriesToPlotFiltered.push(series); + } + }); + } + + const seriesToUse = seriesToPlotFiltered !== undefined ? seriesToPlotFiltered : seriesToPlot; + const mlUrlGenerator = useMemo(() => getUrlGenerator(ML_APP_URL_GENERATOR), [getUrlGenerator]); // doesn't allow a setting of `columns={1}` when chartsPerRow would be 1. @@ -208,13 +250,13 @@ export const ExplorerChartsContainerUI = ({ const chartsWidth = chartsPerRow === 1 ? 'calc(100% - 20px)' : 'auto'; const chartsColumns = chartsPerRow === 1 ? 0 : chartsPerRow; - const wrapLabel = seriesToPlot.some((series) => isLabelLengthAboveThreshold(series)); + const wrapLabel = seriesToUse.some((series) => isLabelLengthAboveThreshold(series)); return ( <> - {seriesToPlot.length > 0 && - seriesToPlot.map((series) => ( + {seriesToUse.length > 0 && + seriesToUse.map((series) => ( + (record.function_description || recordsToPlot.function) === ML_JOB_AGGREGATION.LAT_LONG + ); + const seriesConfigs = recordsToPlot.map(buildConfig); + const seriesConfigsNoGeoData = []; + + // initialize the charts with loading indicators + data.seriesToPlot = seriesConfigs.map((config) => ({ + ...config, + loading: true, + chartData: null, + })); + + const mapData = []; + + if (hasGeoData !== undefined) { + for (let i = 0; i < seriesConfigs.length; i++) { + const config = seriesConfigs[i]; + let records; + if (config.detectorLabel.includes(ML_JOB_AGGREGATION.LAT_LONG)) { + if (config.entityFields.length) { + records = [ + recordsToPlot.find((record) => { + const entityFieldName = config.entityFields[0].fieldName; + const entityFieldValue = config.entityFields[0].fieldValue; + return (record[entityFieldName] && record[entityFieldName][0]) === entityFieldValue; + }), + ]; + } else { + records = recordsToPlot; + } + + mapData.push({ + ...config, + loading: false, + mapData: records, + }); + } else { + seriesConfigsNoGeoData.push(config); + } + } + } // Calculate the time range of the charts, which is a function of the chart width and max job bucket span. data.tooManyBuckets = false; @@ -92,13 +137,6 @@ export const anomalyDataChange = function ( ); data.tooManyBuckets = tooManyBuckets; - // initialize the charts with loading indicators - data.seriesToPlot = seriesConfigs.map((config) => ({ - ...config, - loading: true, - chartData: null, - })); - data.errorMessages = errorMessages; explorerService.setCharts({ ...data }); @@ -269,22 +307,27 @@ export const anomalyDataChange = function ( // only after that trigger data processing and page render. // TODO - if query returns no results e.g. source data has been deleted, // display a message saying 'No data between earliest/latest'. - const seriesPromises = seriesConfigs.map((seriesConfig) => - Promise.all([ - getMetricData(seriesConfig, chartRange), - getRecordsForCriteria(seriesConfig, chartRange), - getScheduledEvents(seriesConfig, chartRange), - getEventDistribution(seriesConfig, chartRange), - ]) - ); + const seriesPromises = []; + // Use seriesConfigs list without geo data config so indices match up after seriesPromises are resolved and we map through the responses + const seriesCongifsForPromises = hasGeoData ? seriesConfigsNoGeoData : seriesConfigs; + seriesCongifsForPromises.forEach((seriesConfig) => { + seriesPromises.push( + Promise.all([ + getMetricData(seriesConfig, chartRange), + getRecordsForCriteria(seriesConfig, chartRange), + getScheduledEvents(seriesConfig, chartRange), + getEventDistribution(seriesConfig, chartRange), + ]) + ); + }); function processChartData(response, seriesIndex) { const metricData = response[0].results; const records = response[1].records; - const jobId = seriesConfigs[seriesIndex].jobId; + const jobId = seriesCongifsForPromises[seriesIndex].jobId; const scheduledEvents = response[2].events[jobId]; const eventDistribution = response[3]; - const chartType = getChartType(seriesConfigs[seriesIndex]); + const chartType = getChartType(seriesCongifsForPromises[seriesIndex]); // Sort records in ascending time order matching up with chart data records.sort((recordA, recordB) => { @@ -409,16 +452,25 @@ export const anomalyDataChange = function ( ); const overallChartLimits = chartLimits(allDataPoints); - data.seriesToPlot = response.map((d, i) => ({ - ...seriesConfigs[i], - loading: false, - chartData: processedData[i], - plotEarliest: chartRange.min, - plotLatest: chartRange.max, - selectedEarliest: selectedEarliestMs, - selectedLatest: selectedLatestMs, - chartLimits: USE_OVERALL_CHART_LIMITS ? overallChartLimits : chartLimits(processedData[i]), - })); + data.seriesToPlot = response.map((d, i) => { + return { + ...seriesCongifsForPromises[i], + loading: false, + chartData: processedData[i], + plotEarliest: chartRange.min, + plotLatest: chartRange.max, + selectedEarliest: selectedEarliestMs, + selectedLatest: selectedLatestMs, + chartLimits: USE_OVERALL_CHART_LIMITS + ? overallChartLimits + : chartLimits(processedData[i]), + }; + }); + + if (mapData.length) { + // push map data in if it's available + data.seriesToPlot.push(...mapData); + } explorerService.setCharts({ ...data }); }) .catch((error) => { @@ -447,7 +499,10 @@ function processRecordsForDisplay(anomalyRecords) { return; } - let isChartable = isSourceDataChartableForDetector(job, record.detector_index); + let isChartable = + isSourceDataChartableForDetector(job, record.detector_index) || + isMappableJob(job, record.detector_index); + if (isChartable === false) { if (isModelPlotChartableForDetector(job, record.detector_index)) { // Check if model plot is enabled for this job. diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/map_config.ts b/x-pack/plugins/ml/public/application/explorer/explorer_charts/map_config.ts new file mode 100644 index 000000000000..451fa602315d --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/map_config.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FIELD_ORIGIN, STYLE_TYPE } from '../../../../../maps/common/constants'; +import { ANOMALY_THRESHOLD, SEVERITY_COLORS } from '../../../../common'; + +const FEATURE = 'Feature'; +const POINT = 'Point'; +const SEVERITY_COLOR_RAMP = [ + { + stop: ANOMALY_THRESHOLD.LOW, + color: SEVERITY_COLORS.WARNING, + }, + { + stop: ANOMALY_THRESHOLD.MINOR, + color: SEVERITY_COLORS.MINOR, + }, + { + stop: ANOMALY_THRESHOLD.MAJOR, + color: SEVERITY_COLORS.MAJOR, + }, + { + stop: ANOMALY_THRESHOLD.CRITICAL, + color: SEVERITY_COLORS.CRITICAL, + }, +]; + +function getAnomalyFeatures(anomalies: any[], type: 'actual_point' | 'typical_point') { + const anomalyFeatures = []; + for (let i = 0; i < anomalies.length; i++) { + const anomaly = anomalies[i]; + const geoResults = anomaly.geo_results || (anomaly?.causes && anomaly?.causes[0]?.geo_results); + const coordinateStr = geoResults && geoResults[type]; + if (coordinateStr !== undefined) { + // Must reverse coordinates here. Map expects [lon, lat] - anomalies are stored as [lat, lon] for lat_lon jobs + const coordinates = coordinateStr + .split(',') + .map((point: string) => Number(point)) + .reverse(); + + anomalyFeatures.push({ + type: FEATURE, + geometry: { + type: POINT, + coordinates, + }, + properties: { + record_score: Math.floor(anomaly.record_score), + [type]: coordinates.map((point: number) => point.toFixed(2)), + }, + }); + } + } + return anomalyFeatures; +} + +export const getMLAnomaliesTypicalLayer = (anomalies: any) => { + return { + id: 'anomalies_typical_layer', + label: 'Typical', + sourceDescriptor: { + id: 'b7486535-171b-4d3b-bb2e-33c1a0a2854e', + type: 'GEOJSON_FILE', + __featureCollection: { + features: getAnomalyFeatures(anomalies, 'typical_point'), + type: 'FeatureCollection', + }, + }, + visible: true, + style: { + type: 'VECTOR', + properties: { + fillColor: { + type: 'STATIC', + options: { + color: '#98A2B2', + }, + }, + lineColor: { + type: 'STATIC', + options: { + color: '#fff', + }, + }, + lineWidth: { + type: 'STATIC', + options: { + size: 2, + }, + }, + iconSize: { + type: 'STATIC', + options: { + size: 6, + }, + }, + }, + }, + type: 'VECTOR', + }; +}; + +export const getMLAnomaliesActualLayer = (anomalies: any) => { + return { + id: 'anomalies_actual_layer', + label: 'Actual', + sourceDescriptor: { + id: 'b7486535-171b-4d3b-bb2e-33c1a0a2854d', + type: 'GEOJSON_FILE', + __fields: [ + { + name: 'record_score', + type: 'number', + }, + ], + __featureCollection: { + features: getAnomalyFeatures(anomalies, 'actual_point'), + type: 'FeatureCollection', + }, + }, + visible: true, + style: { + type: 'VECTOR', + properties: { + fillColor: { + type: STYLE_TYPE.DYNAMIC, + options: { + customColorRamp: SEVERITY_COLOR_RAMP, + field: { + name: 'record_score', + origin: FIELD_ORIGIN.SOURCE, + }, + useCustomColorRamp: true, + }, + }, + lineColor: { + type: 'STATIC', + options: { + color: '#fff', + }, + }, + lineWidth: { + type: 'STATIC', + options: { + size: 2, + }, + }, + iconSize: { + type: 'STATIC', + options: { + size: 6, + }, + }, + }, + }, + type: 'VECTOR', + }; +}; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts b/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts index 3f5f016fc365..2178c837458e 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts +++ b/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts @@ -48,6 +48,7 @@ export const CHART_TYPE = { EVENT_DISTRIBUTION: 'event_distribution', POPULATION_DISTRIBUTION: 'population_distribution', SINGLE_METRIC: 'single_metric', + GEO_MAP: 'geo_map', }; export const MAX_CATEGORY_EXAMPLES = 10; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_utils.js b/x-pack/plugins/ml/public/application/explorer/explorer_utils.js index f6889c9a6f24..4ba9d4ea14f1 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_utils.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_utils.js @@ -511,6 +511,7 @@ export async function loadAnomaliesTableData( const entityFields = getEntityFieldList(anomaly.source); isChartable = isModelPlotEnabled(job, anomaly.detectorIndex, entityFields); } + anomaly.isTimeSeriesViewRecord = isChartable; if (mlJobService.customUrlsByJob[jobId] !== undefined) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/descriptions.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/descriptions.tsx index 280ac85a5a2b..470fe11759d2 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/descriptions.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/descriptions.tsx @@ -46,7 +46,7 @@ export const FieldDescription: FC = memo(({ children }) => { description={ } > diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index 514449385bf0..3747e84f4376 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -156,7 +156,8 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { } body.aggs.byTime.aggs = {}; - if (metricFieldName !== undefined && metricFieldName !== '') { + + if (metricFieldName !== undefined && metricFieldName !== '' && metricFunction) { const metricAgg: any = { [metricFunction]: {}, }; diff --git a/x-pack/plugins/ml/public/application/util/chart_config_builder.js b/x-pack/plugins/ml/public/application/util/chart_config_builder.js index a30280f1220c..a306211defc8 100644 --- a/x-pack/plugins/ml/public/application/util/chart_config_builder.js +++ b/x-pack/plugins/ml/public/application/util/chart_config_builder.js @@ -24,7 +24,10 @@ export function buildConfigFromDetector(job, detectorIndex) { const config = { jobId: job.job_id, detectorIndex: detectorIndex, - metricFunction: mlFunctionToESAggregation(detector.function), + metricFunction: + detector.function === ML_JOB_AGGREGATION.LAT_LONG + ? ML_JOB_AGGREGATION.LAT_LONG + : mlFunctionToESAggregation(detector.function), timeField: job.data_description.time_field, interval: job.analysis_config.bucket_span, datafeedConfig: job.datafeed_config, diff --git a/x-pack/plugins/ml/public/application/util/chart_utils.js b/x-pack/plugins/ml/public/application/util/chart_utils.js index 402c922a0034..799187cc37df 100644 --- a/x-pack/plugins/ml/public/application/util/chart_utils.js +++ b/x-pack/plugins/ml/public/application/util/chart_utils.js @@ -176,6 +176,11 @@ const POPULATION_DISTRIBUTION_ENABLED = true; // get the chart type based on its configuration export function getChartType(config) { let chartType = CHART_TYPE.SINGLE_METRIC; + + if (config.functionDescription === 'lat_long' || config.mapData !== undefined) { + return CHART_TYPE.GEO_MAP; + } + if ( EVENT_DISTRIBUTION_ENABLED && config.functionDescription === 'rare' &&