[ML] Anomaly Detection: add anomalies map to explorer for jobs with 'lat_long' function (#88416)

* wip: create embedded map component for explorer

* add embeddedMap component to explorer

* use geo_results

* remove charts callout when map is shown

* add translation, round geo coordinates

* create GEO_MAP chart type and move embedded map to charts area

* remove embedded map that is no longer used

* fix type and fail silently if plugin not available

* fix multiple type of jobs charts view

* fix tooltip function and remove single viewer link for latlong

* ensure diff types of jobs show correct charts. fix jest test

* show errorCallout if maps not enabled and is lat_long job

* use shared MlEmbeddedMapComponent in explorer

* ensure latLong jobs not viewable in single metric viewer

* update jest test

x-pack/plugins/ml/common/util/job_utils.ts

@@ -78,6 +78,18 @@ export function isTimeSeriesViewDetector(job: CombinedJob, detectorIndex: number
   );
 }
 
+// Returns a flag to indicate whether the specified job is suitable for embedded map viewing.
+export function isMappableJob(job: CombinedJob, detectorIndex: number): boolean {
+  let isMappable = false;
+  const { detectors } = job.analysis_config;
+  if (detectorIndex >= 0 && detectorIndex < detectors.length) {
+    const dtr = detectors[detectorIndex];
+    const functionName = dtr.function;
+    isMappable = functionName === ML_JOB_AGGREGATION.LAT_LONG;
+  }
+  return isMappable;
+}
+
 // Returns a flag to indicate whether the source data can be plotted in a time
 // series chart for the specified detector.
 export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex: number): boolean {

x-pack/plugins/ml/public/application/components/ml_embedded_map/ml_embedded_map.tsx

@@ -8,6 +8,7 @@ import React, { useEffect, useRef, useState } from 'react';
 
 import { htmlIdGenerator } from '@elastic/eui';
 import { LayerDescriptor } from '../../../../../maps/common/descriptor_types';
+import { INITIAL_LOCATION } from '../../../../../maps/common/constants';
 import {
   MapEmbeddable,
   MapEmbeddableInput,
@@ -81,21 +82,13 @@ export function MlEmbeddedMapComponent({
         viewMode: ViewMode.VIEW,
         isLayerTOCOpen: false,
         hideFilterActions: true,
-        // Zoom Lat/Lon values are set to make sure map is in center in the panel
-        // It will also omit Greenland/Antarctica etc. NOTE: Can be removed when initialLocation is set
-        mapCenter: {
-          lon: 11,
-          lat: 20,
-          zoom: 1,
-        },
         // can use mapSettings to center map on anomalies
         mapSettings: {
           disableInteractive: false,
           hideToolbarOverlay: false,
           hideLayerControl: false,
           hideViewControl: false,
-          // Doesn't currently work with GEO_JSON. Will uncomment when https://github.com/elastic/kibana/pull/88294 is in
+          initialLocation: INITIAL_LOCATION.AUTO_FIT_TO_BOUNDS, // this will startup based on data-extent
-          // initialLocation: INITIAL_LOCATION.AUTO_FIT_TO_BOUNDS, // this will startup based on data-extent
           autoFitToDataBounds: true, // this will auto-fit when there are changes to the filter and/or query
         },
       };

x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_embedded_map.tsx

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useState, useEffect } from 'react';
+import { Dictionary } from '../../../../common/types/common';
+import { LayerDescriptor } from '../../../../../maps/common/descriptor_types';
+import { getMLAnomaliesActualLayer, getMLAnomaliesTypicalLayer } from './map_config';
+import { MlEmbeddedMapComponent } from '../../components/ml_embedded_map';
+interface Props {
+  seriesConfig: Dictionary<any>;
+}
+
+export function EmbeddedMapComponentWrapper({ seriesConfig }: Props) {
+  const [layerList, setLayerList] = useState<LayerDescriptor[]>([]);
+
+  useEffect(() => {
+    if (seriesConfig.mapData && seriesConfig.mapData.length > 0) {
+      setLayerList([
+        getMLAnomaliesActualLayer(seriesConfig.mapData),
+        getMLAnomaliesTypicalLayer(seriesConfig.mapData),
+      ]);
+    }
+  }, [seriesConfig]);
+
+  return (
+    <div data-test-subj="xpack.ml.explorer.embeddedMap" style={{ width: '100%', height: 300 }}>
+      <MlEmbeddedMapComponent layerList={layerList} />
+    </div>
+  );
+}

x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js

@@ -22,6 +22,7 @@ import {
 } from '../../util/chart_utils';
 import { ExplorerChartDistribution } from './explorer_chart_distribution';
 import { ExplorerChartSingleMetric } from './explorer_chart_single_metric';
+import { EmbeddedMapComponentWrapper } from './explorer_chart_embedded_map';
 import { ExplorerChartLabel } from './components/explorer_chart_label';
 
 import { CHART_TYPE } from '../explorer_constants';
@@ -30,6 +31,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { MlTooltipComponent } from '../../components/chart_tooltip';
 import { withKibana } from '../../../../../../../src/plugins/kibana_react/public';
 import { ML_APP_URL_GENERATOR } from '../../../../common/constants/ml_url_generator';
+import { ML_JOB_AGGREGATION } from '../../../../common/constants/aggregation_types';
 import { addItemToRecentlyAccessed } from '../../util/recently_accessed';
 import { ExplorerChartsErrorCallOuts } from './explorer_charts_error_callouts';
 
@@ -43,6 +45,9 @@ const textViewButton = i18n.translate(
     defaultMessage: 'Open in Single Metric Viewer',
   }
 );
+const mapsPluginMessage = i18n.translate('xpack.ml.explorer.charts.mapsPluginMissingMessage', {
+  defaultMessage: 'maps or embeddable start plugin not found',
+});
 
 // create a somewhat unique ID
 // from charts metadata for React's key attribute
@@ -67,8 +72,8 @@ function ExplorerChartContainer({
   useEffect(() => {
     let isCancelled = false;
     const generateLink = async () => {
-      const singleMetricViewerLink = await getExploreSeriesLink(mlUrlGenerator, series);
+      if (!isCancelled && series.functionDescription !== ML_JOB_AGGREGATION.LAT_LONG) {
-      if (!isCancelled) {
+        const singleMetricViewerLink = await getExploreSeriesLink(mlUrlGenerator, series);
         setExplorerSeriesLink(singleMetricViewerLink);
       }
     };
@@ -150,6 +155,18 @@ function ExplorerChartContainer({
         </EuiFlexItem>
       </EuiFlexGroup>
       {(() => {
+        if (chartType === CHART_TYPE.GEO_MAP) {
+          return (
+            <MlTooltipComponent>
+              {(tooltipService) => (
+                <EmbeddedMapComponentWrapper
+                  seriesConfig={series}
+                  tooltipService={tooltipService}
+                />
+              )}
+            </MlTooltipComponent>
+          );
+        }
         if (
           chartType === CHART_TYPE.EVENT_DISTRIBUTION ||
           chartType === CHART_TYPE.POPULATION_DISTRIBUTION
@@ -167,18 +184,20 @@ function ExplorerChartContainer({
             </MlTooltipComponent>
           );
         }
-        return (
+        if (chartType === CHART_TYPE.SINGLE_METRIC) {
-          <MlTooltipComponent>
+          return (
-            {(tooltipService) => (
+            <MlTooltipComponent>
-              <ExplorerChartSingleMetric
+              {(tooltipService) => (
-                tooManyBuckets={tooManyBuckets}
+                <ExplorerChartSingleMetric
-                seriesConfig={series}
+                  tooManyBuckets={tooManyBuckets}
-                severity={severity}
+                  seriesConfig={series}
-                tooltipService={tooltipService}
+                  severity={severity}
-              />
+                  tooltipService={tooltipService}
-            )}
+                />
-          </MlTooltipComponent>
+              )}
-        );
+            </MlTooltipComponent>
+          );
+        }
       })()}
     </React.Fragment>
   );
@@ -199,8 +218,31 @@ export const ExplorerChartsContainerUI = ({
       share: {
         urlGenerators: { getUrlGenerator },
       },
+      embeddable: embeddablePlugin,
+      maps: mapsPlugin,
     },
   } = kibana;
+
+  let seriesToPlotFiltered;
+
+  if (!embeddablePlugin || !mapsPlugin) {
+    seriesToPlotFiltered = [];
+    // Show missing plugin callout
+    seriesToPlot.forEach((series) => {
+      if (series.functionDescription === 'lat_long') {
+        if (errorMessages[mapsPluginMessage] === undefined) {
+          errorMessages[mapsPluginMessage] = new Set([series.jobId]);
+        } else {
+          errorMessages[mapsPluginMessage].add(series.jobId);
+        }
+      } else {
+        seriesToPlotFiltered.push(series);
+      }
+    });
+  }
+
+  const seriesToUse = seriesToPlotFiltered !== undefined ? seriesToPlotFiltered : seriesToPlot;
+
   const mlUrlGenerator = useMemo(() => getUrlGenerator(ML_APP_URL_GENERATOR), [getUrlGenerator]);
 
   // <EuiFlexGrid> doesn't allow a setting of `columns={1}` when chartsPerRow would be 1.
@@ -208,13 +250,13 @@ export const ExplorerChartsContainerUI = ({
   const chartsWidth = chartsPerRow === 1 ? 'calc(100% - 20px)' : 'auto';
   const chartsColumns = chartsPerRow === 1 ? 0 : chartsPerRow;
 
-  const wrapLabel = seriesToPlot.some((series) => isLabelLengthAboveThreshold(series));
+  const wrapLabel = seriesToUse.some((series) => isLabelLengthAboveThreshold(series));
   return (
     <>
       <ExplorerChartsErrorCallOuts errorMessagesByType={errorMessages} />
       <EuiFlexGrid columns={chartsColumns}>
-        {seriesToPlot.length > 0 &&
+        {seriesToUse.length > 0 &&
-          seriesToPlot.map((series) => (
+          seriesToUse.map((series) => (
             <EuiFlexItem
               key={getChartId(series)}
               className="ml-explorer-chart-container"

x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js

@@ -22,12 +22,14 @@ import {
   isSourceDataChartableForDetector,
   isModelPlotChartableForDetector,
   isModelPlotEnabled,
+  isMappableJob,
 } from '../../../../common/util/job_utils';
 import { mlResultsService } from '../../services/results_service';
 import { mlJobService } from '../../services/job_service';
 import { explorerService } from '../explorer_dashboard_service';
 
 import { CHART_TYPE } from '../explorer_constants';
+import { ML_JOB_AGGREGATION } from '../../../../common/constants/aggregation_types';
 import { i18n } from '@kbn/i18n';
 import { SWIM_LANE_LABEL_WIDTH } from '../swimlane_container';
 
@@ -77,7 +79,50 @@ export const anomalyDataChange = function (
   // For now just take first 6 (or 8 if 4 charts per row).
   const maxSeriesToPlot = Math.max(chartsPerRow * 2, 6);
   const recordsToPlot = allSeriesRecords.slice(0, maxSeriesToPlot);
+  const hasGeoData = recordsToPlot.find(
+    (record) =>
+      (record.function_description || recordsToPlot.function) === ML_JOB_AGGREGATION.LAT_LONG
+  );
+
   const seriesConfigs = recordsToPlot.map(buildConfig);
+  const seriesConfigsNoGeoData = [];
+
+  // initialize the charts with loading indicators
+  data.seriesToPlot = seriesConfigs.map((config) => ({
+    ...config,
+    loading: true,
+    chartData: null,
+  }));
+
+  const mapData = [];
+
+  if (hasGeoData !== undefined) {
+    for (let i = 0; i < seriesConfigs.length; i++) {
+      const config = seriesConfigs[i];
+      let records;
+      if (config.detectorLabel.includes(ML_JOB_AGGREGATION.LAT_LONG)) {
+        if (config.entityFields.length) {
+          records = [
+            recordsToPlot.find((record) => {
+              const entityFieldName = config.entityFields[0].fieldName;
+              const entityFieldValue = config.entityFields[0].fieldValue;
+              return (record[entityFieldName] && record[entityFieldName][0]) === entityFieldValue;
+            }),
+          ];
+        } else {
+          records = recordsToPlot;
+        }
+
+        mapData.push({
+          ...config,
+          loading: false,
+          mapData: records,
+        });
+      } else {
+        seriesConfigsNoGeoData.push(config);
+      }
+    }
+  }
 
   // Calculate the time range of the charts, which is a function of the chart width and max job bucket span.
   data.tooManyBuckets = false;
@@ -92,13 +137,6 @@ export const anomalyDataChange = function (
   );
   data.tooManyBuckets = tooManyBuckets;
 
-  // initialize the charts with loading indicators
-  data.seriesToPlot = seriesConfigs.map((config) => ({
-    ...config,
-    loading: true,
-    chartData: null,
-  }));
-
   data.errorMessages = errorMessages;
 
   explorerService.setCharts({ ...data });
@@ -269,22 +307,27 @@ export const anomalyDataChange = function (
   // only after that trigger data processing and page render.
   // TODO - if query returns no results e.g. source data has been deleted,
   // display a message saying 'No data between earliest/latest'.
-  const seriesPromises = seriesConfigs.map((seriesConfig) =>
+  const seriesPromises = [];
-    Promise.all([
+  // Use seriesConfigs list without geo data config so indices match up after seriesPromises are resolved and we map through the responses
-      getMetricData(seriesConfig, chartRange),
+  const seriesCongifsForPromises = hasGeoData ? seriesConfigsNoGeoData : seriesConfigs;
-      getRecordsForCriteria(seriesConfig, chartRange),
+  seriesCongifsForPromises.forEach((seriesConfig) => {
-      getScheduledEvents(seriesConfig, chartRange),
+    seriesPromises.push(
-      getEventDistribution(seriesConfig, chartRange),
+      Promise.all([
-    ])
+        getMetricData(seriesConfig, chartRange),
-  );
+        getRecordsForCriteria(seriesConfig, chartRange),
+        getScheduledEvents(seriesConfig, chartRange),
+        getEventDistribution(seriesConfig, chartRange),
+      ])
+    );
+  });
 
   function processChartData(response, seriesIndex) {
     const metricData = response[0].results;
     const records = response[1].records;
-    const jobId = seriesConfigs[seriesIndex].jobId;
+    const jobId = seriesCongifsForPromises[seriesIndex].jobId;
     const scheduledEvents = response[2].events[jobId];
     const eventDistribution = response[3];
-    const chartType = getChartType(seriesConfigs[seriesIndex]);
+    const chartType = getChartType(seriesCongifsForPromises[seriesIndex]);
 
     // Sort records in ascending time order matching up with chart data
     records.sort((recordA, recordB) => {
@@ -409,16 +452,25 @@ export const anomalyDataChange = function (
       );
       const overallChartLimits = chartLimits(allDataPoints);
 
-      data.seriesToPlot = response.map((d, i) => ({
+      data.seriesToPlot = response.map((d, i) => {
-        ...seriesConfigs[i],
+        return {
-        loading: false,
+          ...seriesCongifsForPromises[i],
-        chartData: processedData[i],
+          loading: false,
-        plotEarliest: chartRange.min,
+          chartData: processedData[i],
-        plotLatest: chartRange.max,
+          plotEarliest: chartRange.min,
-        selectedEarliest: selectedEarliestMs,
+          plotLatest: chartRange.max,
-        selectedLatest: selectedLatestMs,
+          selectedEarliest: selectedEarliestMs,
-        chartLimits: USE_OVERALL_CHART_LIMITS ? overallChartLimits : chartLimits(processedData[i]),
+          selectedLatest: selectedLatestMs,
-      }));
+          chartLimits: USE_OVERALL_CHART_LIMITS
+            ? overallChartLimits
+            : chartLimits(processedData[i]),
+        };
+      });
+
+      if (mapData.length) {
+        // push map data in if it's available
+        data.seriesToPlot.push(...mapData);
+      }
       explorerService.setCharts({ ...data });
     })
     .catch((error) => {
@@ -447,7 +499,10 @@ function processRecordsForDisplay(anomalyRecords) {
       return;
     }
 
-    let isChartable = isSourceDataChartableForDetector(job, record.detector_index);
+    let isChartable =
+      isSourceDataChartableForDetector(job, record.detector_index) ||
+      isMappableJob(job, record.detector_index);
+
     if (isChartable === false) {
       if (isModelPlotChartableForDetector(job, record.detector_index)) {
         // Check if model plot is enabled for this job.

x-pack/plugins/ml/public/application/explorer/explorer_charts/map_config.ts

@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FIELD_ORIGIN, STYLE_TYPE } from '../../../../../maps/common/constants';
+import { ANOMALY_THRESHOLD, SEVERITY_COLORS } from '../../../../common';
+
+const FEATURE = 'Feature';
+const POINT = 'Point';
+const SEVERITY_COLOR_RAMP = [
+  {
+    stop: ANOMALY_THRESHOLD.LOW,
+    color: SEVERITY_COLORS.WARNING,
+  },
+  {
+    stop: ANOMALY_THRESHOLD.MINOR,
+    color: SEVERITY_COLORS.MINOR,
+  },
+  {
+    stop: ANOMALY_THRESHOLD.MAJOR,
+    color: SEVERITY_COLORS.MAJOR,
+  },
+  {
+    stop: ANOMALY_THRESHOLD.CRITICAL,
+    color: SEVERITY_COLORS.CRITICAL,
+  },
+];
+
+function getAnomalyFeatures(anomalies: any[], type: 'actual_point' | 'typical_point') {
+  const anomalyFeatures = [];
+  for (let i = 0; i < anomalies.length; i++) {
+    const anomaly = anomalies[i];
+    const geoResults = anomaly.geo_results || (anomaly?.causes && anomaly?.causes[0]?.geo_results);
+    const coordinateStr = geoResults && geoResults[type];
+    if (coordinateStr !== undefined) {
+      // Must reverse coordinates here. Map expects [lon, lat] - anomalies are stored as [lat, lon] for lat_lon jobs
+      const coordinates = coordinateStr
+        .split(',')
+        .map((point: string) => Number(point))
+        .reverse();
+
+      anomalyFeatures.push({
+        type: FEATURE,
+        geometry: {
+          type: POINT,
+          coordinates,
+        },
+        properties: {
+          record_score: Math.floor(anomaly.record_score),
+          [type]: coordinates.map((point: number) => point.toFixed(2)),
+        },
+      });
+    }
+  }
+  return anomalyFeatures;
+}
+
+export const getMLAnomaliesTypicalLayer = (anomalies: any) => {
+  return {
+    id: 'anomalies_typical_layer',
+    label: 'Typical',
+    sourceDescriptor: {
+      id: 'b7486535-171b-4d3b-bb2e-33c1a0a2854e',
+      type: 'GEOJSON_FILE',
+      __featureCollection: {
+        features: getAnomalyFeatures(anomalies, 'typical_point'),
+        type: 'FeatureCollection',
+      },
+    },
+    visible: true,
+    style: {
+      type: 'VECTOR',
+      properties: {
+        fillColor: {
+          type: 'STATIC',
+          options: {
+            color: '#98A2B2',
+          },
+        },
+        lineColor: {
+          type: 'STATIC',
+          options: {
+            color: '#fff',
+          },
+        },
+        lineWidth: {
+          type: 'STATIC',
+          options: {
+            size: 2,
+          },
+        },
+        iconSize: {
+          type: 'STATIC',
+          options: {
+            size: 6,
+          },
+        },
+      },
+    },
+    type: 'VECTOR',
+  };
+};
+
+export const getMLAnomaliesActualLayer = (anomalies: any) => {
+  return {
+    id: 'anomalies_actual_layer',
+    label: 'Actual',
+    sourceDescriptor: {
+      id: 'b7486535-171b-4d3b-bb2e-33c1a0a2854d',
+      type: 'GEOJSON_FILE',
+      __fields: [
+        {
+          name: 'record_score',
+          type: 'number',
+        },
+      ],
+      __featureCollection: {
+        features: getAnomalyFeatures(anomalies, 'actual_point'),
+        type: 'FeatureCollection',
+      },
+    },
+    visible: true,
+    style: {
+      type: 'VECTOR',
+      properties: {
+        fillColor: {
+          type: STYLE_TYPE.DYNAMIC,
+          options: {
+            customColorRamp: SEVERITY_COLOR_RAMP,
+            field: {
+              name: 'record_score',
+              origin: FIELD_ORIGIN.SOURCE,
+            },
+            useCustomColorRamp: true,
+          },
+        },
+        lineColor: {
+          type: 'STATIC',
+          options: {
+            color: '#fff',
+          },
+        },
+        lineWidth: {
+          type: 'STATIC',
+          options: {
+            size: 2,
+          },
+        },
+        iconSize: {
+          type: 'STATIC',
+          options: {
+            size: 6,
+          },
+        },
+      },
+    },
+    type: 'VECTOR',
+  };
+};

x-pack/plugins/ml/public/application/explorer/explorer_constants.ts

@@ -48,6 +48,7 @@ export const CHART_TYPE = {
   EVENT_DISTRIBUTION: 'event_distribution',
   POPULATION_DISTRIBUTION: 'population_distribution',
   SINGLE_METRIC: 'single_metric',
+  GEO_MAP: 'geo_map',
 };
 
 export const MAX_CATEGORY_EXAMPLES = 10;

x-pack/plugins/ml/public/application/explorer/explorer_utils.js

@@ -511,6 +511,7 @@ export async function loadAnomaliesTableData(
             const entityFields = getEntityFieldList(anomaly.source);
             isChartable = isModelPlotEnabled(job, anomaly.detectorIndex, entityFields);
           }
+
           anomaly.isTimeSeriesViewRecord = isChartable;
 
           if (mlJobService.customUrlsByJob[jobId] !== undefined) {

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/descriptions.tsx

@@ -46,7 +46,7 @@ export const FieldDescription: FC = memo(({ children }) => {
       description={
         <FormattedMessage
           id="xpack.ml.newJob.wizard.pickFieldsStep.advancedDetectorModal.fieldSelect.description"
-          defaultMessage="Required for functions: sum, mean, median, max, min, info_content, distinct_count."
+          defaultMessage="Required for functions: sum, mean, median, max, min, info_content, distinct_count, lat_long."
         />
       }
     >

x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts

@@ -156,7 +156,8 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
       }
 
       body.aggs.byTime.aggs = {};
-      if (metricFieldName !== undefined && metricFieldName !== '') {
+
+      if (metricFieldName !== undefined && metricFieldName !== '' && metricFunction) {
         const metricAgg: any = {
           [metricFunction]: {},
         };

x-pack/plugins/ml/public/application/util/chart_config_builder.js

@@ -24,7 +24,10 @@ export function buildConfigFromDetector(job, detectorIndex) {
   const config = {
     jobId: job.job_id,
     detectorIndex: detectorIndex,
-    metricFunction: mlFunctionToESAggregation(detector.function),
+    metricFunction:
+      detector.function === ML_JOB_AGGREGATION.LAT_LONG
+        ? ML_JOB_AGGREGATION.LAT_LONG
+        : mlFunctionToESAggregation(detector.function),
     timeField: job.data_description.time_field,
     interval: job.analysis_config.bucket_span,
     datafeedConfig: job.datafeed_config,

x-pack/plugins/ml/public/application/util/chart_utils.js

@@ -176,6 +176,11 @@ const POPULATION_DISTRIBUTION_ENABLED = true;
 // get the chart type based on its configuration
 export function getChartType(config) {
   let chartType = CHART_TYPE.SINGLE_METRIC;
+
+  if (config.functionDescription === 'lat_long' || config.mapData !== undefined) {
+    return CHART_TYPE.GEO_MAP;
+  }
+
   if (
     EVENT_DISTRIBUTION_ENABLED &&
     config.functionDescription === 'rare' &&