maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Anomaly Detection: add anomalies map to explorer for jobs with 'lat_long' function (#88416)

Browse source 
* wip: create embedded map component for explorer

* add embeddedMap component to explorer

* use geo_results

* remove charts callout when map is shown

* add translation, round geo coordinates

* create GEO_MAP chart type and move embedded map to charts area

* remove embedded map that is no longer used

* fix type and fail silently if plugin not available

* fix multiple type of jobs charts view

* fix tooltip function and remove single viewer link for latlong

* ensure diff types of jobs show correct charts. fix jest test

* show errorCallout if maps not enabled and is lat_long job

* use shared MlEmbeddedMapComponent in explorer

* ensure latLong jobs not viewable in single metric viewer

* update jest test
This commit is contained in:
Melissa Alvarez 2021-01-29 10:42:35 -05:00 committed by GitHub
parent e7cbdd3050
commit a08895dbfc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 364 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
x-pack/plugins/ml/common/util/job_utils.ts
View file

@ -78,6 +78,18 @@ export function isTimeSeriesViewDetector(job: CombinedJob, detectorIndex: number
);
}
// Returns a flag to indicate whether the specified job is suitable for embedded map viewing.
export function isMappableJob(job: CombinedJob, detectorIndex: number): boolean {
let isMappable = false;
const { detectors } = job.analysis_config;
if (detectorIndex >= 0 && detectorIndex < detectors.length) {
const dtr = detectors[detectorIndex];
const functionName = dtr.function;
isMappable = functionName === ML_JOB_AGGREGATION.LAT_LONG;
}
return isMappable;
}
// Returns a flag to indicate whether the source data can be plotted in a time
// series chart for the specified detector.
export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex: number): boolean {

11
x-pack/plugins/ml/public/application/components/ml_embedded_map/ml_embedded_map.tsx
View file

@ -8,6 +8,7 @@ import React, { useEffect, useRef, useState } from 'react';
import { htmlIdGenerator } from '@elastic/eui';
import { LayerDescriptor } from '../../../../../maps/common/descriptor_types';
import { INITIAL_LOCATION } from '../../../../../maps/common/constants';
import {
MapEmbeddable,
MapEmbeddableInput,
@ -81,21 +82,13 @@ export function MlEmbeddedMapComponent({
viewMode: ViewMode.VIEW,
isLayerTOCOpen: false,
hideFilterActions: true,
// Zoom Lat/Lon values are set to make sure map is in center in the panel
// It will also omit Greenland/Antarctica etc. NOTE: Can be removed when initialLocation is set
mapCenter: {
lon: 11,
lat: 20,
zoom: 1,
},
// can use mapSettings to center map on anomalies
mapSettings: {
disableInteractive: false,
hideToolbarOverlay: false,
hideLayerControl: false,
hideViewControl: false,
// Doesn't currently work with GEO_JSON. Will uncomment when https://github.com/elastic/kibana/pull/88294 is in
// initialLocation: INITIAL_LOCATION.AUTO_FIT_TO_BOUNDS, // this will startup based on data-extent
initialLocation: INITIAL_LOCATION.AUTO_FIT_TO_BOUNDS, // this will startup based on data-extent
autoFitToDataBounds: true, // this will auto-fit when there are changes to the filter and/or query
},
};

33
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_embedded_map.tsx Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useState, useEffect } from 'react';
import { Dictionary } from '../../../../common/types/common';
import { LayerDescriptor } from '../../../../../maps/common/descriptor_types';
import { getMLAnomaliesActualLayer, getMLAnomaliesTypicalLayer } from './map_config';
import { MlEmbeddedMapComponent } from '../../components/ml_embedded_map';
interface Props {
seriesConfig: Dictionary<any>;
}
export function EmbeddedMapComponentWrapper({ seriesConfig }: Props) {
const [layerList, setLayerList] = useState<LayerDescriptor[]>([]);
useEffect(() => {
if (seriesConfig.mapData && seriesConfig.mapData.length > 0) {
setLayerList([
getMLAnomaliesActualLayer(seriesConfig.mapData),
getMLAnomaliesTypicalLayer(seriesConfig.mapData),
]);
}
}, [seriesConfig]);
return (
<div data-test-subj="xpack.ml.explorer.embeddedMap" style={{ width: '100%', height: 300 }}>
<MlEmbeddedMapComponent layerList={layerList} />
</div>
);
}

76
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js
View file

@ -22,6 +22,7 @@ import {
} from '../../util/chart_utils';
import { ExplorerChartDistribution } from './explorer_chart_distribution';
import { ExplorerChartSingleMetric } from './explorer_chart_single_metric';
import { EmbeddedMapComponentWrapper } from './explorer_chart_embedded_map';
import { ExplorerChartLabel } from './components/explorer_chart_label';
import { CHART_TYPE } from '../explorer_constants';
@ -30,6 +31,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
import { MlTooltipComponent } from '../../components/chart_tooltip';
import { withKibana } from '../../../../../../../src/plugins/kibana_react/public';
import { ML_APP_URL_GENERATOR } from '../../../../common/constants/ml_url_generator';
import { ML_JOB_AGGREGATION } from '../../../../common/constants/aggregation_types';
import { addItemToRecentlyAccessed } from '../../util/recently_accessed';
import { ExplorerChartsErrorCallOuts } from './explorer_charts_error_callouts';
@ -43,6 +45,9 @@ const textViewButton = i18n.translate(
defaultMessage: 'Open in Single Metric Viewer',
}
);
const mapsPluginMessage = i18n.translate('xpack.ml.explorer.charts.mapsPluginMissingMessage', {
defaultMessage: 'maps or embeddable start plugin not found',
});
// create a somewhat unique ID
// from charts metadata for React's key attribute
@ -67,8 +72,8 @@ function ExplorerChartContainer({
useEffect(() => {
let isCancelled = false;
const generateLink = async () => {
const singleMetricViewerLink = await getExploreSeriesLink(mlUrlGenerator, series);
if (!isCancelled) {
if (!isCancelled && series.functionDescription !== ML_JOB_AGGREGATION.LAT_LONG) {
const singleMetricViewerLink = await getExploreSeriesLink(mlUrlGenerator, series);
setExplorerSeriesLink(singleMetricViewerLink);
}
};
@ -150,6 +155,18 @@ function ExplorerChartContainer({
</EuiFlexItem>
</EuiFlexGroup>
{(() => {
if (chartType === CHART_TYPE.GEO_MAP) {
return (
<MlTooltipComponent>
{(tooltipService) => (
<EmbeddedMapComponentWrapper
seriesConfig={series}
tooltipService={tooltipService}
/>
)}
</MlTooltipComponent>
);
}
if (
chartType === CHART_TYPE.EVENT_DISTRIBUTION ||
chartType === CHART_TYPE.POPULATION_DISTRIBUTION
@ -167,18 +184,20 @@ function ExplorerChartContainer({
</MlTooltipComponent>
);
}
return (
<MlTooltipComponent>
{(tooltipService) => (
<ExplorerChartSingleMetric
tooManyBuckets={tooManyBuckets}
seriesConfig={series}
severity={severity}
tooltipService={tooltipService}
/>
)}
</MlTooltipComponent>
);
if (chartType === CHART_TYPE.SINGLE_METRIC) {
return (
<MlTooltipComponent>
{(tooltipService) => (
<ExplorerChartSingleMetric
tooManyBuckets={tooManyBuckets}
seriesConfig={series}
severity={severity}
tooltipService={tooltipService}
/>
)}
</MlTooltipComponent>
);
}
})()}
</React.Fragment>
);
@ -199,8 +218,31 @@ export const ExplorerChartsContainerUI = ({
share: {
urlGenerators: { getUrlGenerator },
},
embeddable: embeddablePlugin,
maps: mapsPlugin,
},
} = kibana;
let seriesToPlotFiltered;
if (!embeddablePlugin || !mapsPlugin) {
seriesToPlotFiltered = [];
// Show missing plugin callout
seriesToPlot.forEach((series) => {
if (series.functionDescription === 'lat_long') {
if (errorMessages[mapsPluginMessage] === undefined) {
errorMessages[mapsPluginMessage] = new Set([series.jobId]);
} else {
errorMessages[mapsPluginMessage].add(series.jobId);
}
} else {
seriesToPlotFiltered.push(series);
}
});
}
const seriesToUse = seriesToPlotFiltered !== undefined ? seriesToPlotFiltered : seriesToPlot;
const mlUrlGenerator = useMemo(() => getUrlGenerator(ML_APP_URL_GENERATOR), [getUrlGenerator]);
// <EuiFlexGrid> doesn't allow a setting of `columns={1}` when chartsPerRow would be 1.
@ -208,13 +250,13 @@ export const ExplorerChartsContainerUI = ({
const chartsWidth = chartsPerRow === 1 ? 'calc(100% - 20px)' : 'auto';
const chartsColumns = chartsPerRow === 1 ? 0 : chartsPerRow;
const wrapLabel = seriesToPlot.some((series) => isLabelLengthAboveThreshold(series));
const wrapLabel = seriesToUse.some((series) => isLabelLengthAboveThreshold(series));
return (
<>
<ExplorerChartsErrorCallOuts errorMessagesByType={errorMessages} />
<EuiFlexGrid columns={chartsColumns}>
{seriesToPlot.length > 0 &&
seriesToPlot.map((series) => (
{seriesToUse.length > 0 &&
seriesToUse.map((series) => (
<EuiFlexItem
key={getChartId(series)}
className="ml-explorer-chart-container"

111
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js
View file

@ -22,12 +22,14 @@ import {
isSourceDataChartableForDetector,
isModelPlotChartableForDetector,
isModelPlotEnabled,
isMappableJob,
} from '../../../../common/util/job_utils';
import { mlResultsService } from '../../services/results_service';
import { mlJobService } from '../../services/job_service';
import { explorerService } from '../explorer_dashboard_service';
import { CHART_TYPE } from '../explorer_constants';
import { ML_JOB_AGGREGATION } from '../../../../common/constants/aggregation_types';
import { i18n } from '@kbn/i18n';
import { SWIM_LANE_LABEL_WIDTH } from '../swimlane_container';
@ -77,7 +79,50 @@ export const anomalyDataChange = function (
// For now just take first 6 (or 8 if 4 charts per row).
const maxSeriesToPlot = Math.max(chartsPerRow * 2, 6);
const recordsToPlot = allSeriesRecords.slice(0, maxSeriesToPlot);
const hasGeoData = recordsToPlot.find(
(record) =>
(record.function_description || recordsToPlot.function) === ML_JOB_AGGREGATION.LAT_LONG
);
const seriesConfigs = recordsToPlot.map(buildConfig);
const seriesConfigsNoGeoData = [];
// initialize the charts with loading indicators
data.seriesToPlot = seriesConfigs.map((config) => ({
...config,
loading: true,
chartData: null,
}));
const mapData = [];
if (hasGeoData !== undefined) {
for (let i = 0; i < seriesConfigs.length; i++) {
const config = seriesConfigs[i];
let records;
if (config.detectorLabel.includes(ML_JOB_AGGREGATION.LAT_LONG)) {
if (config.entityFields.length) {
records = [
recordsToPlot.find((record) => {
const entityFieldName = config.entityFields[0].fieldName;
const entityFieldValue = config.entityFields[0].fieldValue;
return (record[entityFieldName] && record[entityFieldName][0]) === entityFieldValue;
}),
];
} else {
records = recordsToPlot;
}
mapData.push({
...config,
loading: false,
mapData: records,
});
} else {
seriesConfigsNoGeoData.push(config);
}
}
}
// Calculate the time range of the charts, which is a function of the chart width and max job bucket span.
data.tooManyBuckets = false;
@ -92,13 +137,6 @@ export const anomalyDataChange = function (
);
data.tooManyBuckets = tooManyBuckets;
// initialize the charts with loading indicators
data.seriesToPlot = seriesConfigs.map((config) => ({
...config,
loading: true,
chartData: null,
}));
data.errorMessages = errorMessages;
explorerService.setCharts({ ...data });
@ -269,22 +307,27 @@ export const anomalyDataChange = function (
// only after that trigger data processing and page render.
// TODO - if query returns no results e.g. source data has been deleted,
// display a message saying 'No data between earliest/latest'.
const seriesPromises = seriesConfigs.map((seriesConfig) =>
Promise.all([
getMetricData(seriesConfig, chartRange),
getRecordsForCriteria(seriesConfig, chartRange),
getScheduledEvents(seriesConfig, chartRange),
getEventDistribution(seriesConfig, chartRange),
])
);
const seriesPromises = [];
// Use seriesConfigs list without geo data config so indices match up after seriesPromises are resolved and we map through the responses
const seriesCongifsForPromises = hasGeoData ? seriesConfigsNoGeoData : seriesConfigs;
seriesCongifsForPromises.forEach((seriesConfig) => {
seriesPromises.push(
Promise.all([
getMetricData(seriesConfig, chartRange),
getRecordsForCriteria(seriesConfig, chartRange),
getScheduledEvents(seriesConfig, chartRange),
getEventDistribution(seriesConfig, chartRange),
])
);
});
function processChartData(response, seriesIndex) {
const metricData = response[0].results;
const records = response[1].records;
const jobId = seriesConfigs[seriesIndex].jobId;
const jobId = seriesCongifsForPromises[seriesIndex].jobId;
const scheduledEvents = response[2].events[jobId];
const eventDistribution = response[3];
const chartType = getChartType(seriesConfigs[seriesIndex]);
const chartType = getChartType(seriesCongifsForPromises[seriesIndex]);
// Sort records in ascending time order matching up with chart data
records.sort((recordA, recordB) => {
@ -409,16 +452,25 @@ export const anomalyDataChange = function (
);
const overallChartLimits = chartLimits(allDataPoints);
data.seriesToPlot = response.map((d, i) => ({
...seriesConfigs[i],
loading: false,
chartData: processedData[i],
plotEarliest: chartRange.min,
plotLatest: chartRange.max,
selectedEarliest: selectedEarliestMs,
selectedLatest: selectedLatestMs,
chartLimits: USE_OVERALL_CHART_LIMITS ? overallChartLimits : chartLimits(processedData[i]),
}));
data.seriesToPlot = response.map((d, i) => {
return {
...seriesCongifsForPromises[i],
loading: false,
chartData: processedData[i],
plotEarliest: chartRange.min,
plotLatest: chartRange.max,
selectedEarliest: selectedEarliestMs,
selectedLatest: selectedLatestMs,
chartLimits: USE_OVERALL_CHART_LIMITS
? overallChartLimits
: chartLimits(processedData[i]),
};
});
if (mapData.length) {
// push map data in if it's available
data.seriesToPlot.push(...mapData);
}
explorerService.setCharts({ ...data });
})
.catch((error) => {
@ -447,7 +499,10 @@ function processRecordsForDisplay(anomalyRecords) {
return;
}
let isChartable = isSourceDataChartableForDetector(job, record.detector_index);
let isChartable =
isSourceDataChartableForDetector(job, record.detector_index) ||
isMappableJob(job, record.detector_index);
if (isChartable === false) {
if (isModelPlotChartableForDetector(job, record.detector_index)) {
// Check if model plot is enabled for this job.

161
x-pack/plugins/ml/public/application/explorer/explorer_charts/map_config.ts Normal file
View file

@ -0,0 +1,161 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { FIELD_ORIGIN, STYLE_TYPE } from '../../../../../maps/common/constants';
import { ANOMALY_THRESHOLD, SEVERITY_COLORS } from '../../../../common';
const FEATURE = 'Feature';
const POINT = 'Point';
const SEVERITY_COLOR_RAMP = [
{
stop: ANOMALY_THRESHOLD.LOW,
color: SEVERITY_COLORS.WARNING,
},
{
stop: ANOMALY_THRESHOLD.MINOR,
color: SEVERITY_COLORS.MINOR,
},
{
stop: ANOMALY_THRESHOLD.MAJOR,
color: SEVERITY_COLORS.MAJOR,
},
{
stop: ANOMALY_THRESHOLD.CRITICAL,
color: SEVERITY_COLORS.CRITICAL,
},
];
function getAnomalyFeatures(anomalies: any[], type: 'actual_point' | 'typical_point') {
const anomalyFeatures = [];
for (let i = 0; i < anomalies.length; i++) {
const anomaly = anomalies[i];
const geoResults = anomaly.geo_results || (anomaly?.causes && anomaly?.causes[0]?.geo_results);
const coordinateStr = geoResults && geoResults[type];
if (coordinateStr !== undefined) {
// Must reverse coordinates here. Map expects [lon, lat] - anomalies are stored as [lat, lon] for lat_lon jobs
const coordinates = coordinateStr
.split(',')
.map((point: string) => Number(point))
.reverse();
anomalyFeatures.push({
type: FEATURE,
geometry: {
type: POINT,
coordinates,
},
properties: {
record_score: Math.floor(anomaly.record_score),
[type]: coordinates.map((point: number) => point.toFixed(2)),
},
});
}
}
return anomalyFeatures;
}
export const getMLAnomaliesTypicalLayer = (anomalies: any) => {
return {
id: 'anomalies_typical_layer',
label: 'Typical',
sourceDescriptor: {
id: 'b7486535-171b-4d3b-bb2e-33c1a0a2854e',
type: 'GEOJSON_FILE',
__featureCollection: {
features: getAnomalyFeatures(anomalies, 'typical_point'),
type: 'FeatureCollection',
},
},
visible: true,
style: {
type: 'VECTOR',
properties: {
fillColor: {
type: 'STATIC',
options: {
color: '#98A2B2',
},
},
lineColor: {
type: 'STATIC',
options: {
color: '#fff',
},
},
lineWidth: {
type: 'STATIC',
options: {
size: 2,
},
},
iconSize: {
type: 'STATIC',
options: {
size: 6,
},
},
},
},
type: 'VECTOR',
};
};
export const getMLAnomaliesActualLayer = (anomalies: any) => {
return {
id: 'anomalies_actual_layer',
label: 'Actual',
sourceDescriptor: {
id: 'b7486535-171b-4d3b-bb2e-33c1a0a2854d',
type: 'GEOJSON_FILE',
__fields: [
{
name: 'record_score',
type: 'number',
},
],
__featureCollection: {
features: getAnomalyFeatures(anomalies, 'actual_point'),
type: 'FeatureCollection',
},
},
visible: true,
style: {
type: 'VECTOR',
properties: {
fillColor: {
type: STYLE_TYPE.DYNAMIC,
options: {
customColorRamp: SEVERITY_COLOR_RAMP,
field: {
name: 'record_score',
origin: FIELD_ORIGIN.SOURCE,
},
useCustomColorRamp: true,
},
},
lineColor: {
type: 'STATIC',
options: {
color: '#fff',
},
},
lineWidth: {
type: 'STATIC',
options: {
size: 2,
},
},
iconSize: {
type: 'STATIC',
options: {
size: 6,
},
},
},
},
type: 'VECTOR',
};
};

1
x-pack/plugins/ml/public/application/explorer/explorer_constants.ts
View file

@ -48,6 +48,7 @@ export const CHART_TYPE = {
EVENT_DISTRIBUTION: 'event_distribution',
POPULATION_DISTRIBUTION: 'population_distribution',
SINGLE_METRIC: 'single_metric',
GEO_MAP: 'geo_map',
};
export const MAX_CATEGORY_EXAMPLES = 10;

1
x-pack/plugins/ml/public/application/explorer/explorer_utils.js
View file

@ -511,6 +511,7 @@ export async function loadAnomaliesTableData(
const entityFields = getEntityFieldList(anomaly.source);
isChartable = isModelPlotEnabled(job, anomaly.detectorIndex, entityFields);
}
anomaly.isTimeSeriesViewRecord = isChartable;
if (mlJobService.customUrlsByJob[jobId] !== undefined) {

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/descriptions.tsx
View file

@ -46,7 +46,7 @@ export const FieldDescription: FC = memo(({ children }) => {
description={
<FormattedMessage
id="xpack.ml.newJob.wizard.pickFieldsStep.advancedDetectorModal.fieldSelect.description"
defaultMessage="Required for functions: sum, mean, median, max, min, info_content, distinct_count."
defaultMessage="Required for functions: sum, mean, median, max, min, info_content, distinct_count, lat_long."
/>
}
>

3
x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts
View file

@ -156,7 +156,8 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
}
body.aggs.byTime.aggs = {};
if (metricFieldName !== undefined && metricFieldName !== '') {
if (metricFieldName !== undefined && metricFieldName !== '' && metricFunction) {
const metricAgg: any = {
[metricFunction]: {},
};

5
x-pack/plugins/ml/public/application/util/chart_config_builder.js
View file

@ -24,7 +24,10 @@ export function buildConfigFromDetector(job, detectorIndex) {
const config = {
jobId: job.job_id,
detectorIndex: detectorIndex,
metricFunction: mlFunctionToESAggregation(detector.function),
metricFunction:
detector.function === ML_JOB_AGGREGATION.LAT_LONG
? ML_JOB_AGGREGATION.LAT_LONG
: mlFunctionToESAggregation(detector.function),
timeField: job.data_description.time_field,
interval: job.analysis_config.bucket_span,
datafeedConfig: job.datafeed_config,

5
x-pack/plugins/ml/public/application/util/chart_utils.js
View file

@ -176,6 +176,11 @@ const POPULATION_DISTRIBUTION_ENABLED = true;
// get the chart type based on its configuration
export function getChartType(config) {
let chartType = CHART_TYPE.SINGLE_METRIC;
if (config.functionDescription === 'lat_long' || config.mapData !== undefined) {
return CHART_TYPE.GEO_MAP;
}
if (
EVENT_DISTRIBUTION_ENABLED &&
config.functionDescription === 'rare' &&