maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Extends support for anomaly charts when model plot is enabled (#34079)

Browse source 
* [ML] Extends support for anomaly charts when model plot is enabled

* [ML] Edits to util functions following review
This commit is contained in:
Pete Harverson 2019-03-29 14:12:54 +00:00 committed by GitHub
parent 979c8a750c
commit e00dae7405
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 502 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
x-pack/plugins/ml/common/util/__tests__/anomaly_utils.js
View file

@ -14,6 +14,7 @@ import {
getMultiBucketImpactLabel,
getEntityFieldName,
getEntityFieldValue,
getEntityFieldList,
showActualForFunction,
showTypicalForFunction,
isRuleSupported,
@ -349,6 +350,52 @@ describe('ML - anomaly utils', () => {
});
describe('getEntityFieldList', () => {
it('returns an empty list for a record with no by, over or partition fields', () => {
expect(getEntityFieldList(noEntityRecord)).to.be.empty();
});
it('returns correct list for a record with a by field', () => {
expect(getEntityFieldList(byEntityRecord)).to.eql([
{
fieldName: 'airline',
fieldValue: 'JZA',
fieldType: 'by'
}
]);
});
it('returns correct list for a record with a partition field', () => {
expect(getEntityFieldList(partitionEntityRecord)).to.eql([
{
fieldName: 'airline',
fieldValue: 'AAL',
fieldType: 'partition'
}
]);
});
it('returns correct list for a record with an over field', () => {
expect(getEntityFieldList(overEntityRecord)).to.eql([
{
fieldName: 'clientip',
fieldValue: '37.157.32.164',
fieldType: 'over'
}
]);
});
it('returns correct list for a record with a by and over field', () => {
expect(getEntityFieldList(rareEntityRecord)).to.eql([
{
fieldName: 'clientip',
fieldValue: '173.252.74.112',
fieldType: 'over'
}
]);
});
});
describe('showActualForFunction', () => {
it('returns true for expected function descriptions', () => {
expect(showActualForFunction('count')).to.be(true);

177
x-pack/plugins/ml/common/util/__tests__/job_utils.js
View file

@ -11,7 +11,8 @@ import {
calculateDatafeedFrequencyDefaultSeconds,
isTimeSeriesViewJob,
isTimeSeriesViewDetector,
isTimeSeriesViewFunction,
isSourceDataChartableForDetector,
isModelPlotChartableForDetector,
getPartitioningFieldNames,
isModelPlotEnabled,
isJobVersionGte,
@ -158,47 +159,145 @@ describe('ML - job utils', () => {
});
describe('isTimeSeriesViewFunction', () => {
describe('isSourceDataChartableForDetector', () => {
it('returns true for expected functions', () => {
expect(isTimeSeriesViewFunction('count')).to.be(true);
expect(isTimeSeriesViewFunction('low_count')).to.be(true);
expect(isTimeSeriesViewFunction('high_count')).to.be(true);
expect(isTimeSeriesViewFunction('non_zero_count')).to.be(true);
expect(isTimeSeriesViewFunction('low_non_zero_count')).to.be(true);
expect(isTimeSeriesViewFunction('high_non_zero_count')).to.be(true);
expect(isTimeSeriesViewFunction('distinct_count')).to.be(true);
expect(isTimeSeriesViewFunction('low_distinct_count')).to.be(true);
expect(isTimeSeriesViewFunction('high_distinct_count')).to.be(true);
expect(isTimeSeriesViewFunction('metric')).to.be(true);
expect(isTimeSeriesViewFunction('mean')).to.be(true);
expect(isTimeSeriesViewFunction('low_mean')).to.be(true);
expect(isTimeSeriesViewFunction('high_mean')).to.be(true);
expect(isTimeSeriesViewFunction('median')).to.be(true);
expect(isTimeSeriesViewFunction('low_median')).to.be(true);
expect(isTimeSeriesViewFunction('high_median')).to.be(true);
expect(isTimeSeriesViewFunction('min')).to.be(true);
expect(isTimeSeriesViewFunction('max')).to.be(true);
expect(isTimeSeriesViewFunction('sum')).to.be(true);
expect(isTimeSeriesViewFunction('low_sum')).to.be(true);
expect(isTimeSeriesViewFunction('high_sum')).to.be(true);
expect(isTimeSeriesViewFunction('non_null_sum')).to.be(true);
expect(isTimeSeriesViewFunction('low_non_null_sum')).to.be(true);
expect(isTimeSeriesViewFunction('high_non_null_sum')).to.be(true);
expect(isTimeSeriesViewFunction('rare')).to.be(true);
const job = {
analysis_config: {
detectors: [
{ function: 'count' }, // 0
{ function: 'low_count' }, // 1
{ function: 'high_count' }, // 2
{ function: 'non_zero_count' }, // 3
{ function: 'low_non_zero_count' }, // 4
{ function: 'high_non_zero_count' }, // 5
{ function: 'distinct_count' }, // 6
{ function: 'low_distinct_count' }, // 7
{ function: 'high_distinct_count' }, // 8
{ function: 'metric' }, // 9
{ function: 'mean' }, // 10
{ function: 'low_mean' }, // 11
{ function: 'high_mean' }, // 12
{ function: 'median' }, // 13
{ function: 'low_median' }, // 14
{ function: 'high_median' }, // 15
{ function: 'min' }, // 16
{ function: 'max' }, // 17
{ function: 'sum' }, // 18
{ function: 'low_sum' }, // 19
{ function: 'high_sum' }, // 20
{ function: 'non_null_sum' }, // 21
{ function: 'low_non_null_sum' }, // 22
{ function: 'high_non_null_sum' }, // 23
{ function: 'rare' }, // 24
{ function: 'count', 'by_field_name': 'mlcategory', }, // 25
{ function: 'count', 'by_field_name': 'hrd', }, // 26
{ function: 'freq_rare' }, // 27
{ function: 'info_content' }, // 28
{ function: 'low_info_content' }, // 29
{ function: 'high_info_content' }, // 30
{ function: 'varp' }, // 31
{ function: 'low_varp' }, // 32
{ function: 'high_varp' }, // 33
{ function: 'time_of_day' }, // 34
{ function: 'time_of_week' }, // 35
{ function: 'lat_long' }, // 36
{ function: 'mean', 'field_name': 'NetworkDiff' }, //37
]
},
datafeed_config: {
script_fields: {
hrd: {
script: {
inline: 'return domainSplit(doc["query"].value, params).get(1);',
lang: 'painless'
}
},
NetworkDiff: {
script: {
source: 'doc["NetworkOut"].value - doc["NetworkIn"].value',
lang: 'painless'
}
}
}
}
};
it('returns true for expected detectors', () => {
expect(isSourceDataChartableForDetector(job, 0)).to.be(true);
expect(isSourceDataChartableForDetector(job, 1)).to.be(true);
expect(isSourceDataChartableForDetector(job, 2)).to.be(true);
expect(isSourceDataChartableForDetector(job, 3)).to.be(true);
expect(isSourceDataChartableForDetector(job, 4)).to.be(true);
expect(isSourceDataChartableForDetector(job, 5)).to.be(true);
expect(isSourceDataChartableForDetector(job, 6)).to.be(true);
expect(isSourceDataChartableForDetector(job, 7)).to.be(true);
expect(isSourceDataChartableForDetector(job, 8)).to.be(true);
expect(isSourceDataChartableForDetector(job, 9)).to.be(true);
expect(isSourceDataChartableForDetector(job, 10)).to.be(true);
expect(isSourceDataChartableForDetector(job, 11)).to.be(true);
expect(isSourceDataChartableForDetector(job, 12)).to.be(true);
expect(isSourceDataChartableForDetector(job, 13)).to.be(true);
expect(isSourceDataChartableForDetector(job, 14)).to.be(true);
expect(isSourceDataChartableForDetector(job, 15)).to.be(true);
expect(isSourceDataChartableForDetector(job, 16)).to.be(true);
expect(isSourceDataChartableForDetector(job, 17)).to.be(true);
expect(isSourceDataChartableForDetector(job, 18)).to.be(true);
expect(isSourceDataChartableForDetector(job, 19)).to.be(true);
expect(isSourceDataChartableForDetector(job, 20)).to.be(true);
expect(isSourceDataChartableForDetector(job, 21)).to.be(true);
expect(isSourceDataChartableForDetector(job, 22)).to.be(true);
expect(isSourceDataChartableForDetector(job, 23)).to.be(true);
expect(isSourceDataChartableForDetector(job, 24)).to.be(true);
});
it('returns false for expected functions', () => {
expect(isTimeSeriesViewFunction('freq_rare')).to.be(false);
expect(isTimeSeriesViewFunction('info_content')).to.be(false);
expect(isTimeSeriesViewFunction('low_info_content')).to.be(false);
expect(isTimeSeriesViewFunction('high_info_content')).to.be(false);
expect(isTimeSeriesViewFunction('varp')).to.be(false);
expect(isTimeSeriesViewFunction('low_varp')).to.be(false);
expect(isTimeSeriesViewFunction('high_varp')).to.be(false);
expect(isTimeSeriesViewFunction('time_of_day')).to.be(false);
expect(isTimeSeriesViewFunction('time_of_week')).to.be(false);
expect(isTimeSeriesViewFunction('lat_long')).to.be(false);
it('returns false for expected detectors', () => {
expect(isSourceDataChartableForDetector(job, 25)).to.be(false);
expect(isSourceDataChartableForDetector(job, 26)).to.be(false);
expect(isSourceDataChartableForDetector(job, 27)).to.be(false);
expect(isSourceDataChartableForDetector(job, 28)).to.be(false);
expect(isSourceDataChartableForDetector(job, 29)).to.be(false);
expect(isSourceDataChartableForDetector(job, 30)).to.be(false);
expect(isSourceDataChartableForDetector(job, 31)).to.be(false);
expect(isSourceDataChartableForDetector(job, 32)).to.be(false);
expect(isSourceDataChartableForDetector(job, 33)).to.be(false);
expect(isSourceDataChartableForDetector(job, 34)).to.be(false);
expect(isSourceDataChartableForDetector(job, 35)).to.be(false);
expect(isSourceDataChartableForDetector(job, 36)).to.be(false);
expect(isSourceDataChartableForDetector(job, 37)).to.be(false);
});
});
describe('isModelPlotChartableForDetector', () => {
const job1 = {
analysis_config: {
detectors: [
{ function: 'count' }
]
}
};
const job2 = {
analysis_config: {
detectors: [
{ function: 'count' },
{ function: 'info_content' }
]
},
model_plot_config: {
enabled: true
}
};
it('returns false when model plot is not enabled', () => {
expect(isModelPlotChartableForDetector(job1, 0)).to.be(false);
});
it('returns true for count detector when model plot is enabled', () => {
expect(isModelPlotChartableForDetector(job2, 0)).to.be(true);
});
it('returns true for info_content detector when model plot is enabled', () => {
expect(isModelPlotChartableForDetector(job2, 1)).to.be(true);
});
});

34
x-pack/plugins/ml/common/util/anomaly_utils.js
View file

@ -173,6 +173,40 @@ export function getEntityFieldValue(record) {
return undefined;
}
// Returns the list of partitioning entity fields for the source record as a list
// of objects in the form { fieldName: airline, fieldValue: AAL, fieldType: partition }
export function getEntityFieldList(record) {
const entityFields = [];
if (record.partition_field_name !== undefined) {
entityFields.push({
fieldName: record.partition_field_name,
fieldValue: record.partition_field_value,
fieldType: 'partition'
});
}
if (record.over_field_name !== undefined) {
entityFields.push({
fieldName: record.over_field_name,
fieldValue: record.over_field_value,
fieldType: 'over'
});
}
// For jobs with by and over fields, don't add the 'by' field as this
// field will only be added to the top-level fields for record type results
// if it also an influencer over the bucket.
if (record.by_field_name !== undefined && record.over_field_name === undefined) {
entityFields.push({
fieldName: record.by_field_name,
fieldValue: record.by_field_value,
fieldType: 'by'
});
}
return entityFields;
}
// Returns whether actual values should be displayed for a record with the specified function description.
// Note that the 'function' field in a record contains what the user entered e.g. 'high_count',
// whereas the 'function_description' field holds a ML-built display hint for function e.g. 'count'.

70
x-pack/plugins/ml/common/util/job_utils.js
View file

@ -31,14 +31,10 @@ export function calculateDatafeedFrequencyDefaultSeconds(bucketSpanSeconds) {
// Returns a flag to indicate whether the job is suitable for viewing
// in the Time Series dashboard.
export function isTimeSeriesViewJob(job) {
// TODO - do we need another function which returns whether to enable the
// link to the Single Metric dashboard in the Jobs list, only allowing single
// metric jobs with only one detector with no by/over/partition fields
// only allow jobs with at least one detector whose function corresponds to
// an ES aggregation which can be viewed in the single metric view and which
// doesn't use a scripted field which can be very difficult or impossible to
// invert to a reverse search.
// invert to a reverse search, or when model plot has been enabled.
let isViewable = false;
const dtrs = job.analysis_config.detectors;
@ -55,38 +51,68 @@ export function isTimeSeriesViewJob(job) {
// Returns a flag to indicate whether the detector at the index in the specified job
// is suitable for viewing in the Time Series dashboard.
export function isTimeSeriesViewDetector(job, dtrIndex) {
// Check that the detector function is suitable for viewing in the Time Series dashboard,
// and that the partition, by and over fields are not using mlcategory or a scripted field which
// can be very difficult or impossible to invert to a reverse search of the underlying metric data.
let isDetectorViewable = false;
return isSourceDataChartableForDetector(job, dtrIndex) ||
isModelPlotChartableForDetector(job, dtrIndex);
}
// Returns a flag to indicate whether the source data can be plotted in a time
// series chart for the specified detector.
export function isSourceDataChartableForDetector(job, detectorIndex) {
let isSourceDataChartable = false;
const dtrs = job.analysis_config.detectors;
if (dtrIndex >= 0 && dtrIndex < dtrs.length) {
const dtr = dtrs[dtrIndex];
isDetectorViewable = (isTimeSeriesViewFunction(dtr.function) === true) &&
if (detectorIndex >= 0 && detectorIndex < dtrs.length) {
const dtr = dtrs[detectorIndex];
const functionName = dtr.function;
// Check that the function maps to an ES aggregation,
// and that the partitioning field isn't mlcategory
// (since mlcategory is a derived field which won't exist in the source data).
// Note that the 'function' field in a record contains what the user entered e.g. 'high_count',
// whereas the 'function_description' field holds an ML-built display hint for function e.g. 'count'.
isSourceDataChartable = (mlFunctionToESAggregation(functionName) !== null) &&
(dtr.by_field_name !== 'mlcategory') &&
(dtr.partition_field_name !== 'mlcategory') &&
(dtr.over_field_name !== 'mlcategory');
// If the datafeed uses script fields, we can only plot the time series if
// model plot is enabled. Without model plot it will be very difficult or impossible
// to invert to a reverse search of the underlying metric data.
const usesScriptFields = _.has(job, 'datafeed_config.script_fields');
if (isDetectorViewable === true && usesScriptFields === true) {
if (isSourceDataChartable === true && usesScriptFields === true) {
// Perform extra check to see if the detector is using a scripted field.
const scriptFields = usesScriptFields ? _.keys(job.datafeed_config.script_fields) : [];
isDetectorViewable = scriptFields.indexOf(dtr.field_name) === -1 &&
isSourceDataChartable = (
scriptFields.indexOf(dtr.field_name) === -1 &&
scriptFields.indexOf(dtr.partition_field_name) === -1 &&
scriptFields.indexOf(dtr.by_field_name) === -1 &&
scriptFields.indexOf(dtr.over_field_name) === -1;
scriptFields.indexOf(dtr.over_field_name) === -1);
}
}
return isDetectorViewable;
return isSourceDataChartable;
}
// Returns a flag to indicate whether a detector with the specified function is
// suitable for viewing in the Time Series dashboard.
export function isTimeSeriesViewFunction(functionName) {
return mlFunctionToESAggregation(functionName) !== null;
// Returns a flag to indicate whether model plot data can be plotted in a time
// series chart for the specified detector.
export function isModelPlotChartableForDetector(job, detectorIndex) {
let isModelPlotChartable = false;
const modelPlotEnabled = _.get(job, ['model_plot_config', 'enabled'], false);
const dtrs = job.analysis_config.detectors;
if (detectorIndex >= 0 && detectorIndex < dtrs.length && modelPlotEnabled === true) {
const dtr = dtrs[detectorIndex];
const functionName = dtr.function;
// Model plot can be charted for any of the functions which map to ES aggregations,
// plus varp and info_content functions.
isModelPlotChartable = (mlFunctionToESAggregation(functionName) !== null) ||
(['varp', 'high_varp', 'low_varp', 'info_content',
'high_info_content', 'low_info_content'].includes(functionName) === true);
}
return isModelPlotChartable;
}
// Returns the names of the partition, by, and over fields for the detector with the
@ -117,7 +143,7 @@ export function isModelPlotEnabled(job, detectorIndex, entityFields) {
// Check if model_plot_config is enabled.
let isEnabled = _.get(job, ['model_plot_config', 'enabled'], false);
if (isEnabled === true) {
if (isEnabled === true && entityFields !== undefined && entityFields.length > 0) {
// If terms filter is configured in model_plot_config, check supplied entities.
const termsStr = _.get(job, ['model_plot_config', 'terms'], '');
if (termsStr !== '') {

2
x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_columns.js
View file

@ -51,7 +51,7 @@ function renderTime(date, aggregationInterval) {
function showLinksMenuForItem(item) {
const canConfigureRules = (isRuleSupported(item) && checkPermission('canUpdateJob'));
return (canConfigureRules ||
item.isTimeSeriesViewDetector ||
item.isTimeSeriesViewRecord ||
item.entityName === 'mlcategory' ||
item.customUrls !== undefined);
}

2
x-pack/plugins/ml/public/components/anomalies_table/anomaly_details.test.js
View file

@ -53,7 +53,7 @@ const props = {
typicalSort: 0.012071679592192066,
metricDescriptionSort: 82.83851409101328,
detector: 'count by mlcategory',
isTimeSeriesViewDetector: false
isTimeSeriesViewRecord: false
},
examples: [
'Actual Transaction Already Voided / Reversed;hostname=dbserver.acme.com;physicalhost=esxserver1.acme.com;vmhost=app1.acme.com',

2
x-pack/plugins/ml/public/components/anomalies_table/links_menu.js
View file

@ -395,7 +395,7 @@ export const LinksMenu = injectI18n(class LinksMenu extends Component {
});
}
if (showViewSeriesLink === true && anomaly.isTimeSeriesViewDetector === true) {
if (showViewSeriesLink === true && anomaly.isTimeSeriesViewRecord === true) {
items.push(
<EuiContextMenuItem
key="view_series"

29
x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_config_builder.js
View file

@ -14,6 +14,7 @@
import _ from 'lodash';
import { parseInterval } from 'ui/utils/parse_interval';
import { getEntityFieldList } from '../../../common/util/anomaly_utils';
import { buildConfigFromDetector } from '../../util/chart_config_builder';
import { mlJobService } from '../../services/job_service';
@ -46,33 +47,7 @@ export function buildConfig(record) {
// Add the 'entity_fields' i.e. the partition, by, over fields which
// define the metric series to be plotted.
config.entityFields = [];
if (_.has(record, 'partition_field_name')) {
config.entityFields.push({
fieldName: record.partition_field_name,
fieldValue: record.partition_field_value,
fieldType: 'partition'
});
}
if (_.has(record, 'over_field_name')) {
config.entityFields.push({
fieldName: record.over_field_name,
fieldValue: record.over_field_value,
fieldType: 'over'
});
}
// For jobs with by and over fields, don't add the 'by' field as this
// field will only be added to the top-level fields for record type results
// if it also an influencer over the bucket.
if (_.has(record, 'by_field_name') && !(_.has(record, 'over_field_name'))) {
config.entityFields.push({
fieldName: record.by_field_name,
fieldValue: record.by_field_value,
fieldType: 'by'
});
}
config.entityFields = getEntityFieldList(record);
// Build the tooltip data for the chart info icon, showing further details on what is being plotted.
let functionLabel = config.metricFunction;

118
x-pack/plugins/ml/public/explorer/explorer_charts/explorer_charts_container_service.js
View file

@ -18,7 +18,9 @@ import {
chartLimits,
getChartType
} from '../../util/chart_utils';
import { isTimeSeriesViewDetector } from '../../../common/util/job_utils';
import { getEntityFieldList } from '../../../common/util/anomaly_utils';
import { isSourceDataChartableForDetector, isModelPlotEnabled } from '../../../common/util/job_utils';
import { mlResultsService } from '../../services/results_service';
import { mlJobService } from '../../services/job_service';
import { severity$ } from '../../components/controls/select_severity/select_severity';
@ -37,7 +39,6 @@ export function getDefaultChartsData() {
}
export function explorerChartsContainerServiceFactory(callback) {
const FUNCTION_DESCRIPTIONS_TO_PLOT = ['mean', 'min', 'max', 'sum', 'count', 'distinct_count', 'median', 'rare'];
const CHART_MAX_POINTS = 500;
const ANOMALIES_MAX_RESULTS = 500;
const MAX_SCHEDULED_EVENTS = 10; // Max number of scheduled events displayed per bucket.
@ -100,18 +101,90 @@ export function explorerChartsContainerServiceFactory(callback) {
// Query 1 - load the raw metric data.
function getMetricData(config, range) {
const datafeedQuery = _.get(config, 'datafeedConfig.query', null);
return mlResultsService.getMetricData(
config.datafeedConfig.indices,
config.entityFields,
datafeedQuery,
config.metricFunction,
config.metricFieldName,
config.timeField,
range.min,
range.max,
config.interval
);
const {
jobId,
detectorIndex,
entityFields,
interval
} = config;
const job = mlJobService.getJob(jobId);
// If source data can be plotted, use that, otherwise model plot will be available.
const useSourceData = isSourceDataChartableForDetector(job, detectorIndex);
if (useSourceData === true) {
const datafeedQuery = _.get(config, 'datafeedConfig.query', null);
return mlResultsService.getMetricData(
config.datafeedConfig.indices,
config.entityFields,
datafeedQuery,
config.metricFunction,
config.metricFieldName,
config.timeField,
range.min,
range.max,
config.interval
);
} else {
// Extract the partition, by, over fields on which to filter.
const criteriaFields = [];
const detector = job.analysis_config.detectors[detectorIndex];
if (_.has(detector, 'partition_field_name')) {
const partitionEntity = _.find(entityFields, { 'fieldName': detector.partition_field_name });
if (partitionEntity !== undefined) {
criteriaFields.push(
{ fieldName: 'partition_field_name', fieldValue: partitionEntity.fieldName },
{ fieldName: 'partition_field_value', fieldValue: partitionEntity.fieldValue });
}
}
if (_.has(detector, 'over_field_name')) {
const overEntity = _.find(entityFields, { 'fieldName': detector.over_field_name });
if (overEntity !== undefined) {
criteriaFields.push(
{ fieldName: 'over_field_name', fieldValue: overEntity.fieldName },
{ fieldName: 'over_field_value', fieldValue: overEntity.fieldValue });
}
}
if (_.has(detector, 'by_field_name')) {
const byEntity = _.find(entityFields, { 'fieldName': detector.by_field_name });
if (byEntity !== undefined) {
criteriaFields.push(
{ fieldName: 'by_field_name', fieldValue: byEntity.fieldName },
{ fieldName: 'by_field_value', fieldValue: byEntity.fieldValue });
}
}
return new Promise((resolve, reject) => {
const obj = {
success: true,
results: {}
};
return mlResultsService.getModelPlotOutput(
jobId,
detectorIndex,
criteriaFields,
range.min,
range.max,
interval
)
.then((resp) => {
// Return data in format required by the explorer charts.
const results = resp.results;
Object.keys(results).forEach((time) => {
obj.results[time] = results[time].actual;
});
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
});
}
}
// Query 2 - load the anomalies.
@ -346,13 +419,18 @@ export function explorerChartsContainerServiceFactory(callback) {
// Aggregate by job, detector, and analysis fields (partition, by, over).
const aggregatedData = {};
_.each(anomalyRecords, (record) => {
// Only plot charts for metric functions, and for detectors which don't use categorization
// or scripted fields which can be very difficult or impossible to invert to a reverse search.
// Check if we can plot a chart for this record, depending on whether the source data
// is chartable, and if model plot is enabled for the job.
const job = mlJobService.getJob(record.job_id);
if (
isTimeSeriesViewDetector(job, record.detector_index) === false ||
FUNCTION_DESCRIPTIONS_TO_PLOT.includes(record.function_description) === false
) {
let isChartable = isSourceDataChartableForDetector(job, record.detector_index);
if (isChartable === false) {
// Check if model plot is enabled for this job.
// Need to check the entity fields for the record in case the model plot config has a terms list.
const entityFields = getEntityFieldList(record);
isChartable = isModelPlotEnabled(job, record.detector_index, entityFields);
}
if (isChartable === false) {
return;
}
const jobId = record.job_id;

15
x-pack/plugins/ml/public/explorer/explorer_utils.js
View file

@ -11,7 +11,8 @@
import { chain, each, get, union, uniq } from 'lodash';
import { parseInterval } from 'ui/utils/parse_interval';
import { isTimeSeriesViewDetector } from '../../common/util/job_utils';
import { getEntityFieldList } from '../../common/util/anomaly_utils';
import { isSourceDataChartableForDetector, isModelPlotEnabled } from '../../common/util/job_utils';
import { ml } from '../services/ml_api_service';
import { mlJobService } from '../services/job_service';
import { mlResultsService } from 'plugins/ml/services/results_service';
@ -495,7 +496,17 @@ export async function loadAnomaliesTableData(
// Add properties used for building the links menu.
// TODO - when job_service is moved server_side, move this to server endpoint.
anomaly.isTimeSeriesViewDetector = isTimeSeriesViewDetector(mlJobService.getJob(jobId), anomaly.detectorIndex);
const job = mlJobService.getJob(jobId);
let isChartable = isSourceDataChartableForDetector(job, anomaly.detectorIndex);
if (isChartable === false) {
// Check if model plot is enabled for this job.
// Need to check the entity fields for the record in case the model plot config has a terms list.
// If terms is specified, model plot is only stored if both the partition and by fields appear in the list.
const entityFields = getEntityFieldList(anomaly.source);
isChartable = isModelPlotEnabled(job, anomaly.detectorIndex, entityFields);
}
anomaly.isTimeSeriesViewRecord = isChartable;
if (mlJobService.customUrlsByJob[jobId] !== undefined) {
anomaly.customUrls = mlJobService.customUrlsByJob[jobId];
}

23
x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer.html
View file

@ -71,7 +71,8 @@
is-loading="loading === true"
/>
<div class="no-results-container" ng-show="jobs.length > 0 && loading === false && hasResults === false">
<div class="no-results-container"
ng-show="jobs.length > 0 && loading === false && hasResults === false && dataNotChartable === false">
<div class="no-results">
<div
i18n-id="xpack.ml.timeSeriesExplorer.noResultsFoundLabel"
@ -85,6 +86,24 @@
</div>
</div>
<div class="no-results-container"
ng-show="jobs.length > 0 && loading === false && hasResults === false && dataNotChartable === true">
<div class="no-results">
<div
i18n-id="xpack.ml.timeSeriesExplorer.noResultsFoundLabel"
i18n-default-message="{icon} No results found"
i18n-values="{ html_icon: '<i class=\'fa fa-info-circle\'></i>' }"
></div>
<div
i18n-id="xpack.ml.timeSeriesExplorer.dataNotChartableDescription"
i18n-default-message="Model plot is not collected for the selected {entityCount, plural, one {entity} other {entities}} and the source data cannot be plotted for this detector"
i18n-values="{
entityCount: entities.length
}"
></div>
</div>
</div>
<div ng-show="jobs.length > 0 && loading === false && hasResults === true">
<div class="results-container">
@ -189,7 +208,7 @@
i18n-id="xpack.ml.timeSeriesExplorer.annotationsTitle"
i18n-default-message="Annotations"
></span>
<ml-annotation-table
annotations="focusAnnotationData"
drill-down="false"

16
x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_controller.js
View file

@ -30,6 +30,7 @@ import {
isTimeSeriesViewJob,
isTimeSeriesViewDetector,
isModelPlotEnabled,
isSourceDataChartableForDetector,
mlFunctionToESAggregation } from 'plugins/ml/../common/util/job_utils';
import { loadIndexPatterns, getIndexPatterns } from 'plugins/ml/util/index_utils';
import { getSingleMetricViewerBreadcrumbs } from './breadcrumbs';
@ -105,6 +106,7 @@ module.controller('MlTimeSeriesExplorerController', function (
$scope.loading = true;
$scope.loadCounter = 0;
$scope.hasResults = false;
$scope.dataNotChartable = false; // e.g. model plot with terms for a varp detector
$scope.anomalyRecords = [];
$scope.modelPlotEnabled = false;
@ -229,6 +231,7 @@ module.controller('MlTimeSeriesExplorerController', function (
$scope.loading = true;
$scope.hasResults = false;
$scope.dataNotChartable = false;
delete $scope.chartDetails;
delete $scope.contextChartData;
delete $scope.focusChartData;
@ -282,6 +285,7 @@ module.controller('MlTimeSeriesExplorerController', function (
const detectorIndex = +$scope.detectorId;
$scope.modelPlotEnabled = isModelPlotEnabled($scope.selectedJob, detectorIndex, $scope.entities);
// Only filter on the entity if the field has a value.
const nonBlankEntities = _.filter($scope.entities, (entity) => { return entity.fieldValue.length > 0; });
$scope.criteriaFields = [{
@ -289,6 +293,18 @@ module.controller('MlTimeSeriesExplorerController', function (
'fieldValue': detectorIndex }
].concat(nonBlankEntities);
if ($scope.modelPlotEnabled === false &&
isSourceDataChartableForDetector($scope.selectedJob, detectorIndex) === false &&
nonBlankEntities.length > 0) {
// For detectors where model plot has been enabled with a terms filter and the
// selected entity(s) are not in the terms list, indicate that data cannot be viewed.
$scope.hasResults = false;
$scope.loading = false;
$scope.dataNotChartable = true;
$scope.$applyAsync();
return;
}
// Calculate the aggregation interval for the context chart.
// Context chart swimlane will display bucket anomaly score at the same interval.
$scope.contextAggregationInterval = calculateAggregationInterval(bounds, CHARTS_POINT_TARGET, CHARTS_POINT_TARGET);

80
x-pack/plugins/ml/public/util/__tests__/chart_utils.js
View file

@ -12,11 +12,13 @@ import expect from '@kbn/expect';
import {
chartLimits,
filterAxisLabels,
getChartType,
numTicks,
showMultiBucketAnomalyMarker,
showMultiBucketAnomalyTooltip,
} from '../chart_utils';
import { MULTI_BUCKET_IMPACT } from 'plugins/ml/../common/constants/multi_bucket_impact';
import { MULTI_BUCKET_IMPACT } from '../../../common/constants/multi_bucket_impact';
import { CHART_TYPE } from '../../explorer/explorer_constants';
describe('ML - chart utils', () => {
@ -139,6 +141,82 @@ describe('ML - chart utils', () => {
});
describe('getChartType', () => {
const singleMetricConfig = {
metricFunction: 'avg',
functionDescription: 'mean',
fieldName: 'responsetime',
entityFields: [],
};
const multiMetricConfig = {
metricFunction: 'avg',
functionDescription: 'mean',
fieldName: 'responsetime',
entityFields: [
{
fieldName: 'airline',
fieldValue: 'AAL',
fieldType: 'partition',
}
],
};
const populationConfig = {
metricFunction: 'avg',
functionDescription: 'mean',
fieldName: 'http.response.body.bytes',
entityFields: [
{
fieldName: 'source.ip',
fieldValue: '10.11.12.13',
fieldType: 'over',
}
],
};
const rareConfig = {
metricFunction: 'count',
functionDescription: 'rare',
entityFields: [
{
fieldName: 'http.response.status_code',
fieldValue: '404',
fieldType: 'by',
}
],
};
const varpModelPlotConfig = {
metricFunction: null,
functionDescription: 'varp',
fieldName: 'NetworkOut',
entityFields: [
{
fieldName: 'instance',
fieldValue: 'i-ef74d410',
fieldType: 'over',
}
],
};
it('returns single metric chart type as expected for configs', () => {
expect(getChartType(singleMetricConfig)).to.be(CHART_TYPE.SINGLE_METRIC);
expect(getChartType(multiMetricConfig)).to.be(CHART_TYPE.SINGLE_METRIC);
expect(getChartType(varpModelPlotConfig)).to.be(CHART_TYPE.SINGLE_METRIC);
});
it('returns event distribution chart type as expected for configs', () => {
expect(getChartType(rareConfig)).to.be(CHART_TYPE.EVENT_DISTRIBUTION);
});
it('returns population distribution chart type as expected for configs', () => {
expect(getChartType(populationConfig)).to.be(CHART_TYPE.POPULATION_DISTRIBUTION);
});
});
describe('numTicks', () => {
it('returns 10 for 1000', () => {

4
x-pack/plugins/ml/public/util/chart_utils.js
View file

@ -140,6 +140,7 @@ const POPULATION_DISTRIBUTION_ENABLED = true;
// get the chart type based on its configuration
export function getChartType(config) {
if (
EVENT_DISTRIBUTION_ENABLED &&
config.functionDescription === 'rare' &&
@ -149,7 +150,8 @@ export function getChartType(config) {
} else if (
POPULATION_DISTRIBUTION_ENABLED &&
config.functionDescription !== 'rare' &&
config.entityFields.some(f => f.fieldType === 'over')
config.entityFields.some(f => f.fieldType === 'over') &&
config.metricFunction !== null // Event distribution chart relies on the ML function mapping to an ES aggregation
) {
return CHART_TYPE.POPULATION_DISTRIBUTION;
}