diff --git a/x-pack/plugins/ml/public/constants/index_patterns.js b/x-pack/plugins/ml/common/constants/index_patterns.js similarity index 100% rename from x-pack/plugins/ml/public/constants/index_patterns.js rename to x-pack/plugins/ml/common/constants/index_patterns.js diff --git a/x-pack/plugins/ml/public/util/__tests__/anomaly_utils.js b/x-pack/plugins/ml/common/util/__tests__/anomaly_utils.js similarity index 100% rename from x-pack/plugins/ml/public/util/__tests__/anomaly_utils.js rename to x-pack/plugins/ml/common/util/__tests__/anomaly_utils.js diff --git a/x-pack/plugins/ml/public/util/anomaly_utils.js b/x-pack/plugins/ml/common/util/anomaly_utils.js similarity index 100% rename from x-pack/plugins/ml/public/util/anomaly_utils.js rename to x-pack/plugins/ml/common/util/anomaly_utils.js diff --git a/x-pack/plugins/ml/index.js b/x-pack/plugins/ml/index.js index b2be3dd628ad..38a2ce773e65 100644 --- a/x-pack/plugins/ml/index.js +++ b/x-pack/plugins/ml/index.js @@ -20,6 +20,7 @@ import { dataRecognizer } from './server/routes/modules'; import { dataVisualizerRoutes } from './server/routes/data_visualizer'; import { calendars } from './server/routes/calendars'; import { fieldsService } from './server/routes/fields_service'; +import { resultsServiceRoutes } from './server/routes/results_service'; export const ml = (kibana) => { return new kibana.Plugin({ @@ -82,6 +83,7 @@ export const ml = (kibana) => { dataVisualizerRoutes(server, commonRouteConfig); calendars(server, commonRouteConfig); fieldsService(server, commonRouteConfig); + resultsServiceRoutes(server, commonRouteConfig); } }); diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js index 6a67fd3baa45..da5f64fb5a96 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js +++ b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js @@ -28,9 +28,9 @@ import { showActualForFunction, showTypicalForFunction, getSeverity -} from 'plugins/ml/util/anomaly_utils'; +} from 'plugins/ml/../common/util/anomaly_utils'; import { getFieldTypeFromMapping } from 'plugins/ml/services/mapping_service'; -import { mlResultsService } from 'plugins/ml/services/results_service'; +import { ml } from 'plugins/ml/services/ml_api_service'; import { mlJobService } from 'plugins/ml/services/job_service'; import { mlFieldFormatService } from 'plugins/ml/services/field_format_service'; import template from './anomalies_table.html'; @@ -222,7 +222,7 @@ module.directive('mlAnomaliesTable', function ( // Get the definition of the category and use the terms or regex to view the // matching events in the Kibana Discover tab depending on whether the // categorization field is of mapping type text (preferred) or keyword. - mlResultsService.getCategoryDefinition(record.job_id, categoryId) + ml.results.getCategoryDefinition(record.job_id, categoryId) .then((resp) => { let query = null; // Build query using categorization regex (if keyword type) or terms (if text type). @@ -338,8 +338,7 @@ module.directive('mlAnomaliesTable', function ( // mlcategory in the source record will be an array // - use first value (will only ever be more than one if influenced by category other than by/partition/over). const categoryId = record.mlcategory[0]; - - mlResultsService.getCategoryDefinition(jobId, categoryId) + ml.results.getCategoryDefinition(jobId, categoryId) .then((resp) => { // Prefix each of the terms with '+' so that the Elasticsearch Query String query // run in a drilldown Kibana dashboard has to match on all terms. @@ -720,7 +719,7 @@ module.directive('mlAnomaliesTable', function ( rowScope.initRow = function () { if (_.has(record, 'entityValue') && record.entityName === 'mlcategory') { // Obtain the category definition and display the examples in the expanded row. - mlResultsService.getCategoryDefinition(record.jobId, record.entityValue) + ml.results.getCategoryDefinition(record.jobId, record.entityValue) .then((resp) => { rowScope.categoryDefinition = { 'examples': _.slice(resp.examples, 0, Math.min(resp.examples.length, MAX_NUMBER_CATEGORY_EXAMPLES)) }; @@ -934,9 +933,9 @@ module.directive('mlAnomaliesTable', function ( // Load the example events for the specified map of job_ids and categoryIds from Elasticsearch. scope.categoryExamplesByJob = {}; _.each(categoryIdsByJobId, (categoryIds, jobId) => { - mlResultsService.getCategoryExamples(jobId, categoryIds, MAX_NUMBER_CATEGORY_EXAMPLES) + ml.results.getCategoryExamples(jobId, categoryIds, MAX_NUMBER_CATEGORY_EXAMPLES) .then((resp) => { - scope.categoryExamplesByJob[jobId] = resp.examplesByCategoryId; + scope.categoryExamplesByJob[jobId] = resp; }).catch((resp) => { console.log('Anomalies table - error getting category examples:', resp); }); diff --git a/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js b/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js index fd87f5b7c2a5..bd3422f26b1a 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js +++ b/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js @@ -22,7 +22,7 @@ import { getSeverity, showActualForFunction, showTypicalForFunction -} from 'plugins/ml/util/anomaly_utils'; +} from 'plugins/ml/../common/util/anomaly_utils'; import 'plugins/ml/formatters/format_value'; import { uiModules } from 'ui/modules'; diff --git a/x-pack/plugins/ml/public/components/influencers_list/influencers_list.js b/x-pack/plugins/ml/public/components/influencers_list/influencers_list.js index f0f7061d3d60..eedac15ef044 100644 --- a/x-pack/plugins/ml/public/components/influencers_list/influencers_list.js +++ b/x-pack/plugins/ml/public/components/influencers_list/influencers_list.js @@ -22,7 +22,7 @@ import { } from '@elastic/eui'; import { abbreviateWholeNumber } from 'plugins/ml/formatters/abbreviate_whole_number'; -import { getSeverity } from 'plugins/ml/util/anomaly_utils'; +import { getSeverity } from 'plugins/ml/../common/util/anomaly_utils'; function getTooltipContent(maxScoreLabel, totalScoreLabel) { diff --git a/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_directive.js b/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_directive.js index 035f61a9d425..98fea34be0d0 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_directive.js +++ b/x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_directive.js @@ -19,7 +19,7 @@ import angular from 'angular'; import moment from 'moment'; import { formatValue } from 'plugins/ml/formatters/format_value'; -import { getSeverityWithLow } from 'plugins/ml/util/anomaly_utils'; +import { getSeverityWithLow } from 'plugins/ml/../common/util/anomaly_utils'; import { drawLineChartDots, numTicksForDateFormat } from 'plugins/ml/util/chart_utils'; import { TimeBuckets } from 'ui/time_buckets'; import loadingIndicatorWrapperTemplate from 'plugins/ml/components/loading_indicator/loading_indicator_wrapper.html'; diff --git a/x-pack/plugins/ml/public/explorer/explorer_swimlane_directive.js b/x-pack/plugins/ml/public/explorer/explorer_swimlane_directive.js index a7367266013d..3760c463e956 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_swimlane_directive.js +++ b/x-pack/plugins/ml/public/explorer/explorer_swimlane_directive.js @@ -15,7 +15,7 @@ import $ from 'jquery'; import moment from 'moment'; import d3 from 'd3'; -import { getSeverityColor } from 'plugins/ml/util/anomaly_utils'; +import { getSeverityColor } from 'plugins/ml/../common/util/anomaly_utils'; import { numTicksForDateFormat } from 'plugins/ml/util/chart_utils'; import { IntervalHelperProvider } from 'plugins/ml/util/ml_time_buckets'; import { mlEscape } from 'plugins/ml/util/string_utils'; diff --git a/x-pack/plugins/ml/public/jobs/components/custom_url_editor/custom_url_editor_service.js b/x-pack/plugins/ml/public/jobs/components/custom_url_editor/custom_url_editor_service.js index a96a46648efa..f9bad4f31076 100644 --- a/x-pack/plugins/ml/public/jobs/components/custom_url_editor/custom_url_editor_service.js +++ b/x-pack/plugins/ml/public/jobs/components/custom_url_editor/custom_url_editor_service.js @@ -10,7 +10,7 @@ import { parseInterval } from 'ui/utils/parse_interval'; -import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns'; +import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns'; import { replaceTokensInUrlValue } from 'plugins/ml/util/custom_url_utils'; import { mlJobService } from 'plugins/ml/services/job_service'; import { ml } from 'plugins/ml/services/ml_api_service'; diff --git a/x-pack/plugins/ml/public/jobs/new_job/simple/components/utils/search_service.js b/x-pack/plugins/ml/public/jobs/new_job/simple/components/utils/search_service.js index 2830ca7b71d4..b6f9723ca82a 100644 --- a/x-pack/plugins/ml/public/jobs/new_job/simple/components/utils/search_service.js +++ b/x-pack/plugins/ml/public/jobs/new_job/simple/components/utils/search_service.js @@ -8,7 +8,7 @@ import _ from 'lodash'; -import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns'; +import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns'; import { escapeForElasticsearchQuery } from 'plugins/ml/util/string_utils'; import { ml } from 'plugins/ml/services/ml_api_service'; diff --git a/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js b/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js index 0ea79a4bf3a8..16ced43876fa 100644 --- a/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js +++ b/x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns'; +import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns'; export const watch = { trigger: { diff --git a/x-pack/plugins/ml/public/services/forecast_service.js b/x-pack/plugins/ml/public/services/forecast_service.js index ad567527a1a6..09a6b03f4658 100644 --- a/x-pack/plugins/ml/public/services/forecast_service.js +++ b/x-pack/plugins/ml/public/services/forecast_service.js @@ -10,7 +10,7 @@ // data on forecasts that have been performed. import _ from 'lodash'; -import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns'; +import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns'; import { ml } from 'plugins/ml/services/ml_api_service'; diff --git a/x-pack/plugins/ml/public/services/job_messages_service.js b/x-pack/plugins/ml/public/services/job_messages_service.js index f8ad1e74bfa0..dd2481886d4f 100644 --- a/x-pack/plugins/ml/public/services/job_messages_service.js +++ b/x-pack/plugins/ml/public/services/job_messages_service.js @@ -10,7 +10,7 @@ // Service for carrying out Elasticsearch queries to obtain data for the // Ml Results dashboards. -import { ML_NOTIFICATION_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns'; +import { ML_NOTIFICATION_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns'; import { ml } from 'plugins/ml/services/ml_api_service'; // search for audit messages, jobId is optional. diff --git a/x-pack/plugins/ml/public/services/job_service.js b/x-pack/plugins/ml/public/services/job_service.js index 1e8eb5109cf4..530550945600 100644 --- a/x-pack/plugins/ml/public/services/job_service.js +++ b/x-pack/plugins/ml/public/services/job_service.js @@ -13,7 +13,7 @@ import moment from 'moment'; import { parseInterval } from 'ui/utils/parse_interval'; import { ml } from 'plugins/ml/services/ml_api_service'; -import { labelDuplicateDetectorDescriptions } from 'plugins/ml/util/anomaly_utils'; +import { labelDuplicateDetectorDescriptions } from 'plugins/ml/../common/util/anomaly_utils'; import { mlMessageBarService } from 'plugins/ml/components/messagebar/messagebar_service'; import { isWebUrl } from 'plugins/ml/util/string_utils'; import { ML_DATA_PREVIEW_COUNT } from 'plugins/ml/../common/util/job_utils'; diff --git a/x-pack/plugins/ml/public/services/ml_api_service.js b/x-pack/plugins/ml/public/services/ml_api_service/index.js similarity index 98% rename from x-pack/plugins/ml/public/services/ml_api_service.js rename to x-pack/plugins/ml/public/services/ml_api_service/index.js index 1b8128174d7b..a91cdf258d2d 100644 --- a/x-pack/plugins/ml/public/services/ml_api_service.js +++ b/x-pack/plugins/ml/public/services/ml_api_service/index.js @@ -9,7 +9,9 @@ import { pick } from 'lodash'; import chrome from 'ui/chrome'; -import { http } from './http_service'; +import { http } from 'plugins/ml/services/http_service'; + +import { results } from './results'; const basePath = chrome.addBasePath('/api/ml'); @@ -403,4 +405,6 @@ export const ml = { data: obj }); }, + + results }; diff --git a/x-pack/plugins/ml/public/services/ml_api_service/results.js b/x-pack/plugins/ml/public/services/ml_api_service/results.js new file mode 100644 index 000000000000..a405eacecdeb --- /dev/null +++ b/x-pack/plugins/ml/public/services/ml_api_service/results.js @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// Service for obtaining data for the ML Results dashboards. + +import chrome from 'ui/chrome'; + +import { http } from 'plugins/ml/services/http_service'; + +const basePath = chrome.addBasePath('/api/ml'); + +export const results = { + getAnomaliesTableData( + jobIds, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + maxRecords, + maxExamples) { + + return http({ + url: `${basePath}/results/anomalies_table_data`, + method: 'POST', + data: { + jobIds, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + maxRecords, + maxExamples + } + }); + }, + + getCategoryDefinition(jobId, categoryId) { + return http({ + url: `${basePath}/results/category_definition`, + method: 'POST', + data: { jobId, categoryId } + }); + }, + + getCategoryExamples( + jobId, + categoryIds, + maxExamples + ) { + + return http({ + url: `${basePath}/results/category_examples`, + method: 'POST', + data: { + jobId, + categoryIds, + maxExamples + } + }); + } +}; diff --git a/x-pack/plugins/ml/public/services/results_service.js b/x-pack/plugins/ml/public/services/results_service.js index cd76db517001..fb13ed381ab3 100644 --- a/x-pack/plugins/ml/public/services/results_service.js +++ b/x-pack/plugins/ml/public/services/results_service.js @@ -12,7 +12,7 @@ import _ from 'lodash'; import { ML_MEDIAN_PERCENTS } from 'plugins/ml/../common/util/job_utils'; import { escapeForElasticsearchQuery } from 'plugins/ml/util/string_utils'; -import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns'; +import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns'; import { ml } from 'plugins/ml/services/ml_api_service'; @@ -699,88 +699,6 @@ function getInfluencerValueMaxScoreByTime( }); } - -// Obtains the definition of the category with the specified ID and job ID. -// Returned response contains four properties - categoryId, regex, examples -// and terms (space delimited String of the common tokens matched in values of the category). -function getCategoryDefinition(jobId, categoryId) { - return new Promise((resolve, reject) => { - const obj = { success: true, categoryId: categoryId, terms: null, regex: null, examples: [] }; - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 1, - body: { - query: { - bool: { - filter: [ - { term: { job_id: jobId } }, - { term: { category_id: categoryId } } - ] - } - } - } - }) - .then((resp) => { - if (resp.hits.total !== 0) { - const source = _.first(resp.hits.hits)._source; - obj.categoryId = source.category_id; - obj.regex = source.regex; - obj.terms = source.terms; - obj.examples = source.examples; - } - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} - - -// Obtains the categorization examples for the categories with the specified IDs -// from the given index and job ID. -// Returned response contains two properties - jobId and -// examplesByCategoryId (list of examples against categoryId). -function getCategoryExamples(jobId, categoryIds, maxExamples) { - return new Promise((resolve, reject) => { - const obj = { success: true, jobId: jobId, examplesByCategoryId: {} }; - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 500, // Matches size of records in anomaly summary table. - body: { - query: { - bool: { - filter: [ - { term: { job_id: jobId } }, - { terms: { category_id: categoryIds } } - ] - } - } - } - }) - .then((resp) => { - if (resp.hits.total !== 0) { - _.each(resp.hits.hits, (hit) => { - if (maxExamples) { - obj.examplesByCategoryId[hit._source.category_id] = - _.slice(hit._source.examples, 0, Math.min(hit._source.examples.length, maxExamples)); - } else { - obj.examplesByCategoryId[hit._source.category_id] = hit._source.examples; - } - - }); - } - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} - - // Queries Elasticsearch to obtain record level results containing the influencers // for the specified job(s), record score threshold, and time range. // Pass an empty array or ['*'] to search over all job IDs. @@ -1738,8 +1656,6 @@ export const mlResultsService = { getTopInfluencerValues, getOverallBucketScores, getInfluencerValueMaxScoreByTime, - getCategoryDefinition, - getCategoryExamples, getRecordInfluencers, getRecordsForInfluencer, getRecordsForDetector, diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js b/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js index e25ee1620bb8..681cda5a6b14 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js +++ b/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js @@ -20,8 +20,8 @@ import 'ui/timefilter'; import { ResizeChecker } from 'ui/resize_checker'; +import { getSeverityWithLow } from 'plugins/ml/../common/util/anomaly_utils'; import { formatValue } from 'plugins/ml/formatters/format_value'; -import { getSeverityWithLow } from 'plugins/ml/util/anomaly_utils'; import { drawLineChartDots, filterAxisLabels, diff --git a/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js b/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js new file mode 100644 index 000000000000..56094b7832ce --- /dev/null +++ b/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +import _ from 'lodash'; +import moment from 'moment'; + +import { + getEntityFieldName, + getEntityFieldValue, + showActualForFunction, + showTypicalForFunction +} from '../../../common/util/anomaly_utils'; + + +// Builds the items for display in the anomalies table from the supplied list of anomaly records. +export function buildAnomalyTableItems(anomalyRecords, aggregationInterval) { + + // Aggregate the anomaly records if necessary, and create skeleton display records with + // time, detector (description) and source record properties set. + let displayRecords = []; + if (aggregationInterval !== 'second') { + displayRecords = aggregateAnomalies(anomalyRecords, aggregationInterval); + } else { + // Show all anomaly records. + displayRecords = anomalyRecords.map((record) => { + return { + time: record.timestamp, + source: record, + }; + }); + } + + // Fill out the remaining properties in each display record + // for the columns to be displayed in the table. + return displayRecords.map((record) => { + const source = record.source; + const jobId = source.job_id; + + record.jobId = jobId; + record.detectorIndex = source.detector_index; + record.severity = source.record_score; + + const entityName = getEntityFieldName(source); + if (entityName !== undefined) { + record.entityName = entityName; + record.entityValue = getEntityFieldValue(source); + } + + if (source.influencers !== undefined) { + const influencers = []; + const sourceInfluencers = _.sortBy(source.influencers, 'influencer_field_name'); + sourceInfluencers.forEach((influencer) => { + const influencerFieldName = influencer.influencer_field_name; + influencer.influencer_field_values.forEach((influencerFieldValue) => { + influencers.push({ + [influencerFieldName]: influencerFieldValue + }); + }); + }); + record.influencers = influencers; + } + + const functionDescription = source.function_description || ''; + const causes = source.causes || []; + if (showActualForFunction(functionDescription) === true) { + if (source.actual !== undefined) { + record.actual = source.actual; + } else { + // If only a single cause, copy values to the top level. + if (causes.length === 1) { + record.actual = causes[0].actual; + } + } + } + if (showTypicalForFunction(functionDescription) === true) { + if (source.typical !== undefined) { + record.typical = source.typical; + } else { + // If only a single cause, copy values to the top level. + if (causes.length === 1) { + record.typical = causes[0].typical; + } + } + } + + return record; + + }); +} + +function aggregateAnomalies(anomalyRecords, interval) { + // Aggregate the anomaly records by time, jobId, detectorIndex, and entity (by/over/partition). + // anomalyRecords assumed to be supplied in ascending time order. + if (anomalyRecords.length === 0) { + return []; + } + + const aggregatedData = {}; + anomalyRecords.forEach((record) => { + // Use moment.js to get start of interval. + const roundedTime = moment(record.timestamp).startOf(interval).valueOf(); + if (aggregatedData[roundedTime] === undefined) { + aggregatedData[roundedTime] = {}; + } + + // Aggregate by job, then detectorIndex. + const jobId = record.job_id; + const jobsAtTime = aggregatedData[roundedTime]; + if (jobsAtTime[jobId] === undefined) { + jobsAtTime[jobId] = {}; + } + + // Aggregate by detector - default to function_description if no description available. + const detectorIndex = record.detector_index; + const detectorsForJob = jobsAtTime[jobId]; + if (detectorsForJob[detectorIndex] === undefined) { + detectorsForJob[detectorIndex] = {}; + } + + // Now add an object for the anomaly with the highest anomaly score per entity. + // For the choice of entity, look in order for byField, overField, partitionField. + // If no by/over/partition, default to an empty String. + const entitiesForDetector = detectorsForJob[detectorIndex]; + + // TODO - are we worried about different byFields having the same + // value e.g. host=server1 and machine=server1? + let entity = getEntityFieldValue(record); + if (entity === undefined) { + entity = ''; + } + if (entitiesForDetector[entity] === undefined) { + entitiesForDetector[entity] = record; + } else { + if (record.record_score > entitiesForDetector[entity].record_score) { + entitiesForDetector[entity] = record; + } + } + }); + + // Flatten the aggregatedData to give a list of records with + // the highest score per bucketed time / jobId / detectorIndex. + const summaryRecords = []; + _.each(aggregatedData, (times, roundedTime) => { + _.each(times, (jobIds) => { + _.each(jobIds, (entityDetectors) => { + _.each(entityDetectors, (record) => { + summaryRecords.push({ + time: +roundedTime, + source: record + }); + }); + }); + }); + }); + + return summaryRecords; + +} diff --git a/x-pack/plugins/ml/server/models/results_service/index.js b/x-pack/plugins/ml/server/models/results_service/index.js new file mode 100644 index 000000000000..995e9a06f632 --- /dev/null +++ b/x-pack/plugins/ml/server/models/results_service/index.js @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +export { resultsServiceProvider } from './results_service'; diff --git a/x-pack/plugins/ml/server/models/results_service/results_service.js b/x-pack/plugins/ml/server/models/results_service/results_service.js new file mode 100644 index 000000000000..07ddcf983b8b --- /dev/null +++ b/x-pack/plugins/ml/server/models/results_service/results_service.js @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +import _ from 'lodash'; +import moment from 'moment'; + +import { buildAnomalyTableItems } from './build_anomaly_table_items'; +import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patterns'; + + +// Service for carrying out Elasticsearch queries to obtain data for the +// ML Results dashboards. + +const DEFAULT_QUERY_SIZE = 500; + +export function resultsServiceProvider(callWithRequest) { + + // Obtains data for the anomalies table, aggregating anomalies by day or hour as requested. + // Return an Object with properties 'anomalies' and 'interval' (interval used to aggregate anomalies, + // one of day, hour or second. Note 'auto' can be provided as the aggregationInterval in the request, + // in which case the interval is determined according to the time span between the first and + // last anomalies), plus an examplesByJobId property if any of the + // anomalies are categorization anomalies in mlcategory. + async function getAnomaliesTableData( + jobIds, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + maxRecords, + maxExamples) { + + // Build the query to return the matching anomaly record results. + // Add criteria for the time range, record score, plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis' + } + } + }, + { + range: { + record_score: { + gte: threshold, + } + } + } + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + jobIds.forEach((jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr + } + }); + } + + // Add a nested query to filter for each of the specified influencers. + if (influencers.length > 0) { + boolCriteria.push({ + bool: { + should: influencers.map((influencer) => { + return { + nested: { + path: 'influencers', + query: { + bool: { + must: [ + { + match: { + 'influencers.influencer_field_name': influencer.fieldName + } + }, + { + match: { + 'influencers.influencer_field_values': influencer.fieldValue + } + } + ] + } + } + } + }; + }), + minimum_should_match: 1, + } + }); + } + + const resp = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: maxRecords !== undefined ? maxRecords : DEFAULT_QUERY_SIZE, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false + } + }, + { + bool: { + must: boolCriteria + } + } + ] + } + }, + sort: [ + { record_score: { order: 'desc' } } + ] + } + }); + + const tableData = { anomalies: [], interval: 'second' }; + if (resp.hits.total !== 0) { + let records = []; + resp.hits.hits.forEach((hit) => { + records.push(hit._source); + }); + + // Sort anomalies in ascending time order. + records = _.sortBy(records, 'timestamp'); + tableData.interval = aggregationInterval; + if (aggregationInterval === 'auto') { + // Determine the actual interval to use if aggregating. + const earliest = moment(records[0].timestamp); + const latest = moment(records[records.length - 1].timestamp); + + const daysDiff = latest.diff(earliest, 'days'); + tableData.interval = (daysDiff < 2 ? 'hour' : 'day'); + } + + tableData.anomalies = buildAnomalyTableItems(records, tableData.interval); + + // Load examples for any categorization anomalies. + const categoryAnomalies = tableData.anomalies.filter(item => item.entityName === 'mlcategory'); + if (categoryAnomalies.length > 0) { + tableData.examplesByJobId = {}; + + const categoryIdsByJobId = {}; + categoryAnomalies.forEach((anomaly) => { + if (!_.has(categoryIdsByJobId, anomaly.jobId)) { + categoryIdsByJobId[anomaly.jobId] = []; + } + if (categoryIdsByJobId[anomaly.jobId].indexOf(anomaly.entityValue) === -1) { + categoryIdsByJobId[anomaly.jobId].push(anomaly.entityValue); + } + }); + + const categoryJobIds = Object.keys(categoryIdsByJobId); + await Promise.all(categoryJobIds.map(async (jobId) => { + const examplesByCategoryId = await getCategoryExamples(jobId, categoryIdsByJobId[jobId], maxExamples); + tableData.examplesByJobId[jobId] = examplesByCategoryId; + })); + } + + } + + return tableData; + + } + + + // Obtains the categorization examples for the categories with the specified IDs + // from the given index and job ID. + // Returned response consists of a list of examples against category ID. + async function getCategoryExamples(jobId, categoryIds, maxExamples) { + const resp = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: DEFAULT_QUERY_SIZE, // Matches size of records in anomaly summary table. + body: { + query: { + bool: { + filter: [ + { term: { job_id: jobId } }, + { terms: { category_id: categoryIds } } + ] + } + } + } + }); + + const examplesByCategoryId = {}; + if (resp.hits.total !== 0) { + resp.hits.hits.forEach((hit) => { + if (maxExamples) { + examplesByCategoryId[hit._source.category_id] = + _.slice(hit._source.examples, 0, Math.min(hit._source.examples.length, maxExamples)); + } else { + examplesByCategoryId[hit._source.category_id] = hit._source.examples; + } + }); + } + + return examplesByCategoryId; + } + + // Obtains the definition of the category with the specified ID and job ID. + // Returned response contains four properties - categoryId, regex, examples + // and terms (space delimited String of the common tokens matched in values of the category). + async function getCategoryDefinition(jobId, categoryId) { + const resp = await callWithRequest('search', { + index: ML_RESULTS_INDEX_PATTERN, + size: 1, + body: { + query: { + bool: { + filter: [ + { term: { job_id: jobId } }, + { term: { category_id: categoryId } } + ] + } + } + } + }); + + const definition = { categoryId, terms: null, regex: null, examples: [] }; + if (resp.hits.total !== 0) { + const source = resp.hits.hits[0]._source; + definition.categoryId = source.category_id; + definition.regex = source.regex; + definition.terms = source.terms; + definition.examples = source.examples; + } + + return definition; + } + + return { + getAnomaliesTableData, + getCategoryDefinition, + getCategoryExamples + }; + +} diff --git a/x-pack/plugins/ml/server/routes/results_service.js b/x-pack/plugins/ml/server/routes/results_service.js new file mode 100644 index 000000000000..efce4f9c50bf --- /dev/null +++ b/x-pack/plugins/ml/server/routes/results_service.js @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +import { callWithRequestFactory } from '../client/call_with_request_factory'; +import { wrapError } from '../client/errors'; +import { resultsServiceProvider } from '../models/results_service'; + + +function getAnomaliesTableData(callWithRequest, payload) { + const rs = resultsServiceProvider(callWithRequest); + const { + jobIds, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + maxRecords, + maxExamples } = payload; + return rs.getAnomaliesTableData( + jobIds, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + maxRecords, + maxExamples); +} + +function getCategoryDefinition(callWithRequest, payload) { + const rs = resultsServiceProvider(callWithRequest); + return rs.getCategoryDefinition( + payload.jobId, + payload.categoryId); +} + +function getCategoryExamples(callWithRequest, payload) { + const rs = resultsServiceProvider(callWithRequest); + const { + jobId, + categoryIds, + maxExamples } = payload; + return rs.getCategoryExamples( + jobId, + categoryIds, + maxExamples); +} + +export function resultsServiceRoutes(server, commonRouteConfig) { + + server.route({ + method: 'POST', + path: '/api/ml/results/anomalies_table_data', + handler(request, reply) { + const callWithRequest = callWithRequestFactory(server, request); + return getAnomaliesTableData(callWithRequest, request.payload) + .then(resp => reply(resp)) + .catch(resp => reply(wrapError(resp))); + }, + config: { + ...commonRouteConfig + } + }); + + server.route({ + method: 'POST', + path: '/api/ml/results/category_definition', + handler(request, reply) { + const callWithRequest = callWithRequestFactory(server, request); + return getCategoryDefinition(callWithRequest, request.payload) + .then(resp => reply(resp)) + .catch(resp => reply(wrapError(resp))); + }, + config: { + ...commonRouteConfig + } + }); + + server.route({ + method: 'POST', + path: '/api/ml/results/category_examples', + handler(request, reply) { + const callWithRequest = callWithRequestFactory(server, request); + return getCategoryExamples(callWithRequest, request.payload) + .then(resp => reply(resp)) + .catch(resp => reply(wrapError(resp))); + }, + config: { + ...commonRouteConfig + } + }); + +}