[ML] Create server-side results service (#19044)

* [ML] Create server-side results service

* [ML] Edits to server side results service following review

x-pack/plugins/ml/index.js

@@ -20,6 +20,7 @@ import { dataRecognizer } from './server/routes/modules';
 import { dataVisualizerRoutes } from './server/routes/data_visualizer';
 import { calendars } from './server/routes/calendars';
 import { fieldsService } from './server/routes/fields_service';
+import { resultsServiceRoutes } from './server/routes/results_service';
 
 export const ml = (kibana) => {
   return new kibana.Plugin({
@@ -82,6 +83,7 @@ export const ml = (kibana) => {
       dataVisualizerRoutes(server, commonRouteConfig);
       calendars(server, commonRouteConfig);
       fieldsService(server, commonRouteConfig);
+      resultsServiceRoutes(server, commonRouteConfig);
     }
 
   });

x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js

@@ -28,9 +28,9 @@ import {
   showActualForFunction,
   showTypicalForFunction,
   getSeverity
-} from 'plugins/ml/util/anomaly_utils';
+} from 'plugins/ml/../common/util/anomaly_utils';
 import { getFieldTypeFromMapping } from 'plugins/ml/services/mapping_service';
-import { mlResultsService } from 'plugins/ml/services/results_service';
+import { ml } from 'plugins/ml/services/ml_api_service';
 import { mlJobService } from 'plugins/ml/services/job_service';
 import { mlFieldFormatService } from 'plugins/ml/services/field_format_service';
 import template from './anomalies_table.html';
@@ -222,7 +222,7 @@ module.directive('mlAnomaliesTable', function (
           // Get the definition of the category and use the terms or regex to view the
           // matching events in the Kibana Discover tab depending on whether the
           // categorization field is of mapping type text (preferred) or keyword.
-          mlResultsService.getCategoryDefinition(record.job_id, categoryId)
+          ml.results.getCategoryDefinition(record.job_id, categoryId)
             .then((resp) => {
               let query = null;
               // Build query using categorization regex (if keyword type) or terms (if text type).
@@ -338,8 +338,7 @@ module.directive('mlAnomaliesTable', function (
           // mlcategory in the source record will be an array
           // - use first value (will only ever be more than one if influenced by category other than by/partition/over).
           const categoryId = record.mlcategory[0];
-
-          mlResultsService.getCategoryDefinition(jobId, categoryId)
+          ml.results.getCategoryDefinition(jobId, categoryId)
             .then((resp) => {
             // Prefix each of the terms with '+' so that the Elasticsearch Query String query
             // run in a drilldown Kibana dashboard has to match on all terms.
@@ -720,7 +719,7 @@ module.directive('mlAnomaliesTable', function (
         rowScope.initRow = function () {
           if (_.has(record, 'entityValue') && record.entityName === 'mlcategory') {
             // Obtain the category definition and display the examples in the expanded row.
-            mlResultsService.getCategoryDefinition(record.jobId, record.entityValue)
+            ml.results.getCategoryDefinition(record.jobId, record.entityValue)
               .then((resp) => {
                 rowScope.categoryDefinition = {
                   'examples': _.slice(resp.examples, 0, Math.min(resp.examples.length, MAX_NUMBER_CATEGORY_EXAMPLES)) };
@@ -934,9 +933,9 @@ module.directive('mlAnomaliesTable', function (
         // Load the example events for the specified map of job_ids and categoryIds from Elasticsearch.
         scope.categoryExamplesByJob = {};
         _.each(categoryIdsByJobId, (categoryIds, jobId) => {
-          mlResultsService.getCategoryExamples(jobId, categoryIds, MAX_NUMBER_CATEGORY_EXAMPLES)
+          ml.results.getCategoryExamples(jobId, categoryIds, MAX_NUMBER_CATEGORY_EXAMPLES)
             .then((resp) => {
-              scope.categoryExamplesByJob[jobId] = resp.examplesByCategoryId;
+              scope.categoryExamplesByJob[jobId] = resp;
             }).catch((resp) => {
               console.log('Anomalies table - error getting category examples:', resp);
             });

x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js

@@ -22,7 +22,7 @@ import {
   getSeverity,
   showActualForFunction,
   showTypicalForFunction
-} from 'plugins/ml/util/anomaly_utils';
+} from 'plugins/ml/../common/util/anomaly_utils';
 import 'plugins/ml/formatters/format_value';
 
 import { uiModules } from 'ui/modules';

x-pack/plugins/ml/public/components/influencers_list/influencers_list.js

@@ -22,7 +22,7 @@ import {
 } from '@elastic/eui';
 
 import { abbreviateWholeNumber } from 'plugins/ml/formatters/abbreviate_whole_number';
-import { getSeverity } from 'plugins/ml/util/anomaly_utils';
+import { getSeverity } from 'plugins/ml/../common/util/anomaly_utils';
 
 
 function getTooltipContent(maxScoreLabel, totalScoreLabel) {

x-pack/plugins/ml/public/explorer/explorer_charts/explorer_chart_directive.js

@@ -19,7 +19,7 @@ import angular from 'angular';
 import moment from 'moment';
 
 import { formatValue } from 'plugins/ml/formatters/format_value';
-import { getSeverityWithLow } from 'plugins/ml/util/anomaly_utils';
+import { getSeverityWithLow } from 'plugins/ml/../common/util/anomaly_utils';
 import { drawLineChartDots, numTicksForDateFormat } from 'plugins/ml/util/chart_utils';
 import { TimeBuckets } from 'ui/time_buckets';
 import loadingIndicatorWrapperTemplate from 'plugins/ml/components/loading_indicator/loading_indicator_wrapper.html';

x-pack/plugins/ml/public/explorer/explorer_swimlane_directive.js

@@ -15,7 +15,7 @@ import $ from 'jquery';
 import moment from 'moment';
 import d3 from 'd3';
 
-import { getSeverityColor } from 'plugins/ml/util/anomaly_utils';
+import { getSeverityColor } from 'plugins/ml/../common/util/anomaly_utils';
 import { numTicksForDateFormat } from 'plugins/ml/util/chart_utils';
 import { IntervalHelperProvider } from 'plugins/ml/util/ml_time_buckets';
 import { mlEscape } from 'plugins/ml/util/string_utils';

x-pack/plugins/ml/public/jobs/components/custom_url_editor/custom_url_editor_service.js

@@ -10,7 +10,7 @@
 
 import { parseInterval } from 'ui/utils/parse_interval';
 
-import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
+import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns';
 import { replaceTokensInUrlValue } from 'plugins/ml/util/custom_url_utils';
 import { mlJobService } from 'plugins/ml/services/job_service';
 import { ml } from 'plugins/ml/services/ml_api_service';

x-pack/plugins/ml/public/jobs/new_job/simple/components/utils/search_service.js

@@ -8,7 +8,7 @@
 
 import _ from 'lodash';
 
-import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
+import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns';
 import { escapeForElasticsearchQuery } from 'plugins/ml/util/string_utils';
 import { ml } from 'plugins/ml/services/ml_api_service';
 

x-pack/plugins/ml/public/jobs/new_job/simple/components/watcher/watch.js

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
+import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns';
 
 export const watch = {
   trigger: {

x-pack/plugins/ml/public/services/forecast_service.js

@@ -10,7 +10,7 @@
 // data on forecasts that have been performed.
 import _ from 'lodash';
 
-import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
+import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns';
 import { ml } from 'plugins/ml/services/ml_api_service';
 
 

x-pack/plugins/ml/public/services/job_messages_service.js

@@ -10,7 +10,7 @@
 
 // Service for carrying out Elasticsearch queries to obtain data for the
 // Ml Results dashboards.
-import { ML_NOTIFICATION_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
+import { ML_NOTIFICATION_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns';
 import { ml } from 'plugins/ml/services/ml_api_service';
 
 // search for audit messages, jobId is optional.

x-pack/plugins/ml/public/services/job_service.js

@@ -13,7 +13,7 @@ import moment from 'moment';
 import { parseInterval } from 'ui/utils/parse_interval';
 import { ml } from 'plugins/ml/services/ml_api_service';
 
-import { labelDuplicateDetectorDescriptions } from 'plugins/ml/util/anomaly_utils';
+import { labelDuplicateDetectorDescriptions } from 'plugins/ml/../common/util/anomaly_utils';
 import { mlMessageBarService } from 'plugins/ml/components/messagebar/messagebar_service';
 import { isWebUrl } from 'plugins/ml/util/string_utils';
 import { ML_DATA_PREVIEW_COUNT } from 'plugins/ml/../common/util/job_utils';

x-pack/plugins/ml/public/services/ml_api_service/index.js

@@ -9,7 +9,9 @@
 import { pick } from 'lodash';
 import chrome from 'ui/chrome';
 
-import { http } from './http_service';
+import { http } from 'plugins/ml/services/http_service';
+
+import { results } from './results';
 
 const basePath = chrome.addBasePath('/api/ml');
 
@@ -403,4 +405,6 @@ export const ml = {
       data: obj
     });
   },
+
+  results
 };

x-pack/plugins/ml/public/services/ml_api_service/results.js

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// Service for obtaining data for the ML Results dashboards.
+
+import chrome from 'ui/chrome';
+
+import { http } from 'plugins/ml/services/http_service';
+
+const basePath = chrome.addBasePath('/api/ml');
+
+export const results = {
+  getAnomaliesTableData(
+    jobIds,
+    influencers,
+    aggregationInterval,
+    threshold,
+    earliestMs,
+    latestMs,
+    maxRecords,
+    maxExamples) {
+
+    return http({
+      url: `${basePath}/results/anomalies_table_data`,
+      method: 'POST',
+      data: {
+        jobIds,
+        influencers,
+        aggregationInterval,
+        threshold,
+        earliestMs,
+        latestMs,
+        maxRecords,
+        maxExamples
+      }
+    });
+  },
+
+  getCategoryDefinition(jobId, categoryId) {
+    return http({
+      url: `${basePath}/results/category_definition`,
+      method: 'POST',
+      data: { jobId, categoryId }
+    });
+  },
+
+  getCategoryExamples(
+    jobId,
+    categoryIds,
+    maxExamples
+  ) {
+
+    return http({
+      url: `${basePath}/results/category_examples`,
+      method: 'POST',
+      data: {
+        jobId,
+        categoryIds,
+        maxExamples
+      }
+    });
+  }
+};

x-pack/plugins/ml/public/services/results_service.js

@@ -12,7 +12,7 @@ import _ from 'lodash';
 
 import { ML_MEDIAN_PERCENTS } from 'plugins/ml/../common/util/job_utils';
 import { escapeForElasticsearchQuery } from 'plugins/ml/util/string_utils';
-import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
+import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/../common/constants/index_patterns';
 
 import { ml } from 'plugins/ml/services/ml_api_service';
 
@@ -699,88 +699,6 @@ function getInfluencerValueMaxScoreByTime(
   });
 }
 
-
-// Obtains the definition of the category with the specified ID and job ID.
-// Returned response contains four properties - categoryId, regex, examples
-// and terms (space delimited String of the common tokens matched in values of the category).
-function getCategoryDefinition(jobId, categoryId) {
-  return new Promise((resolve, reject) => {
-    const obj = { success: true, categoryId: categoryId, terms: null, regex: null, examples: [] };
-
-    ml.esSearch({
-      index: ML_RESULTS_INDEX_PATTERN,
-      size: 1,
-      body: {
-        query: {
-          bool: {
-            filter: [
-              { term: { job_id: jobId } },
-              { term: { category_id: categoryId } }
-            ]
-          }
-        }
-      }
-    })
-      .then((resp) => {
-        if (resp.hits.total !== 0) {
-          const source = _.first(resp.hits.hits)._source;
-          obj.categoryId = source.category_id;
-          obj.regex = source.regex;
-          obj.terms = source.terms;
-          obj.examples = source.examples;
-        }
-        resolve(obj);
-      })
-      .catch((resp) => {
-        reject(resp);
-      });
-  });
-}
-
-
-// Obtains the categorization examples for the categories with the specified IDs
-// from the given index and job ID.
-// Returned response contains two properties - jobId and
-// examplesByCategoryId (list of examples against categoryId).
-function getCategoryExamples(jobId, categoryIds, maxExamples) {
-  return new Promise((resolve, reject) => {
-    const obj = { success: true, jobId: jobId, examplesByCategoryId: {} };
-
-    ml.esSearch({
-      index: ML_RESULTS_INDEX_PATTERN,
-      size: 500,  // Matches size of records in anomaly summary table.
-      body: {
-        query: {
-          bool: {
-            filter: [
-              { term: { job_id: jobId } },
-              { terms: { category_id: categoryIds } }
-            ]
-          }
-        }
-      }
-    })
-      .then((resp) => {
-        if (resp.hits.total !== 0) {
-          _.each(resp.hits.hits, (hit) => {
-            if (maxExamples) {
-              obj.examplesByCategoryId[hit._source.category_id] =
-              _.slice(hit._source.examples, 0, Math.min(hit._source.examples.length, maxExamples));
-            } else {
-              obj.examplesByCategoryId[hit._source.category_id] = hit._source.examples;
-            }
-
-          });
-        }
-        resolve(obj);
-      })
-      .catch((resp) => {
-        reject(resp);
-      });
-  });
-}
-
-
 // Queries Elasticsearch to obtain record level results containing the influencers
 // for the specified job(s), record score threshold, and time range.
 // Pass an empty array or ['*'] to search over all job IDs.
@@ -1738,8 +1656,6 @@ export const mlResultsService = {
   getTopInfluencerValues,
   getOverallBucketScores,
   getInfluencerValueMaxScoreByTime,
-  getCategoryDefinition,
-  getCategoryExamples,
   getRecordInfluencers,
   getRecordsForInfluencer,
   getRecordsForDetector,

x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js

@@ -20,8 +20,8 @@ import 'ui/timefilter';
 
 import { ResizeChecker } from 'ui/resize_checker';
 
+import { getSeverityWithLow } from 'plugins/ml/../common/util/anomaly_utils';
 import { formatValue } from 'plugins/ml/formatters/format_value';
-import { getSeverityWithLow } from 'plugins/ml/util/anomaly_utils';
 import {
   drawLineChartDots,
   filterAxisLabels,

x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+import _ from 'lodash';
+import moment from 'moment';
+
+import {
+  getEntityFieldName,
+  getEntityFieldValue,
+  showActualForFunction,
+  showTypicalForFunction
+} from '../../../common/util/anomaly_utils';
+
+
+// Builds the items for display in the anomalies table from the supplied list of anomaly records.
+export function buildAnomalyTableItems(anomalyRecords, aggregationInterval) {
+
+  // Aggregate the anomaly records if necessary, and create skeleton display records with
+  // time, detector (description) and source record properties set.
+  let displayRecords = [];
+  if (aggregationInterval !== 'second') {
+    displayRecords = aggregateAnomalies(anomalyRecords, aggregationInterval);
+  } else {
+    // Show all anomaly records.
+    displayRecords = anomalyRecords.map((record) => {
+      return {
+        time: record.timestamp,
+        source: record,
+      };
+    });
+  }
+
+  // Fill out the remaining properties in each display record
+  // for the columns to be displayed in the table.
+  return displayRecords.map((record) => {
+    const source = record.source;
+    const jobId = source.job_id;
+
+    record.jobId = jobId;
+    record.detectorIndex = source.detector_index;
+    record.severity = source.record_score;
+
+    const entityName = getEntityFieldName(source);
+    if (entityName !== undefined) {
+      record.entityName = entityName;
+      record.entityValue = getEntityFieldValue(source);
+    }
+
+    if (source.influencers !== undefined) {
+      const influencers = [];
+      const sourceInfluencers = _.sortBy(source.influencers, 'influencer_field_name');
+      sourceInfluencers.forEach((influencer) => {
+        const influencerFieldName = influencer.influencer_field_name;
+        influencer.influencer_field_values.forEach((influencerFieldValue) => {
+          influencers.push({
+            [influencerFieldName]: influencerFieldValue
+          });
+        });
+      });
+      record.influencers = influencers;
+    }
+
+    const functionDescription = source.function_description || '';
+    const causes = source.causes || [];
+    if (showActualForFunction(functionDescription) === true) {
+      if (source.actual !== undefined) {
+        record.actual = source.actual;
+      } else {
+        // If only a single cause, copy values to the top level.
+        if (causes.length === 1) {
+          record.actual = causes[0].actual;
+        }
+      }
+    }
+    if (showTypicalForFunction(functionDescription) === true) {
+      if (source.typical !== undefined) {
+        record.typical = source.typical;
+      } else {
+        // If only a single cause, copy values to the top level.
+        if (causes.length === 1) {
+          record.typical = causes[0].typical;
+        }
+      }
+    }
+
+    return record;
+
+  });
+}
+
+function aggregateAnomalies(anomalyRecords, interval) {
+  // Aggregate the anomaly records by time, jobId, detectorIndex, and entity (by/over/partition).
+  // anomalyRecords assumed to be supplied in ascending time order.
+  if (anomalyRecords.length === 0) {
+    return [];
+  }
+
+  const aggregatedData = {};
+  anomalyRecords.forEach((record) => {
+    // Use moment.js to get start of interval.
+    const roundedTime = moment(record.timestamp).startOf(interval).valueOf();
+    if (aggregatedData[roundedTime] === undefined) {
+      aggregatedData[roundedTime] = {};
+    }
+
+    // Aggregate by job, then detectorIndex.
+    const jobId = record.job_id;
+    const jobsAtTime = aggregatedData[roundedTime];
+    if (jobsAtTime[jobId] === undefined) {
+      jobsAtTime[jobId] = {};
+    }
+
+    // Aggregate by detector - default to function_description if no description available.
+    const detectorIndex = record.detector_index;
+    const detectorsForJob = jobsAtTime[jobId];
+    if (detectorsForJob[detectorIndex] === undefined) {
+      detectorsForJob[detectorIndex] = {};
+    }
+
+    // Now add an object for the anomaly with the highest anomaly score per entity.
+    // For the choice of entity, look in order for byField, overField, partitionField.
+    // If no by/over/partition, default to an empty String.
+    const entitiesForDetector = detectorsForJob[detectorIndex];
+
+    // TODO - are we worried about different byFields having the same
+    // value e.g. host=server1 and machine=server1?
+    let entity = getEntityFieldValue(record);
+    if (entity === undefined) {
+      entity = '';
+    }
+    if (entitiesForDetector[entity] === undefined) {
+      entitiesForDetector[entity] = record;
+    } else {
+      if (record.record_score > entitiesForDetector[entity].record_score) {
+        entitiesForDetector[entity] = record;
+      }
+    }
+  });
+
+  // Flatten the aggregatedData to give a list of records with
+  // the highest score per bucketed time / jobId / detectorIndex.
+  const summaryRecords = [];
+  _.each(aggregatedData, (times, roundedTime) => {
+    _.each(times, (jobIds) => {
+      _.each(jobIds, (entityDetectors) => {
+        _.each(entityDetectors, (record) => {
+          summaryRecords.push({
+            time: +roundedTime,
+            source: record
+          });
+        });
+      });
+    });
+  });
+
+  return summaryRecords;
+
+}

x-pack/plugins/ml/server/models/results_service/index.js

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+
+export { resultsServiceProvider } from './results_service';

x-pack/plugins/ml/server/models/results_service/results_service.js

@@ -0,0 +1,257 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+
+import _ from 'lodash';
+import moment from 'moment';
+
+import { buildAnomalyTableItems } from './build_anomaly_table_items';
+import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patterns';
+
+
+// Service for carrying out Elasticsearch queries to obtain data for the
+// ML Results dashboards.
+
+const DEFAULT_QUERY_SIZE = 500;
+
+export function resultsServiceProvider(callWithRequest) {
+
+  // Obtains data for the anomalies table, aggregating anomalies by day or hour as requested.
+  // Return an Object with properties 'anomalies' and 'interval' (interval used to aggregate anomalies,
+  // one of day, hour or second. Note 'auto' can be provided as the aggregationInterval in the request,
+  // in which case the interval is determined according to the time span between the first and
+  // last anomalies),  plus an examplesByJobId property if any of the
+  // anomalies are categorization anomalies in mlcategory.
+  async function getAnomaliesTableData(
+    jobIds,
+    influencers,
+    aggregationInterval,
+    threshold,
+    earliestMs,
+    latestMs,
+    maxRecords,
+    maxExamples) {
+
+    // Build the query to return the matching anomaly record results.
+    // Add criteria for the time range, record score, plus any specified job IDs.
+    const boolCriteria = [
+      {
+        range: {
+          timestamp: {
+            gte: earliestMs,
+            lte: latestMs,
+            format: 'epoch_millis'
+          }
+        }
+      },
+      {
+        range: {
+          record_score: {
+            gte: threshold,
+          }
+        }
+      }
+    ];
+
+    if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) {
+      let jobIdFilterStr = '';
+      jobIds.forEach((jobId, i) => {
+        if (i > 0) {
+          jobIdFilterStr += ' OR ';
+        }
+        jobIdFilterStr += 'job_id:';
+        jobIdFilterStr += jobId;
+      });
+      boolCriteria.push({
+        query_string: {
+          analyze_wildcard: false,
+          query: jobIdFilterStr
+        }
+      });
+    }
+
+    // Add a nested query to filter for each of the specified influencers.
+    if (influencers.length > 0) {
+      boolCriteria.push({
+        bool: {
+          should: influencers.map((influencer) => {
+            return {
+              nested: {
+                path: 'influencers',
+                query: {
+                  bool: {
+                    must: [
+                      {
+                        match: {
+                          'influencers.influencer_field_name': influencer.fieldName
+                        }
+                      },
+                      {
+                        match: {
+                          'influencers.influencer_field_values': influencer.fieldValue
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            };
+          }),
+          minimum_should_match: 1,
+        }
+      });
+    }
+
+    const resp = await callWithRequest('search', {
+      index: ML_RESULTS_INDEX_PATTERN,
+      size: maxRecords !== undefined ? maxRecords : DEFAULT_QUERY_SIZE,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {
+                query_string: {
+                  query: 'result_type:record',
+                  analyze_wildcard: false
+                }
+              },
+              {
+                bool: {
+                  must: boolCriteria
+                }
+              }
+            ]
+          }
+        },
+        sort: [
+          { record_score: { order: 'desc' } }
+        ]
+      }
+    });
+
+    const tableData = { anomalies: [], interval: 'second' };
+    if (resp.hits.total !== 0) {
+      let records = [];
+      resp.hits.hits.forEach((hit) => {
+        records.push(hit._source);
+      });
+
+      // Sort anomalies in ascending time order.
+      records = _.sortBy(records, 'timestamp');
+      tableData.interval = aggregationInterval;
+      if (aggregationInterval === 'auto') {
+        // Determine the actual interval to use if aggregating.
+        const earliest = moment(records[0].timestamp);
+        const latest = moment(records[records.length - 1].timestamp);
+
+        const daysDiff = latest.diff(earliest, 'days');
+        tableData.interval = (daysDiff < 2 ? 'hour' : 'day');
+      }
+
+      tableData.anomalies = buildAnomalyTableItems(records, tableData.interval);
+
+      // Load examples for any categorization anomalies.
+      const categoryAnomalies = tableData.anomalies.filter(item => item.entityName === 'mlcategory');
+      if (categoryAnomalies.length > 0) {
+        tableData.examplesByJobId = {};
+
+        const categoryIdsByJobId = {};
+        categoryAnomalies.forEach((anomaly) => {
+          if (!_.has(categoryIdsByJobId, anomaly.jobId)) {
+            categoryIdsByJobId[anomaly.jobId] = [];
+          }
+          if (categoryIdsByJobId[anomaly.jobId].indexOf(anomaly.entityValue) === -1) {
+            categoryIdsByJobId[anomaly.jobId].push(anomaly.entityValue);
+          }
+        });
+
+        const categoryJobIds = Object.keys(categoryIdsByJobId);
+        await Promise.all(categoryJobIds.map(async (jobId) => {
+          const examplesByCategoryId = await getCategoryExamples(jobId, categoryIdsByJobId[jobId], maxExamples);
+          tableData.examplesByJobId[jobId] = examplesByCategoryId;
+        }));
+      }
+
+    }
+
+    return tableData;
+
+  }
+
+
+  // Obtains the categorization examples for the categories with the specified IDs
+  // from the given index and job ID.
+  // Returned response consists of a list of examples against category ID.
+  async function getCategoryExamples(jobId, categoryIds, maxExamples) {
+    const resp = await callWithRequest('search', {
+      index: ML_RESULTS_INDEX_PATTERN,
+      size: DEFAULT_QUERY_SIZE,    // Matches size of records in anomaly summary table.
+      body: {
+        query: {
+          bool: {
+            filter: [
+              { term: { job_id: jobId } },
+              { terms: { category_id: categoryIds } }
+            ]
+          }
+        }
+      }
+    });
+
+    const examplesByCategoryId = {};
+    if (resp.hits.total !== 0) {
+      resp.hits.hits.forEach((hit) => {
+        if (maxExamples) {
+          examplesByCategoryId[hit._source.category_id] =
+            _.slice(hit._source.examples, 0, Math.min(hit._source.examples.length, maxExamples));
+        } else {
+          examplesByCategoryId[hit._source.category_id] = hit._source.examples;
+        }
+      });
+    }
+
+    return examplesByCategoryId;
+  }
+
+  // Obtains the definition of the category with the specified ID and job ID.
+  // Returned response contains four properties - categoryId, regex, examples
+  // and terms (space delimited String of the common tokens matched in values of the category).
+  async function getCategoryDefinition(jobId, categoryId) {
+    const resp = await callWithRequest('search', {
+      index: ML_RESULTS_INDEX_PATTERN,
+      size: 1,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              { term: { job_id: jobId } },
+              { term: { category_id: categoryId } }
+            ]
+          }
+        }
+      }
+    });
+
+    const definition = { categoryId, terms: null, regex: null, examples: [] };
+    if (resp.hits.total !== 0) {
+      const source = resp.hits.hits[0]._source;
+      definition.categoryId = source.category_id;
+      definition.regex = source.regex;
+      definition.terms = source.terms;
+      definition.examples = source.examples;
+    }
+
+    return definition;
+  }
+
+  return {
+    getAnomaliesTableData,
+    getCategoryDefinition,
+    getCategoryExamples
+  };
+
+}

x-pack/plugins/ml/server/routes/results_service.js

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+
+import { callWithRequestFactory } from '../client/call_with_request_factory';
+import { wrapError } from '../client/errors';
+import { resultsServiceProvider } from '../models/results_service';
+
+
+function getAnomaliesTableData(callWithRequest, payload) {
+  const rs = resultsServiceProvider(callWithRequest);
+  const {
+    jobIds,
+    influencers,
+    aggregationInterval,
+    threshold,
+    earliestMs,
+    latestMs,
+    maxRecords,
+    maxExamples } = payload;
+  return rs.getAnomaliesTableData(
+    jobIds,
+    influencers,
+    aggregationInterval,
+    threshold,
+    earliestMs,
+    latestMs,
+    maxRecords,
+    maxExamples);
+}
+
+function getCategoryDefinition(callWithRequest, payload) {
+  const rs = resultsServiceProvider(callWithRequest);
+  return rs.getCategoryDefinition(
+    payload.jobId,
+    payload.categoryId);
+}
+
+function getCategoryExamples(callWithRequest, payload) {
+  const rs = resultsServiceProvider(callWithRequest);
+  const {
+    jobId,
+    categoryIds,
+    maxExamples } = payload;
+  return rs.getCategoryExamples(
+    jobId,
+    categoryIds,
+    maxExamples);
+}
+
+export function resultsServiceRoutes(server, commonRouteConfig) {
+
+  server.route({
+    method: 'POST',
+    path: '/api/ml/results/anomalies_table_data',
+    handler(request, reply) {
+      const callWithRequest = callWithRequestFactory(server, request);
+      return getAnomaliesTableData(callWithRequest, request.payload)
+        .then(resp => reply(resp))
+        .catch(resp => reply(wrapError(resp)));
+    },
+    config: {
+      ...commonRouteConfig
+    }
+  });
+
+  server.route({
+    method: 'POST',
+    path: '/api/ml/results/category_definition',
+    handler(request, reply) {
+      const callWithRequest = callWithRequestFactory(server, request);
+      return getCategoryDefinition(callWithRequest, request.payload)
+        .then(resp => reply(resp))
+        .catch(resp => reply(wrapError(resp)));
+    },
+    config: {
+      ...commonRouteConfig
+    }
+  });
+
+  server.route({
+    method: 'POST',
+    path: '/api/ml/results/category_examples',
+    handler(request, reply) {
+      const callWithRequest = callWithRequestFactory(server, request);
+      return getCategoryExamples(callWithRequest, request.payload)
+        .then(resp => reply(resp))
+        .catch(resp => reply(wrapError(resp)));
+    },
+    config: {
+      ...commonRouteConfig
+    }
+  });
+
+}