[ML] Only show _doc_count for anomaly detection job summary count field (#85444)

* [ML] Only show _doc_count for anomaly detection job summary count field

* [ML] Use array filter for removing _DOC_COUNT field

x-pack/plugins/ml/common/constants/field_types.ts

@@ -16,7 +16,17 @@ export enum ML_JOB_FIELD_TYPES {
 }
 
 export const MLCATEGORY = 'mlcategory';
+
+/**
+ * For use as summary_count_field_name in datafeeds which use aggregations.
+ */
 export const DOC_COUNT = 'doc_count';
 
+/**
+ * Elasticsearch field showing number of documents aggregated in a single summary field for
+ * pre-aggregated data. For use as summary_count_field_name in datafeeds which do not use aggregations.
+ */
+export const _DOC_COUNT = '_doc_count';
+
 // List of system fields we don't want to display.
 export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score'];

x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts

@@ -13,7 +13,11 @@ import {
   ML_JOB_AGGREGATION,
   SPARSE_DATA_AGGREGATIONS,
 } from '../../../../../../../common/constants/aggregation_types';
-import { MLCATEGORY, DOC_COUNT } from '../../../../../../../common/constants/field_types';
+import {
+  MLCATEGORY,
+  DOC_COUNT,
+  _DOC_COUNT,
+} from '../../../../../../../common/constants/field_types';
 import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
 import {
   EVENT_RATE_FIELD_ID,
@@ -113,7 +117,11 @@ export function createDocCountFieldOption(usingAggregations: boolean) {
           label: DOC_COUNT,
         },
       ]
-    : [];
+    : [
+        {
+          label: _DOC_COUNT,
+        },
+      ];
 }
 
 function getDetectorsAdvanced(job: Job, datafeed: Datafeed) {

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field_select.tsx

@@ -23,8 +23,8 @@ interface Props {
 export const SummaryCountFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
   const { jobCreator } = useContext(JobCreatorContext);
   const options: EuiComboBoxOptionOption[] = [
-    ...createFieldOptions(fields, jobCreator.additionalFields),
     ...createDocCountFieldOption(jobCreator.aggregationFields.length > 0),
+    ...createFieldOptions(fields, jobCreator.additionalFields),
   ];
 
   const selection: EuiComboBoxOptionOption[] = [];

x-pack/plugins/ml/public/application/util/chart_config_builder.js

@@ -11,6 +11,8 @@
 
 import { get } from 'lodash';
 
+import { ES_AGGREGATION, ML_JOB_AGGREGATION } from '../../../common/constants/aggregation_types';
+import { DOC_COUNT, _DOC_COUNT } from '../../../common/constants/field_types';
 import { mlFunctionToESAggregation } from '../../../common/util/job_utils';
 
 // Builds the basic configuration to plot a chart of the source data
@@ -35,9 +37,10 @@ export function buildConfigFromDetector(job, detectorIndex) {
   // Extra checks if the job config uses a summary count field.
   const summaryCountFieldName = analysisConfig.summary_count_field_name;
   if (
-    config.metricFunction === 'count' &&
+    config.metricFunction === ES_AGGREGATION.COUNT &&
     summaryCountFieldName !== undefined &&
-    summaryCountFieldName !== 'doc_count'
+    summaryCountFieldName !== DOC_COUNT &&
+    summaryCountFieldName !== _DOC_COUNT
   ) {
     // Check for a detector looking at cardinality (distinct count) using an aggregation.
     // The cardinality field will be in:
@@ -50,18 +53,23 @@ export function buildConfigFromDetector(job, detectorIndex) {
         get(Object.values(topAgg)[0], [
           'aggregations',
           summaryCountFieldName,
-          'cardinality',
+          ES_AGGREGATION.CARDINALITY,
           'field',
         ]) ||
-        get(Object.values(topAgg)[0], ['aggs', summaryCountFieldName, 'cardinality', 'field']);
+        get(Object.values(topAgg)[0], [
+          'aggs',
+          summaryCountFieldName,
+          ES_AGGREGATION.CARDINALITY,
+          'field',
+        ]);
     }
 
-    if (detector.function === 'non_zero_count' && cardinalityField !== undefined) {
-      config.metricFunction = 'cardinality';
+    if (detector.function === ML_JOB_AGGREGATION.NON_ZERO_COUNT && cardinalityField !== undefined) {
+      config.metricFunction = ES_AGGREGATION.CARDINALITY;
       config.metricFieldName = cardinalityField;
     } else {
       // For count detectors using summary_count_field, plot sum(summary_count_field_name)
-      config.metricFunction = 'sum';
+      config.metricFunction = ES_AGGREGATION.SUM;
       config.metricFieldName = summaryCountFieldName;
     }
   }

x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts

@@ -5,6 +5,7 @@
  */
 
 import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server';
+import { _DOC_COUNT } from '../../../../common/constants/field_types';
 import { Aggregation, Field, NewJobCaps } from '../../../../common/types/fields';
 import { fieldServiceProvider } from './field_service';
 
@@ -22,10 +23,12 @@ export function newJobCapsProvider(client: IScopedClusterClient) {
     const { aggs, fields } = await fieldService.getData();
     convertForStringify(aggs, fields);
 
+    // Remove the _doc_count field as we don't want to display this in the fields lists in the UI
+    const fieldsWithoutDocCount = fields.filter(({ id }) => id !== _DOC_COUNT);
     return {
       [indexPattern]: {
         aggs,
-        fields,
+        fields: fieldsWithoutDocCount,
       },
     };
   }