[ML] Analytics: ensure both keyword/text types are excluded for selected excluded field (#62712)

* exclude keyword and text types of field selected for exclusion

* only show keyword type fields of accepted fields for depVar

* make excludes field logic generic

* fix regex to ensure escaped dot. reset regex and mainfield

* ensure cloned jobs get correct excluded fields

* add clarifying comments

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx

@@ -250,7 +250,7 @@ export const CreateAnalyticsForm: FC<CreateAnalyticsFormProps> = ({ actions, sta
           dependentVariableOptions: [] as State['form']['dependentVariableOptions'],
         };
 
-        await newJobCapsService.initializeFromIndexPattern(indexPattern);
+        await newJobCapsService.initializeFromIndexPattern(indexPattern, false, false);
         // Get fields and filter for supported types for job type
         const { fields } = newJobCapsService;
 

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/form_options_validation.ts

@@ -10,7 +10,7 @@ import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics';
 import { AnalyticsJobType } from '../../hooks/use_create_analytics_form/state';
 import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields';
 
-const CATEGORICAL_TYPES = new Set(['ip', 'keyword', 'text']);
+const CATEGORICAL_TYPES = new Set(['ip', 'keyword']);
 
 // List of system fields we want to ignore for the numeric field check.
 export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score'];

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts

@@ -8,6 +8,7 @@ import { EuiComboBoxOptionOption } from '@elastic/eui';
 import { DeepPartial, DeepReadonly } from '../../../../../../../common/types/common';
 import { checkPermission } from '../../../../../privilege/check_privilege';
 import { mlNodesAvailable } from '../../../../../ml_nodes_check';
+import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
 
 import {
   isClassificationAnalysis,
@@ -158,6 +159,55 @@ export const getInitialState = (): State => ({
   estimatedModelMemoryLimit: '',
 });
 
+const getExcludesFields = (excluded: string[]) => {
+  const { fields } = newJobCapsService;
+  const updatedExcluded: string[] = [];
+  // Loop through excluded fields to check for multiple types of same field
+  for (let i = 0; i < excluded.length; i++) {
+    const fieldName = excluded[i];
+    let mainField;
+
+    // No dot in fieldName - it is the main field
+    if (fieldName.includes('.') === false) {
+      mainField = fieldName;
+    } else {
+      // Dot in fieldName - check if there's a field whose name equals the fieldName with the last dot suffix removed
+      const regex = /\.[^.]*$/;
+      const suffixRemovedField = fieldName.replace(regex, '');
+      const fieldMatch = newJobCapsService.getFieldById(suffixRemovedField);
+
+      // There's a match - set as the main field
+      if (fieldMatch !== null) {
+        mainField = suffixRemovedField;
+      } else {
+        // No main field to be found - add the fieldName to updatedExcluded array if it's not already there
+        if (updatedExcluded.includes(fieldName) === false) {
+          updatedExcluded.push(fieldName);
+        }
+      }
+    }
+
+    if (mainField !== undefined) {
+      // Add the main field to the updatedExcluded array if it's not already there
+      if (updatedExcluded.includes(mainField) === false) {
+        updatedExcluded.push(mainField);
+      }
+      // Create regex to find all other fields whose names begin with main field followed by a dot
+      const regex = new RegExp(`${mainField}\\..+`);
+
+      // Loop through fields and add fields matching the pattern to updatedExcluded array
+      for (let j = 0; j < fields.length; j++) {
+        const field = fields[j].name;
+        if (updatedExcluded.includes(field) === false && field.match(regex) !== null) {
+          updatedExcluded.push(field);
+        }
+      }
+    }
+  }
+
+  return updatedExcluded;
+};
+
 export const getJobConfigFromFormState = (
   formState: State['form']
 ): DeepPartial<DataFrameAnalyticsConfig> => {
@@ -175,7 +225,7 @@ export const getJobConfigFromFormState = (
       index: formState.destinationIndex,
     },
     analyzed_fields: {
-      excludes: formState.excludes,
+      excludes: getExcludesFields(formState.excludes),
     },
     analysis: {
       outlier_detection: {},