From 3e4469c99c28f8bb1b517bf2ed41539af73f4ffa Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Wed, 8 Apr 2020 13:13:40 -0400 Subject: [PATCH] [ML] Analytics: ensure both keyword/text types are excluded for selected excluded field (#62712) * exclude keyword and text types of field selected for exclusion * only show keyword type fields of accepted fields for depVar * make excludes field logic generic * fix regex to ensure escaped dot. reset regex and mainfield * ensure cloned jobs get correct excluded fields * add clarifying comments --- .../create_analytics_form.tsx | 2 +- .../form_options_validation.ts | 2 +- .../hooks/use_create_analytics_form/state.ts | 52 ++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx index e5f30a50ed8f..0c83dfb6a234 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/create_analytics_form.tsx @@ -250,7 +250,7 @@ export const CreateAnalyticsForm: FC = ({ actions, sta dependentVariableOptions: [] as State['form']['dependentVariableOptions'], }; - await newJobCapsService.initializeFromIndexPattern(indexPattern); + await newJobCapsService.initializeFromIndexPattern(indexPattern, false, false); // Get fields and filter for supported types for job type const { fields } = newJobCapsService; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/form_options_validation.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/form_options_validation.ts index 9c0bc69f4b41..4bd03fec7cc7 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/form_options_validation.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/create_analytics_form/form_options_validation.ts @@ -10,7 +10,7 @@ import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics'; import { AnalyticsJobType } from '../../hooks/use_create_analytics_form/state'; import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields'; -const CATEGORICAL_TYPES = new Set(['ip', 'keyword', 'text']); +const CATEGORICAL_TYPES = new Set(['ip', 'keyword']); // List of system fields we want to ignore for the numeric field check. export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score']; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts index 01a39d2ef9f3..e121268e65e8 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts @@ -8,6 +8,7 @@ import { EuiComboBoxOptionOption } from '@elastic/eui'; import { DeepPartial, DeepReadonly } from '../../../../../../../common/types/common'; import { checkPermission } from '../../../../../privilege/check_privilege'; import { mlNodesAvailable } from '../../../../../ml_nodes_check'; +import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; import { isClassificationAnalysis, @@ -158,6 +159,55 @@ export const getInitialState = (): State => ({ estimatedModelMemoryLimit: '', }); +const getExcludesFields = (excluded: string[]) => { + const { fields } = newJobCapsService; + const updatedExcluded: string[] = []; + // Loop through excluded fields to check for multiple types of same field + for (let i = 0; i < excluded.length; i++) { + const fieldName = excluded[i]; + let mainField; + + // No dot in fieldName - it is the main field + if (fieldName.includes('.') === false) { + mainField = fieldName; + } else { + // Dot in fieldName - check if there's a field whose name equals the fieldName with the last dot suffix removed + const regex = /\.[^.]*$/; + const suffixRemovedField = fieldName.replace(regex, ''); + const fieldMatch = newJobCapsService.getFieldById(suffixRemovedField); + + // There's a match - set as the main field + if (fieldMatch !== null) { + mainField = suffixRemovedField; + } else { + // No main field to be found - add the fieldName to updatedExcluded array if it's not already there + if (updatedExcluded.includes(fieldName) === false) { + updatedExcluded.push(fieldName); + } + } + } + + if (mainField !== undefined) { + // Add the main field to the updatedExcluded array if it's not already there + if (updatedExcluded.includes(mainField) === false) { + updatedExcluded.push(mainField); + } + // Create regex to find all other fields whose names begin with main field followed by a dot + const regex = new RegExp(`${mainField}\\..+`); + + // Loop through fields and add fields matching the pattern to updatedExcluded array + for (let j = 0; j < fields.length; j++) { + const field = fields[j].name; + if (updatedExcluded.includes(field) === false && field.match(regex) !== null) { + updatedExcluded.push(field); + } + } + } + } + + return updatedExcluded; +}; + export const getJobConfigFromFormState = ( formState: State['form'] ): DeepPartial => { @@ -175,7 +225,7 @@ export const getJobConfigFromFormState = ( index: formState.destinationIndex, }, analyzed_fields: { - excludes: formState.excludes, + excludes: getExcludesFields(formState.excludes), }, analysis: { outlier_detection: {},