[Transform] Fix transform preview for the latest function (#87168)

* [Transform] retrieve mappings from the source index

* [Transform] fix preview for nested props

* [Transform] exclude meta fields

* [Transform] use agg config to only suggest term agg supported fields

* [Transform] refactor

* [Transform] remove incorrect data mock

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/transform/common/types/transform.ts

@@ -49,9 +49,7 @@ export function isPivotTransform(
   return transform.hasOwnProperty('pivot');
 }
 
-export function isLatestTransform(
-  transform: TransformBaseConfig
-): transform is TransformLatestConfig {
+export function isLatestTransform(transform: any): transform is TransformLatestConfig {
   return transform.hasOwnProperty('latest');
 }
 

x-pack/plugins/transform/common/utils/object_utils.ts

@@ -6,17 +6,32 @@
 
 // This is similar to lodash's get() except that it's TypeScript aware and is able to infer return types.
 // It splits the attribute key string and uses reduce with an idx check to access nested attributes.
-export const getNestedProperty = (
+export function getNestedProperty(
   obj: Record<string, any>,
   accessor: string,
   defaultValue?: any
-) => {
-  const value = accessor.split('.').reduce((o, i) => o?.[i], obj);
+): any {
+  const accessorKeys = accessor.split('.');
 
-  if (value === undefined) return defaultValue;
+  let o = obj;
+  for (let i = 0; i < accessorKeys.length; i++) {
+    const keyPart = accessorKeys[i];
+    o = o?.[keyPart];
+    if (Array.isArray(o)) {
+      o = o.map((v) =>
+        typeof v === 'object'
+          ? // from this point we need to resolve path for each element in the collection
+            getNestedProperty(v, accessorKeys.slice(i + 1, accessorKeys.length).join('.'))
+          : v
+      );
+      break;
+    }
+  }
 
-  return value;
-};
+  if (o === undefined) return defaultValue;
+
+  return o;
+}
 
 export const setNestedProperty = (obj: Record<string, any>, accessor: string, value: any) => {
   let ref = obj;

x-pack/plugins/transform/public/app/__mocks__/app_dependencies.tsx

@@ -31,7 +31,7 @@ const appDependencies = {
 
 export const useAppDependencies = () => {
   const ml = useContext(MlSharedContext);
-  return { ...appDependencies, ml, savedObjects: jest.fn(), data: jest.fn() };
+  return { ...appDependencies, ml, savedObjects: jest.fn() };
 };
 
 export const useToastNotifications = () => {

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts

@@ -11,6 +11,8 @@ import { LatestFunctionConfigUI } from '../../../../../../../common/types/transf
 import { StepDefineFormProps } from '../step_define_form';
 import { StepDefineExposedState } from '../common';
 import { LatestFunctionConfig } from '../../../../../../../common/api_schemas/transforms';
+import { AggConfigs, FieldParamType } from '../../../../../../../../../../src/plugins/data/common';
+import { useAppDependencies } from '../../../../../app_dependencies';
 
 /**
  * Latest function config mapper between API and UI
@@ -32,26 +34,33 @@ export const latestConfigMapper = {
 /**
  * Provides available options for unique_key and sort fields
  * @param indexPattern
+ * @param aggConfigs
  */
-function getOptions(indexPattern: StepDefineFormProps['searchItems']['indexPattern']) {
-  const uniqueKeyOptions: Array<EuiComboBoxOptionOption<string>> = [];
-  const sortFieldOptions: Array<EuiComboBoxOptionOption<string>> = [];
+function getOptions(
+  indexPattern: StepDefineFormProps['searchItems']['indexPattern'],
+  aggConfigs: AggConfigs
+) {
+  const aggConfig = aggConfigs.aggs[0];
+  const param = aggConfig.type.params.find((p) => p.type === 'field');
+  const filteredIndexPatternFields = param
+    ? ((param as unknown) as FieldParamType).getAvailableFields(aggConfig)
+    : [];
 
-  const ignoreFieldNames = new Set(['_id', '_index', '_type']);
+  const ignoreFieldNames = new Set(['_source', '_type', '_index', '_id', '_version', '_score']);
 
-  for (const field of indexPattern.fields) {
-    if (ignoreFieldNames.has(field.name)) {
-      continue;
-    }
+  const uniqueKeyOptions: Array<EuiComboBoxOptionOption<string>> = filteredIndexPatternFields
+    .filter((v) => !ignoreFieldNames.has(v.name))
+    .map((v) => ({
+      label: v.displayName,
+      value: v.name,
+    }));
 
-    if (field.aggregatable) {
-      uniqueKeyOptions.push({ label: field.displayName, value: field.name });
-    }
-
-    if (field.sortable) {
-      sortFieldOptions.push({ label: field.displayName, value: field.name });
-    }
-  }
+  const sortFieldOptions: Array<EuiComboBoxOptionOption<string>> = indexPattern.fields
+    .filter((v) => !ignoreFieldNames.has(v.name) && v.sortable)
+    .map((v) => ({
+      label: v.displayName,
+      value: v.name,
+    }));
 
   return { uniqueKeyOptions, sortFieldOptions };
 }
@@ -92,9 +101,12 @@ export function useLatestFunctionConfig(
     sort: defaults.sort,
   });
 
-  const { uniqueKeyOptions, sortFieldOptions } = useMemo(() => getOptions(indexPattern), [
-    indexPattern,
-  ]);
+  const { data } = useAppDependencies();
+
+  const { uniqueKeyOptions, sortFieldOptions } = useMemo(() => {
+    const aggConfigs = data.search.aggs.createAggConfigs(indexPattern, [{ type: 'terms' }]);
+    return getOptions(indexPattern, aggConfigs);
+  }, [indexPattern, data.search.aggs]);
 
   const updateLatestFunctionConfig = useCallback(
     (update) =>

x-pack/plugins/transform/server/routes/api/transforms.ts

@@ -57,6 +57,7 @@ import { addBasePath } from '../index';
 import { isRequestTimeout, fillResultsWithTimeouts, wrapError, wrapEsError } from './error_utils';
 import { registerTransformsAuditMessagesRoutes } from './transforms_audit_messages';
 import { IIndexPattern } from '../../../../../../src/plugins/data/common/index_patterns';
+import { isLatestTransform } from '../../../common/types/transform';
 
 enum TRANSFORM_ACTIONS {
   STOP = 'stop',
@@ -531,9 +532,36 @@ const previewTransformHandler: RequestHandler<
   PostTransformsPreviewRequestSchema
 > = async (ctx, req, res) => {
   try {
+    const reqBody = req.body;
     const { body } = await ctx.core.elasticsearch.client.asCurrentUser.transform.previewTransform({
-      body: req.body,
+      body: reqBody,
     });
+    if (isLatestTransform(reqBody)) {
+      // for the latest transform mappings properties have to be retrieved from the source
+      const fieldCapsResponse = await ctx.core.elasticsearch.client.asCurrentUser.fieldCaps({
+        index: reqBody.source.index,
+        fields: '*',
+        include_unmapped: false,
+      });
+
+      const fieldNamesSet = new Set(Object.keys(fieldCapsResponse.body.fields));
+
+      const fields = Object.entries(
+        fieldCapsResponse.body.fields as Record<string, Record<string, { type: string }>>
+      ).reduce((acc, [fieldName, fieldCaps]) => {
+        const fieldDefinition = Object.values(fieldCaps)[0];
+        const isMetaField = fieldDefinition.type.startsWith('_') || fieldName === '_doc_count';
+        const isKeywordDuplicate =
+          fieldName.endsWith('.keyword') && fieldNamesSet.has(fieldName.split('.keyword')[0]);
+        if (isMetaField || isKeywordDuplicate) {
+          return acc;
+        }
+        acc[fieldName] = { ...fieldDefinition };
+        return acc;
+      }, {} as Record<string, { type: string }>);
+
+      body.generated_dest_index.mappings.properties = fields;
+    }
     return res.ok({ body });
   } catch (e) {
     return res.customError(wrapError(wrapEsError(e)));