diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.scss b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.scss index 70fb57ee79ee..155b954e9cf1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.scss @@ -10,27 +10,6 @@ margin-bottom: $euiSizeS; } -/** - * 1. Don't cut off the shadow of the field items - */ - -.lnsInnerIndexPatternDataPanel__listWrapper { - @include euiOverflowShadow; - @include euiScrollBar; - margin-left: -$euiSize; /* 1 */ - position: relative; - flex-grow: 1; - overflow: auto; -} - -.lnsInnerIndexPatternDataPanel__list { - padding-top: $euiSizeS; - position: absolute; - top: 0; - left: $euiSize; /* 1 */ - right: $euiSizeXS; /* 1 */ -} - .lnsInnerIndexPatternDataPanel__fieldItems { // Quick fix for making sure the shadow and focus rings are visible outside the accordion bounds padding: $euiSizeXS $euiSizeXS 0; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx index f17bf172b0fb..7fb64d1613d3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx @@ -623,11 +623,40 @@ describe('IndexPattern Data Panel', () => { ).toEqual(['client', 'source', 'timestampLabel']); }); + it('should show meta fields accordion', async () => { + const wrapper = mountWithIntl( + + ); + wrapper + .find('[data-test-subj="lnsIndexPatternMetaFields"]') + .find('button') + .first() + .simulate('click'); + expect( + wrapper + .find('[data-test-subj="lnsIndexPatternMetaFields"]') + .find(FieldItem) + .first() + .prop('field').name + ).toEqual('_id'); + }); + it('should display NoFieldsCallout when all fields are empty', async () => { const wrapper = mountWithIntl( ); - expect(wrapper.find(NoFieldsCallout).length).toEqual(1); + expect(wrapper.find(NoFieldsCallout).length).toEqual(2); expect( wrapper .find('[data-test-subj="lnsIndexPatternAvailableFields"]') @@ -654,7 +683,7 @@ describe('IndexPattern Data Panel', () => { .length ).toEqual(1); wrapper.setProps({ existingFields: { idx1: {} } }); - expect(wrapper.find(NoFieldsCallout).length).toEqual(1); + expect(wrapper.find(NoFieldsCallout).length).toEqual(2); }); it('should filter down by name', () => { @@ -699,7 +728,7 @@ describe('IndexPattern Data Panel', () => { expect(wrapper.find(FieldItem).map((fieldItem) => fieldItem.prop('field').name)).toEqual([ 'Records', ]); - expect(wrapper.find(NoFieldsCallout).length).toEqual(2); + expect(wrapper.find(NoFieldsCallout).length).toEqual(3); }); it('should toggle type if clicked again', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index f7adf91e307d..4e85cb5b5d46 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -5,14 +5,13 @@ */ import './datapanel.scss'; -import { uniq, keyBy, groupBy, throttle } from 'lodash'; -import React, { useState, useEffect, memo, useCallback, useMemo } from 'react'; +import { uniq, keyBy, groupBy } from 'lodash'; +import React, { useState, memo, useCallback, useMemo } from 'react'; import { EuiFlexGroup, EuiFlexItem, EuiContextMenuPanel, EuiContextMenuItem, - EuiContextMenuPanelProps, EuiPopover, EuiCallOut, EuiFormControlLayout, @@ -25,8 +24,6 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { DataPublicPluginStart, EsQueryConfig, Query, Filter } from 'src/plugins/data/public'; import { DatasourceDataPanelProps, DataType, StateSetter } from '../types'; import { ChildDragDropProvider, DragContextState } from '../drag_drop'; -import { FieldItem } from './field_item'; -import { NoFieldsCallout } from './no_fields_callout'; import { IndexPattern, IndexPatternPrivateState, @@ -37,7 +34,6 @@ import { trackUiEvent } from '../lens_ui_telemetry'; import { syncExistingFields } from './loader'; import { fieldExists } from './pure_helpers'; import { Loader } from '../loader'; -import { FieldsAccordion } from './fields_accordion'; import { esQuery, IIndexPattern } from '../../../../../src/plugins/data/public'; export type Props = DatasourceDataPanelProps & { @@ -52,18 +48,13 @@ export type Props = DatasourceDataPanelProps & { import { LensFieldIcon } from './lens_field_icon'; import { ChangeIndexPattern } from './change_indexpattern'; import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; - -// TODO the typings for EuiContextMenuPanel are incorrect - watchedItemProps is missing. This can be removed when the types are adjusted -const FixedEuiContextMenuPanel = (EuiContextMenuPanel as unknown) as React.FunctionComponent< - EuiContextMenuPanelProps & { watchedItemProps: string[] } ->; +import { FieldGroups, FieldList } from './field_list'; function sortFields(fieldA: IndexPatternField, fieldB: IndexPatternField) { return fieldA.displayName.localeCompare(fieldB.displayName, undefined, { sensitivity: 'base' }); } const supportedFieldTypes = new Set(['string', 'number', 'boolean', 'date', 'ip', 'document']); -const PAGINATION_SIZE = 50; const fieldTypeNames: Record = { document: i18n.translate('xpack.lens.datatypes.record', { defaultMessage: 'record' }), @@ -212,18 +203,19 @@ interface DataPanelState { isTypeFilterOpen: boolean; isAvailableAccordionOpen: boolean; isEmptyAccordionOpen: boolean; + isMetaAccordionOpen: boolean; } -export interface FieldsGroup { +const defaultFieldGroups: { specialFields: IndexPatternField[]; availableFields: IndexPatternField[]; emptyFields: IndexPatternField[]; -} - -const defaultFieldGroups = { + metaFields: IndexPatternField[]; +} = { specialFields: [], availableFields: [], emptyFields: [], + metaFields: [], }; const fieldFiltersLabel = i18n.translate('xpack.lens.indexPatterns.fieldFiltersLabel', { @@ -261,9 +253,8 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ isTypeFilterOpen: false, isAvailableAccordionOpen: true, isEmptyAccordionOpen: false, + isMetaAccordionOpen: false, }); - const [pageSize, setPageSize] = useState(PAGINATION_SIZE); - const [scrollContainer, setScrollContainer] = useState(undefined); const currentIndexPattern = indexPatterns[currentIndexPatternId]; const allFields = currentIndexPattern.fields; const clearLocalState = () => setLocalState((s) => ({ ...s, nameFilter: '', typeFilter: [] })); @@ -272,17 +263,11 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ (type) => type in fieldTypeNames ); - useEffect(() => { - // Reset the scroll if we have made material changes to the field list - if (scrollContainer) { - scrollContainer.scrollTop = 0; - setPageSize(PAGINATION_SIZE); - } - }, [localState.nameFilter, localState.typeFilter, currentIndexPatternId, scrollContainer]); + const fieldInfoUnavailable = existenceFetchFailed || currentIndexPattern.hasRestrictions; - const fieldGroups: FieldsGroup = useMemo(() => { + const unfilteredFieldGroups: FieldGroups = useMemo(() => { + const fieldByName = keyBy(allFields, 'name'); const containsData = (field: IndexPatternField) => { - const fieldByName = keyBy(allFields, 'name'); const overallField = fieldByName[field.name]; return ( @@ -294,32 +279,105 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ supportedFieldTypes.has(field.type) ); const sorted = allSupportedTypesFields.sort(sortFields); + let groupedFields; // optimization before existingFields are synced if (!hasSyncedExistingFields) { - return { + groupedFields = { ...defaultFieldGroups, ...groupBy(sorted, (field) => { if (field.type === 'document') { return 'specialFields'; + } else if (field.meta) { + return 'metaFields'; } else { return 'emptyFields'; } }), }; } - return { + groupedFields = { ...defaultFieldGroups, ...groupBy(sorted, (field) => { if (field.type === 'document') { return 'specialFields'; + } else if (field.meta) { + return 'metaFields'; } else if (containsData(field)) { return 'availableFields'; } else return 'emptyFields'; }), }; - }, [allFields, existingFields, currentIndexPattern, hasSyncedExistingFields]); - const filteredFieldGroups: FieldsGroup = useMemo(() => { + const fieldGroupDefinitions: FieldGroups = { + SpecialFields: { + fields: groupedFields.specialFields, + fieldCount: 1, + isAffectedByGlobalFilter: false, + isAffectedByTimeFilter: false, + isInitiallyOpen: false, + showInAccordion: false, + title: '', + hideDetails: true, + }, + AvailableFields: { + fields: groupedFields.availableFields, + fieldCount: groupedFields.availableFields.length, + isInitiallyOpen: true, + showInAccordion: true, + title: fieldInfoUnavailable + ? i18n.translate('xpack.lens.indexPattern.allFieldsLabel', { + defaultMessage: 'All fields', + }) + : i18n.translate('xpack.lens.indexPattern.availableFieldsLabel', { + defaultMessage: 'Available fields', + }), + + isAffectedByGlobalFilter: !!filters.length, + isAffectedByTimeFilter: true, + hideDetails: fieldInfoUnavailable, + }, + EmptyFields: { + fields: groupedFields.emptyFields, + fieldCount: groupedFields.emptyFields.length, + isAffectedByGlobalFilter: false, + isAffectedByTimeFilter: false, + isInitiallyOpen: false, + showInAccordion: true, + hideDetails: false, + title: i18n.translate('xpack.lens.indexPattern.emptyFieldsLabel', { + defaultMessage: 'Empty fields', + }), + }, + MetaFields: { + fields: groupedFields.metaFields, + fieldCount: groupedFields.metaFields.length, + isAffectedByGlobalFilter: false, + isAffectedByTimeFilter: false, + isInitiallyOpen: false, + showInAccordion: true, + hideDetails: false, + title: i18n.translate('xpack.lens.indexPattern.metaFieldsLabel', { + defaultMessage: 'Meta fields', + }), + }, + }; + + // do not show empty field accordion if there is no existence information + if (fieldInfoUnavailable) { + delete fieldGroupDefinitions.EmptyFields; + } + + return fieldGroupDefinitions; + }, [ + allFields, + existingFields, + currentIndexPattern, + hasSyncedExistingFields, + fieldInfoUnavailable, + filters.length, + ]); + + const fieldGroups: FieldGroups = useMemo(() => { const filterFieldGroup = (fieldGroup: IndexPatternField[]) => fieldGroup.filter((field) => { if ( @@ -329,76 +387,18 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ ) { return false; } - if (localState.typeFilter.length > 0) { return localState.typeFilter.includes(field.type as DataType); } return true; }); - - return Object.entries(fieldGroups).reduce((acc, [name, fields]) => { - return { - ...acc, - [name]: filterFieldGroup(fields), - }; - }, defaultFieldGroups); - }, [fieldGroups, localState.nameFilter, localState.typeFilter]); - - const lazyScroll = useCallback(() => { - if (scrollContainer) { - const nearBottom = - scrollContainer.scrollTop + scrollContainer.clientHeight > - scrollContainer.scrollHeight * 0.9; - if (nearBottom) { - const displayedFieldsLength = - (localState.isAvailableAccordionOpen ? filteredFieldGroups.availableFields.length : 0) + - (localState.isEmptyAccordionOpen ? filteredFieldGroups.emptyFields.length : 0); - setPageSize( - Math.max( - PAGINATION_SIZE, - Math.min(pageSize + PAGINATION_SIZE * 0.5, displayedFieldsLength) - ) - ); - } - } - }, [ - scrollContainer, - localState.isAvailableAccordionOpen, - localState.isEmptyAccordionOpen, - filteredFieldGroups, - pageSize, - setPageSize, - ]); - - const [paginatedAvailableFields, paginatedEmptyFields]: [ - IndexPatternField[], - IndexPatternField[] - ] = useMemo(() => { - const { availableFields, emptyFields } = filteredFieldGroups; - const isAvailableAccordionOpen = localState.isAvailableAccordionOpen; - const isEmptyAccordionOpen = localState.isEmptyAccordionOpen; - - if (isAvailableAccordionOpen && isEmptyAccordionOpen) { - if (availableFields.length > pageSize) { - return [availableFields.slice(0, pageSize), []]; - } else { - return [availableFields, emptyFields.slice(0, pageSize - availableFields.length)]; - } - } - if (isAvailableAccordionOpen && !isEmptyAccordionOpen) { - return [availableFields.slice(0, pageSize), []]; - } - - if (!isAvailableAccordionOpen && isEmptyAccordionOpen) { - return [[], emptyFields.slice(0, pageSize)]; - } - return [[], []]; - }, [ - localState.isAvailableAccordionOpen, - localState.isEmptyAccordionOpen, - filteredFieldGroups, - pageSize, - ]); + return Object.fromEntries( + Object.entries(unfilteredFieldGroups).map(([name, group]) => [ + name, + { ...group, fields: filterFieldGroup(group.fields) }, + ]) + ); + }, [unfilteredFieldGroups, localState.nameFilter, localState.typeFilter]); const fieldProps = useMemo( () => ({ @@ -423,8 +423,6 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ ] ); - const fieldInfoUnavailable = existenceFetchFailed || currentIndexPattern.hasRestrictions; - return ( } > - ( @@ -545,115 +543,21 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ -
{ - if (el && !el.dataset.dynamicScroll) { - el.dataset.dynamicScroll = 'true'; - setScrollContainer(el); - } + + field.type === 'document' || + fieldExists(existingFields, currentIndexPattern.title, field.name) + } + fieldProps={fieldProps} + fieldGroups={fieldGroups} + hasSyncedExistingFields={!!hasSyncedExistingFields} + filter={{ + nameFilter: localState.nameFilter, + typeFilter: localState.typeFilter, }} - onScroll={throttle(lazyScroll, 100)} - > -
- {filteredFieldGroups.specialFields.map((field: IndexPatternField) => ( - - ))} - - - { - setLocalState((s) => ({ - ...s, - isAvailableAccordionOpen: open, - })); - const displayedFieldLength = - (open ? filteredFieldGroups.availableFields.length : 0) + - (localState.isEmptyAccordionOpen ? filteredFieldGroups.emptyFields.length : 0); - setPageSize( - Math.max(PAGINATION_SIZE, Math.min(pageSize * 1.5, displayedFieldLength)) - ); - }} - showExistenceFetchError={existenceFetchFailed} - renderCallout={ - - } - /> - - {!fieldInfoUnavailable && ( - { - setLocalState((s) => ({ - ...s, - isEmptyAccordionOpen: open, - })); - const displayedFieldLength = - (localState.isAvailableAccordionOpen - ? filteredFieldGroups.availableFields.length - : 0) + (open ? filteredFieldGroups.emptyFields.length : 0); - setPageSize( - Math.max(PAGINATION_SIZE, Math.min(pageSize * 1.5, displayedFieldLength)) - ); - }} - renderCallout={ - - } - /> - )} - -
-
+ currentIndexPatternId={currentIndexPatternId} + existenceFetchFailed={existenceFetchFailed} + />
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx index 60f60d7cb80c..e71a85868b85 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx @@ -116,7 +116,8 @@ export function FieldSelect({ })); } - const [availableFields, emptyFields] = _.partition(normalFields, containsData); + const [metaFields, nonMetaFields] = _.partition(normalFields, (field) => fieldMap[field].meta); + const [availableFields, emptyFields] = _.partition(nonMetaFields, containsData); const constructFieldsOptions = (fieldsArr: string[], label: string) => fieldsArr.length > 0 && { @@ -138,10 +139,18 @@ export function FieldSelect({ }) ); + const metaFieldsOptions = constructFieldsOptions( + metaFields, + i18n.translate('xpack.lens.indexPattern.metaFieldsLabel', { + defaultMessage: 'Meta fields', + }) + ); + return [ ...fieldNamesToOptions(specialFields), availableFieldsOptions, emptyFieldsOptions, + metaFieldsOptions, ].filter(Boolean); }, [ incompatibleSelectedOperationType, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index 1f6d7911b3a3..1eeb64127310 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -184,7 +184,8 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { defaultMessage: 'Click for a field preview, or drag and drop to visualize.', }) : i18n.translate('xpack.lens.indexPattern.fieldStatsButtonEmptyLabel', { - defaultMessage: "This field doesn't have data. Drag and drop to visualize.", + defaultMessage: + 'This field doesn’t have any data but you can still drag and drop to visualize.', }) } type="iInCircle" @@ -307,7 +308,7 @@ function FieldItemPopoverContents(props: State & FieldItemProps) { {i18n.translate('xpack.lens.indexPattern.fieldStatsNoData', { defaultMessage: - 'This field is empty because it doesn’t exist in the 500 sampled documents.', + 'This field is empty because it doesn’t exist in the 500 sampled documents. Adding this field to the configuration may result in a blank chart.', })} ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_list.scss b/x-pack/plugins/lens/public/indexpattern_datasource/field_list.scss new file mode 100644 index 000000000000..f28581b835b0 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_list.scss @@ -0,0 +1,20 @@ +/** + * 1. Don't cut off the shadow of the field items + */ + +.lnsIndexPatternFieldList { + @include euiOverflowShadow; + @include euiScrollBar; + margin-left: -$euiSize; /* 1 */ + position: relative; + flex-grow: 1; + overflow: auto; +} + +.lnsIndexPatternFieldList__accordionContainer { + padding-top: $euiSizeS; + position: absolute; + top: 0; + left: $euiSize; /* 1 */ + right: $euiSizeXS; /* 1 */ +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx new file mode 100644 index 000000000000..4a9b3a0c63e3 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import './field_list.scss'; +import { throttle } from 'lodash'; +import React, { useState, Fragment, useCallback, useMemo, useEffect } from 'react'; +import { EuiSpacer } from '@elastic/eui'; +import { FieldItem } from './field_item'; +import { NoFieldsCallout } from './no_fields_callout'; +import { IndexPatternField } from './types'; +import { FieldItemSharedProps, FieldsAccordion } from './fields_accordion'; +const PAGINATION_SIZE = 50; + +export interface FieldsGroup { + specialFields: IndexPatternField[]; + availableFields: IndexPatternField[]; + emptyFields: IndexPatternField[]; + metaFields: IndexPatternField[]; +} + +export type FieldGroups = Record< + string, + { + fields: IndexPatternField[]; + fieldCount: number; + showInAccordion: boolean; + isInitiallyOpen: boolean; + title: string; + isAffectedByGlobalFilter: boolean; + isAffectedByTimeFilter: boolean; + hideDetails?: boolean; + } +>; + +function getDisplayedFieldsLength( + fieldGroups: FieldGroups, + accordionState: Partial> +) { + return Object.entries(fieldGroups) + .filter(([key]) => accordionState[key]) + .reduce((allFieldCount, [, { fields }]) => allFieldCount + fields.length, 0); +} + +export function FieldList({ + exists, + fieldGroups, + existenceFetchFailed, + fieldProps, + hasSyncedExistingFields, + filter, + currentIndexPatternId, +}: { + exists: (field: IndexPatternField) => boolean; + fieldGroups: FieldGroups; + fieldProps: FieldItemSharedProps; + hasSyncedExistingFields: boolean; + existenceFetchFailed?: boolean; + filter: { + nameFilter: string; + typeFilter: string[]; + }; + currentIndexPatternId: string; +}) { + const [pageSize, setPageSize] = useState(PAGINATION_SIZE); + const [scrollContainer, setScrollContainer] = useState(undefined); + const [accordionState, setAccordionState] = useState>>(() => + Object.fromEntries( + Object.entries(fieldGroups) + .filter(([, { showInAccordion }]) => showInAccordion) + .map(([key, { isInitiallyOpen }]) => [key, isInitiallyOpen]) + ) + ); + + const isAffectedByFieldFilter = !!(filter.typeFilter.length || filter.nameFilter.length); + + useEffect(() => { + // Reset the scroll if we have made material changes to the field list + if (scrollContainer) { + scrollContainer.scrollTop = 0; + setPageSize(PAGINATION_SIZE); + } + }, [filter.nameFilter, filter.typeFilter, currentIndexPatternId, scrollContainer]); + + const lazyScroll = useCallback(() => { + if (scrollContainer) { + const nearBottom = + scrollContainer.scrollTop + scrollContainer.clientHeight > + scrollContainer.scrollHeight * 0.9; + if (nearBottom) { + setPageSize( + Math.max( + PAGINATION_SIZE, + Math.min( + pageSize + PAGINATION_SIZE * 0.5, + getDisplayedFieldsLength(fieldGroups, accordionState) + ) + ) + ); + } + } + }, [scrollContainer, pageSize, setPageSize, fieldGroups, accordionState]); + + const paginatedFields = useMemo(() => { + let remainingItems = pageSize; + return Object.fromEntries( + Object.entries(fieldGroups) + .filter(([, { showInAccordion }]) => showInAccordion) + .map(([key, fieldGroup]) => { + if (!accordionState[key] || remainingItems <= 0) { + return [key, []]; + } + const slicedFieldList = fieldGroup.fields.slice(0, remainingItems); + remainingItems = remainingItems - slicedFieldList.length; + return [key, slicedFieldList]; + }) + ); + }, [pageSize, fieldGroups, accordionState]); + + return ( +
{ + if (el && !el.dataset.dynamicScroll) { + el.dataset.dynamicScroll = 'true'; + setScrollContainer(el); + } + }} + onScroll={throttle(lazyScroll, 100)} + > +
+ {Object.entries(fieldGroups) + .filter(([, { showInAccordion }]) => !showInAccordion) + .flatMap(([, { fields }]) => + fields.map((field) => ( + + )) + )} + + {Object.entries(fieldGroups) + .filter(([, { showInAccordion }]) => showInAccordion) + .map(([key, fieldGroup]) => ( + + { + setAccordionState((s) => ({ + ...s, + [key]: open, + })); + const displayedFieldLength = getDisplayedFieldsLength(fieldGroups, { + ...accordionState, + [key]: open, + }); + setPageSize( + Math.max(PAGINATION_SIZE, Math.min(pageSize * 1.5, displayedFieldLength)) + ); + }} + showExistenceFetchError={existenceFetchFailed} + renderCallout={ + + } + /> + + + ))} +
+
+ ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx index b0604efff7b8..7d1c80e5a7f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx @@ -71,11 +71,19 @@ describe('Fields Accordion', () => { paginatedFields: indexPattern.fields, fieldProps, renderCallout:
Callout
, - exists: true, + exists: () => true, }; }); it('renders correct number of Field Items', () => { + const wrapper = mountWithIntl( + field.name === 'timestamp'} /> + ); + expect(wrapper.find(FieldItem).at(0).prop('exists')).toEqual(true); + expect(wrapper.find(FieldItem).at(1).prop('exists')).toEqual(false); + }); + + it('passed correct exists flag to each field', () => { const wrapper = mountWithIntl(); expect(wrapper.find(FieldItem).length).toEqual(2); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx index 30a92c21ff66..e531eb72f94c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx @@ -45,7 +45,7 @@ export interface FieldsAccordionProps { paginatedFields: IndexPatternField[]; fieldProps: FieldItemSharedProps; renderCallout: JSX.Element; - exists: boolean; + exists: (field: IndexPatternField) => boolean; showExistenceFetchError?: boolean; hideDetails?: boolean; } @@ -71,7 +71,7 @@ export const InnerFieldsAccordion = function InnerFieldsAccordion({ {...fieldProps} key={field.name} field={field} - exists={exists} + exists={exists(field)} hideDetails={hideDetails} /> ), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts index 19213d4afc9b..ef6abbec9a34 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts @@ -197,7 +197,7 @@ function mockClient() { function mockIndexPatternsService() { return ({ get: jest.fn(async (id: '1' | '2') => { - return sampleIndexPatternsFromService[id]; + return { ...sampleIndexPatternsFromService[id], metaFields: [] }; }), } as unknown) as Pick; } @@ -248,6 +248,7 @@ describe('loader', () => { get: jest.fn(async () => ({ id: 'foo', title: 'Foo index', + metaFields: [], typeMeta: { aggs: { date_histogram: { @@ -295,6 +296,55 @@ describe('loader', () => { date_histogram: { agg: 'date_histogram', fixed_interval: 'm' }, }); }); + + it('should map meta flag', async () => { + const cache = await loadIndexPatterns({ + cache: {}, + patterns: ['foo'], + indexPatternsService: ({ + get: jest.fn(async () => ({ + id: 'foo', + title: 'Foo index', + metaFields: ['timestamp'], + typeMeta: { + aggs: { + date_histogram: { + timestamp: { + agg: 'date_histogram', + fixed_interval: 'm', + }, + }, + sum: { + bytes: { + agg: 'sum', + }, + }, + }, + }, + fields: [ + { + name: 'timestamp', + displayName: 'timestampLabel', + type: 'date', + aggregatable: true, + searchable: true, + }, + { + name: 'bytes', + displayName: 'bytes', + type: 'number', + aggregatable: true, + searchable: true, + }, + ], + })), + } as unknown) as Pick, + }); + + expect(cache.foo.fields.find((f: IndexPatternField) => f.name === 'timestamp')!.meta).toEqual( + true + ); + }); }); describe('loadInitialState', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts index 0ab658b96133..c4b1eb9e0c4c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts @@ -63,6 +63,7 @@ export async function loadIndexPatterns({ type: field.type, aggregatable: field.aggregatable, searchable: field.searchable, + meta: indexPattern.metaFields.includes(field.name), esTypes: field.esTypes, scripted: field.scripted, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index b691c5b5c4c4..a3c0e8aed742 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -26,6 +26,7 @@ export interface IndexPattern { export type IndexPatternField = IFieldType & { displayName: string; aggregationRestrictions?: Partial; + meta?: boolean; }; export interface IndexPatternLayer { diff --git a/x-pack/plugins/lens/server/routes/existing_fields.test.ts b/x-pack/plugins/lens/server/routes/existing_fields.test.ts index 728b78c8e97b..9799dcf92ae4 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.test.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.test.ts @@ -15,6 +15,7 @@ describe('existingFields', () => { name, isScript: false, isAlias: false, + isMeta: false, path: name.split('.'), ...obj, }; @@ -101,6 +102,15 @@ describe('existingFields', () => { expect(result).toEqual(['baz']); }); + + it('supports meta fields', () => { + const result = existingFields( + [{ _mymeta: 'abc', ...indexPattern({}, { bar: 'scriptvalue' }) }], + [field({ name: '_mymeta', isMeta: true, path: ['_mymeta'] })] + ); + + expect(result).toEqual(['_mymeta']); + }); }); describe('buildFieldList', () => { @@ -116,6 +126,7 @@ describe('buildFieldList', () => { { name: 'bar' }, { name: '@bar' }, { name: 'baz' }, + { name: '_mymeta' }, ]), }, references: [], @@ -142,7 +153,7 @@ describe('buildFieldList', () => { ]; it('uses field descriptors to determine the path', () => { - const fields = buildFieldList(indexPattern, mappings, fieldDescriptors); + const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, []); expect(fields.find((f) => f.name === 'baz')).toMatchObject({ isAlias: false, isScript: false, @@ -152,7 +163,7 @@ describe('buildFieldList', () => { }); it('uses aliases to determine the path', () => { - const fields = buildFieldList(indexPattern, mappings, fieldDescriptors); + const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, []); expect(fields.find((f) => f.isAlias)).toMatchObject({ isAlias: true, isScript: false, @@ -162,7 +173,7 @@ describe('buildFieldList', () => { }); it('supports scripted fields', () => { - const fields = buildFieldList(indexPattern, mappings, fieldDescriptors); + const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, []); expect(fields.find((f) => f.isScript)).toMatchObject({ isAlias: false, isScript: true, @@ -173,13 +184,24 @@ describe('buildFieldList', () => { }); }); + it('supports meta fields', () => { + const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, ['_mymeta']); + expect(fields.find((f) => f.isMeta)).toMatchObject({ + isAlias: false, + isScript: false, + isMeta: true, + name: '_mymeta', + path: ['_mymeta'], + }); + }); + it('handles missing mappings', () => { - const fields = buildFieldList(indexPattern, {}, fieldDescriptors); + const fields = buildFieldList(indexPattern, {}, fieldDescriptors, []); expect(fields.every((f) => f.isAlias === false)).toEqual(true); }); it('handles empty fieldDescriptors by skipping multi-mappings', () => { - const fields = buildFieldList(indexPattern, mappings, []); + const fields = buildFieldList(indexPattern, mappings, [], []); expect(fields.find((f) => f.name === 'baz')).toMatchObject({ isAlias: false, isScript: false, diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index 7ab3cdceb214..33fcafacfad7 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -12,6 +12,7 @@ import { BASE_API_URL } from '../../common'; import { IndexPatternsFetcher, IndexPatternAttributes, + UI_SETTINGS, } from '../../../../../src/plugins/data/server'; /** @@ -36,13 +37,12 @@ export interface Field { name: string; isScript: boolean; isAlias: boolean; + isMeta: boolean; path: string[]; lang?: string; script?: string; } -const metaFields = ['_source', '_type']; - export async function existingFieldsRoute(setup: CoreSetup) { const router = setup.http.createRouter(); @@ -104,14 +104,15 @@ async function fetchFieldExistence({ toDate?: string; timeFieldName?: string; }) { + const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS); const { indexPattern, indexPatternTitle, mappings, fieldDescriptors, - } = await fetchIndexPatternDefinition(indexPatternId, context); + } = await fetchIndexPatternDefinition(indexPatternId, context, metaFields); - const fields = buildFieldList(indexPattern, mappings, fieldDescriptors); + const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, metaFields); const docs = await fetchIndexPatternStats({ fromDate, toDate, @@ -128,7 +129,11 @@ async function fetchFieldExistence({ }; } -async function fetchIndexPatternDefinition(indexPatternId: string, context: RequestHandlerContext) { +async function fetchIndexPatternDefinition( + indexPatternId: string, + context: RequestHandlerContext, + metaFields: string[] +) { const savedObjectsClient = context.core.savedObjects.client; const requestClient = context.core.elasticsearch.legacy.client; const indexPattern = await savedObjectsClient.get( @@ -178,7 +183,8 @@ async function fetchIndexPatternDefinition(indexPatternId: string, context: Requ export function buildFieldList( indexPattern: SavedObject, mappings: MappingResult | {}, - fieldDescriptors: FieldDescriptor[] + fieldDescriptors: FieldDescriptor[], + metaFields: string[] ): Field[] { const aliasMap = Object.entries(Object.values(mappings)[0]?.mappings.properties ?? {}) .map(([name, v]) => ({ ...v, name })) @@ -204,6 +210,9 @@ export function buildFieldList( path: path.split('.'), lang: field.lang, script: field.script, + // id is a special case - it doesn't show up in the meta field list, + // but as it's not part of source, it has to be handled separately. + isMeta: metaFields.includes(field.name) || field.name === '_id', }; } ); @@ -312,7 +321,7 @@ function exists(obj: unknown, path: string[], i = 0): boolean { * Exported only for unit tests. */ export function existingFields( - docs: Array<{ _source: unknown; fields: unknown }>, + docs: Array<{ _source: unknown; fields: unknown; [key: string]: unknown }>, fields: Field[] ): string[] { const missingFields = new Set(fields); @@ -323,7 +332,14 @@ export function existingFields( } missingFields.forEach((field) => { - if (exists(field.isScript ? doc.fields : doc._source, field.path)) { + let fieldStore = doc._source; + if (field.isScript) { + fieldStore = doc.fields; + } + if (field.isMeta) { + fieldStore = doc; + } + if (exists(fieldStore, field.path)) { missingFields.delete(field); } }); diff --git a/x-pack/test/api_integration/apis/lens/existing_fields.ts b/x-pack/test/api_integration/apis/lens/existing_fields.ts index 92336f2892f4..10ee7bc9b48e 100644 --- a/x-pack/test/api_integration/apis/lens/existing_fields.ts +++ b/x-pack/test/api_integration/apis/lens/existing_fields.ts @@ -20,6 +20,9 @@ const fieldsWithData = [ '@tags', '@tags.raw', '@timestamp', + '_id', + '_index', + '_source', 'agent', 'agent.raw', 'bytes', @@ -96,6 +99,9 @@ const fieldsWithData = [ const metricBeatData = [ '@timestamp', + '_id', + '_index', + '_source', 'agent.ephemeral_id', 'agent.hostname', 'agent.id', @@ -185,6 +191,9 @@ export default ({ getService }: FtrProviderContext) => { '@tags', '@tags.raw', '@timestamp', + '_id', + '_index', + '_source', 'agent', 'agent.raw', 'bytes',