maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Lens] show meta field data in Lens (#77210)

Browse source
This commit is contained in:
Joe Reuter 2020-09-24 09:55:32 +02:00 committed by GitHub
parent 62ddaa9e20
commit 839817ca9a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 493 additions and 251 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
x-pack/plugins/lens/public/indexpattern_datasource/datapanel.scss
View file

@ -10,27 +10,6 @@
margin-bottom: $euiSizeS;
}
/**
* 1. Don't cut off the shadow of the field items
*/
.lnsInnerIndexPatternDataPanel__listWrapper {
@include euiOverflowShadow;
@include euiScrollBar;
margin-left: -$euiSize; /* 1 */
position: relative;
flex-grow: 1;
overflow: auto;
}
.lnsInnerIndexPatternDataPanel__list {
padding-top: $euiSizeS;
position: absolute;
top: 0;
left: $euiSize; /* 1 */
right: $euiSizeXS; /* 1 */
}
.lnsInnerIndexPatternDataPanel__fieldItems {
// Quick fix for making sure the shadow and focus rings are visible outside the accordion bounds
padding: $euiSizeXS $euiSizeXS 0;

35
x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
View file

@ -623,11 +623,40 @@ describe('IndexPattern Data Panel', () => {
).toEqual(['client', 'source', 'timestampLabel']);
});
it('should show meta fields accordion', async () => {
const wrapper = mountWithIntl(
<InnerIndexPatternDataPanel
{...props}
indexPatterns={{
'1': {
...props.indexPatterns['1'],
fields: [
...props.indexPatterns['1'].fields,
{ name: '_id', displayName: '_id', meta: true, type: 'string' },
],
},
}}
/>
);
wrapper
.find('[data-test-subj="lnsIndexPatternMetaFields"]')
.find('button')
.first()
.simulate('click');
expect(
wrapper
.find('[data-test-subj="lnsIndexPatternMetaFields"]')
.find(FieldItem)
.first()
.prop('field').name
).toEqual('_id');
});
it('should display NoFieldsCallout when all fields are empty', async () => {
const wrapper = mountWithIntl(
<InnerIndexPatternDataPanel {...defaultProps} existingFields={{ idx1: {} }} />
);
expect(wrapper.find(NoFieldsCallout).length).toEqual(1);
expect(wrapper.find(NoFieldsCallout).length).toEqual(2);
expect(
wrapper
.find('[data-test-subj="lnsIndexPatternAvailableFields"]')
@ -654,7 +683,7 @@ describe('IndexPattern Data Panel', () => {
.length
).toEqual(1);
wrapper.setProps({ existingFields: { idx1: {} } });
expect(wrapper.find(NoFieldsCallout).length).toEqual(1);
expect(wrapper.find(NoFieldsCallout).length).toEqual(2);
});
it('should filter down by name', () => {
@ -699,7 +728,7 @@ describe('IndexPattern Data Panel', () => {
expect(wrapper.find(FieldItem).map((fieldItem) => fieldItem.prop('field').name)).toEqual([
'Records',
]);
expect(wrapper.find(NoFieldsCallout).length).toEqual(2);
expect(wrapper.find(NoFieldsCallout).length).toEqual(3);
});
it('should toggle type if clicked again', () => {

318
x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx
View file

@ -5,14 +5,13 @@
*/
import './datapanel.scss';
import { uniq, keyBy, groupBy, throttle } from 'lodash';
import React, { useState, useEffect, memo, useCallback, useMemo } from 'react';
import { uniq, keyBy, groupBy } from 'lodash';
import React, { useState, memo, useCallback, useMemo } from 'react';
import {
EuiFlexGroup,
EuiFlexItem,
EuiContextMenuPanel,
EuiContextMenuItem,
EuiContextMenuPanelProps,
EuiPopover,
EuiCallOut,
EuiFormControlLayout,
@ -25,8 +24,6 @@ import { FormattedMessage } from '@kbn/i18n/react';
import { DataPublicPluginStart, EsQueryConfig, Query, Filter } from 'src/plugins/data/public';
import { DatasourceDataPanelProps, DataType, StateSetter } from '../types';
import { ChildDragDropProvider, DragContextState } from '../drag_drop';
import { FieldItem } from './field_item';
import { NoFieldsCallout } from './no_fields_callout';
import {
IndexPattern,
IndexPatternPrivateState,
@ -37,7 +34,6 @@ import { trackUiEvent } from '../lens_ui_telemetry';
import { syncExistingFields } from './loader';
import { fieldExists } from './pure_helpers';
import { Loader } from '../loader';
import { FieldsAccordion } from './fields_accordion';
import { esQuery, IIndexPattern } from '../../../../../src/plugins/data/public';
export type Props = DatasourceDataPanelProps<IndexPatternPrivateState> & {
@ -52,18 +48,13 @@ export type Props = DatasourceDataPanelProps<IndexPatternPrivateState> & {
import { LensFieldIcon } from './lens_field_icon';
import { ChangeIndexPattern } from './change_indexpattern';
import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public';
// TODO the typings for EuiContextMenuPanel are incorrect - watchedItemProps is missing. This can be removed when the types are adjusted
const FixedEuiContextMenuPanel = (EuiContextMenuPanel as unknown) as React.FunctionComponent<
EuiContextMenuPanelProps & { watchedItemProps: string[] }
>;
import { FieldGroups, FieldList } from './field_list';
function sortFields(fieldA: IndexPatternField, fieldB: IndexPatternField) {
return fieldA.displayName.localeCompare(fieldB.displayName, undefined, { sensitivity: 'base' });
}
const supportedFieldTypes = new Set(['string', 'number', 'boolean', 'date', 'ip', 'document']);
const PAGINATION_SIZE = 50;
const fieldTypeNames: Record<DataType, string> = {
document: i18n.translate('xpack.lens.datatypes.record', { defaultMessage: 'record' }),
@ -212,18 +203,19 @@ interface DataPanelState {
isTypeFilterOpen: boolean;
isAvailableAccordionOpen: boolean;
isEmptyAccordionOpen: boolean;
isMetaAccordionOpen: boolean;
}
export interface FieldsGroup {
const defaultFieldGroups: {
specialFields: IndexPatternField[];
availableFields: IndexPatternField[];
emptyFields: IndexPatternField[];
}
const defaultFieldGroups = {
metaFields: IndexPatternField[];
} = {
specialFields: [],
availableFields: [],
emptyFields: [],
metaFields: [],
};
const fieldFiltersLabel = i18n.translate('xpack.lens.indexPatterns.fieldFiltersLabel', {
@ -261,9 +253,8 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
isTypeFilterOpen: false,
isAvailableAccordionOpen: true,
isEmptyAccordionOpen: false,
isMetaAccordionOpen: false,
});
const [pageSize, setPageSize] = useState(PAGINATION_SIZE);
const [scrollContainer, setScrollContainer] = useState<Element | undefined>(undefined);
const currentIndexPattern = indexPatterns[currentIndexPatternId];
const allFields = currentIndexPattern.fields;
const clearLocalState = () => setLocalState((s) => ({ ...s, nameFilter: '', typeFilter: [] }));
@ -272,17 +263,11 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
(type) => type in fieldTypeNames
);
useEffect(() => {
// Reset the scroll if we have made material changes to the field list
if (scrollContainer) {
scrollContainer.scrollTop = 0;
setPageSize(PAGINATION_SIZE);
}
}, [localState.nameFilter, localState.typeFilter, currentIndexPatternId, scrollContainer]);
const fieldInfoUnavailable = existenceFetchFailed || currentIndexPattern.hasRestrictions;
const fieldGroups: FieldsGroup = useMemo(() => {
const unfilteredFieldGroups: FieldGroups = useMemo(() => {
const fieldByName = keyBy(allFields, 'name');
const containsData = (field: IndexPatternField) => {
const fieldByName = keyBy(allFields, 'name');
const overallField = fieldByName[field.name];
return (
@ -294,32 +279,105 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
supportedFieldTypes.has(field.type)
);
const sorted = allSupportedTypesFields.sort(sortFields);
let groupedFields;
// optimization before existingFields are synced
if (!hasSyncedExistingFields) {
return {
groupedFields = {
...defaultFieldGroups,
...groupBy(sorted, (field) => {
if (field.type === 'document') {
return 'specialFields';
} else if (field.meta) {
return 'metaFields';
} else {
return 'emptyFields';
}
}),
};
}
return {
groupedFields = {
...defaultFieldGroups,
...groupBy(sorted, (field) => {
if (field.type === 'document') {
return 'specialFields';
} else if (field.meta) {
return 'metaFields';
} else if (containsData(field)) {
return 'availableFields';
} else return 'emptyFields';
}),
};
}, [allFields, existingFields, currentIndexPattern, hasSyncedExistingFields]);
const filteredFieldGroups: FieldsGroup = useMemo(() => {
const fieldGroupDefinitions: FieldGroups = {
SpecialFields: {
fields: groupedFields.specialFields,
fieldCount: 1,
isAffectedByGlobalFilter: false,
isAffectedByTimeFilter: false,
isInitiallyOpen: false,
showInAccordion: false,
title: '',
hideDetails: true,
},
AvailableFields: {
fields: groupedFields.availableFields,
fieldCount: groupedFields.availableFields.length,
isInitiallyOpen: true,
showInAccordion: true,
title: fieldInfoUnavailable
? i18n.translate('xpack.lens.indexPattern.allFieldsLabel', {
defaultMessage: 'All fields',
})
: i18n.translate('xpack.lens.indexPattern.availableFieldsLabel', {
defaultMessage: 'Available fields',
}),
isAffectedByGlobalFilter: !!filters.length,
isAffectedByTimeFilter: true,
hideDetails: fieldInfoUnavailable,
},
EmptyFields: {
fields: groupedFields.emptyFields,
fieldCount: groupedFields.emptyFields.length,
isAffectedByGlobalFilter: false,
isAffectedByTimeFilter: false,
isInitiallyOpen: false,
showInAccordion: true,
hideDetails: false,
title: i18n.translate('xpack.lens.indexPattern.emptyFieldsLabel', {
defaultMessage: 'Empty fields',
}),
},
MetaFields: {
fields: groupedFields.metaFields,
fieldCount: groupedFields.metaFields.length,
isAffectedByGlobalFilter: false,
isAffectedByTimeFilter: false,
isInitiallyOpen: false,
showInAccordion: true,
hideDetails: false,
title: i18n.translate('xpack.lens.indexPattern.metaFieldsLabel', {
defaultMessage: 'Meta fields',
}),
},
};
// do not show empty field accordion if there is no existence information
if (fieldInfoUnavailable) {
delete fieldGroupDefinitions.EmptyFields;
}
return fieldGroupDefinitions;
}, [
allFields,
existingFields,
currentIndexPattern,
hasSyncedExistingFields,
fieldInfoUnavailable,
filters.length,
]);
const fieldGroups: FieldGroups = useMemo(() => {
const filterFieldGroup = (fieldGroup: IndexPatternField[]) =>
fieldGroup.filter((field) => {
if (
@ -329,76 +387,18 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
) {
return false;
}
if (localState.typeFilter.length > 0) {
return localState.typeFilter.includes(field.type as DataType);
}
return true;
});
return Object.entries(fieldGroups).reduce((acc, [name, fields]) => {
return {
...acc,
[name]: filterFieldGroup(fields),
};
}, defaultFieldGroups);
}, [fieldGroups, localState.nameFilter, localState.typeFilter]);
const lazyScroll = useCallback(() => {
if (scrollContainer) {
const nearBottom =
scrollContainer.scrollTop + scrollContainer.clientHeight >
scrollContainer.scrollHeight * 0.9;
if (nearBottom) {
const displayedFieldsLength =
(localState.isAvailableAccordionOpen ? filteredFieldGroups.availableFields.length : 0) +
(localState.isEmptyAccordionOpen ? filteredFieldGroups.emptyFields.length : 0);
setPageSize(
Math.max(
PAGINATION_SIZE,
Math.min(pageSize + PAGINATION_SIZE * 0.5, displayedFieldsLength)
)
);
}
}
}, [
scrollContainer,
localState.isAvailableAccordionOpen,
localState.isEmptyAccordionOpen,
filteredFieldGroups,
pageSize,
setPageSize,
]);
const [paginatedAvailableFields, paginatedEmptyFields]: [
IndexPatternField[],
IndexPatternField[]
] = useMemo(() => {
const { availableFields, emptyFields } = filteredFieldGroups;
const isAvailableAccordionOpen = localState.isAvailableAccordionOpen;
const isEmptyAccordionOpen = localState.isEmptyAccordionOpen;
if (isAvailableAccordionOpen && isEmptyAccordionOpen) {
if (availableFields.length > pageSize) {
return [availableFields.slice(0, pageSize), []];
} else {
return [availableFields, emptyFields.slice(0, pageSize - availableFields.length)];
}
}
if (isAvailableAccordionOpen && !isEmptyAccordionOpen) {
return [availableFields.slice(0, pageSize), []];
}
if (!isAvailableAccordionOpen && isEmptyAccordionOpen) {
return [[], emptyFields.slice(0, pageSize)];
}
return [[], []];
}, [
localState.isAvailableAccordionOpen,
localState.isEmptyAccordionOpen,
filteredFieldGroups,
pageSize,
]);
return Object.fromEntries(
Object.entries(unfilteredFieldGroups).map(([name, group]) => [
name,
{ ...group, fields: filterFieldGroup(group.fields) },
])
);
}, [unfilteredFieldGroups, localState.nameFilter, localState.typeFilter]);
const fieldProps = useMemo(
() => ({
@ -423,8 +423,6 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
]
);
const fieldInfoUnavailable = existenceFetchFailed || currentIndexPattern.hasRestrictions;
return (
<ChildDragDropProvider {...dragDropContext}>
<EuiFlexGroup
@ -516,7 +514,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
</EuiFilterButton>
}
>
<FixedEuiContextMenuPanel
<EuiContextMenuPanel
watchedItemProps={['icon', 'disabled']}
data-test-subj="lnsIndexPatternTypeFilterOptions"
items={(availableFieldTypes as DataType[]).map((type) => (
@ -545,115 +543,21 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({
</EuiFilterGroup>
</EuiFlexItem>
<EuiFlexItem>
<div
className="lnsInnerIndexPatternDataPanel__listWrapper"
ref={(el) => {
if (el && !el.dataset.dynamicScroll) {
el.dataset.dynamicScroll = 'true';
setScrollContainer(el);
}
<FieldList
exists={(field) =>
field.type === 'document' ||
fieldExists(existingFields, currentIndexPattern.title, field.name)
}
fieldProps={fieldProps}
fieldGroups={fieldGroups}
hasSyncedExistingFields={!!hasSyncedExistingFields}
filter={{
nameFilter: localState.nameFilter,
typeFilter: localState.typeFilter,
}}
onScroll={throttle(lazyScroll, 100)}
>
<div className="lnsInnerIndexPatternDataPanel__list">
{filteredFieldGroups.specialFields.map((field: IndexPatternField) => (
<FieldItem
{...fieldProps}
exists={!!fieldGroups.availableFields.length}
field={field}
hideDetails={true}
key={field.name}
/>
))}
<EuiSpacer size="s" />
<FieldsAccordion
initialIsOpen={localState.isAvailableAccordionOpen}
id="lnsIndexPatternAvailableFields"
label={
fieldInfoUnavailable
? i18n.translate('xpack.lens.indexPattern.allFieldsLabel', {
defaultMessage: 'All fields',
})
: i18n.translate('xpack.lens.indexPattern.availableFieldsLabel', {
defaultMessage: 'Available fields',
})
}
exists={true}
hideDetails={fieldInfoUnavailable}
hasLoaded={!!hasSyncedExistingFields}
fieldsCount={filteredFieldGroups.availableFields.length}
isFiltered={
filteredFieldGroups.availableFields.length !== fieldGroups.availableFields.length
}
paginatedFields={paginatedAvailableFields}
fieldProps={fieldProps}
onToggle={(open) => {
setLocalState((s) => ({
...s,
isAvailableAccordionOpen: open,
}));
const displayedFieldLength =
(open ? filteredFieldGroups.availableFields.length : 0) +
(localState.isEmptyAccordionOpen ? filteredFieldGroups.emptyFields.length : 0);
setPageSize(
Math.max(PAGINATION_SIZE, Math.min(pageSize * 1.5, displayedFieldLength))
);
}}
showExistenceFetchError={existenceFetchFailed}
renderCallout={
<NoFieldsCallout
isAffectedByGlobalFilter={!!filters.length}
isAffectedByFieldFilter={
!!(localState.typeFilter.length || localState.nameFilter.length)
}
isAffectedByTimerange={true}
existFieldsInIndex={!!fieldGroups.emptyFields.length}
/>
}
/>
<EuiSpacer size="m" />
{!fieldInfoUnavailable && (
<FieldsAccordion
initialIsOpen={localState.isEmptyAccordionOpen}
isFiltered={
filteredFieldGroups.emptyFields.length !== fieldGroups.emptyFields.length
}
fieldsCount={filteredFieldGroups.emptyFields.length}
paginatedFields={paginatedEmptyFields}
hasLoaded={!!hasSyncedExistingFields}
exists={false}
fieldProps={fieldProps}
id="lnsIndexPatternEmptyFields"
label={i18n.translate('xpack.lens.indexPattern.emptyFieldsLabel', {
defaultMessage: 'Empty fields',
})}
onToggle={(open) => {
setLocalState((s) => ({
...s,
isEmptyAccordionOpen: open,
}));
const displayedFieldLength =
(localState.isAvailableAccordionOpen
? filteredFieldGroups.availableFields.length
: 0) + (open ? filteredFieldGroups.emptyFields.length : 0);
setPageSize(
Math.max(PAGINATION_SIZE, Math.min(pageSize * 1.5, displayedFieldLength))
);
}}
renderCallout={
<NoFieldsCallout
isAffectedByFieldFilter={
!!(localState.typeFilter.length || localState.nameFilter.length)
}
existFieldsInIndex={!!fieldGroups.emptyFields.length}
/>
}
/>
)}
<EuiSpacer size="m" />
</div>
</div>
currentIndexPatternId={currentIndexPatternId}
existenceFetchFailed={existenceFetchFailed}
/>
</EuiFlexItem>
</EuiFlexGroup>
</ChildDragDropProvider>

11
x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/field_select.tsx
View file

@ -116,7 +116,8 @@ export function FieldSelect({
}));
}
const [availableFields, emptyFields] = _.partition(normalFields, containsData);
const [metaFields, nonMetaFields] = _.partition(normalFields, (field) => fieldMap[field].meta);
const [availableFields, emptyFields] = _.partition(nonMetaFields, containsData);
const constructFieldsOptions = (fieldsArr: string[], label: string) =>
fieldsArr.length > 0 && {
@ -138,10 +139,18 @@ export function FieldSelect({
})
);
const metaFieldsOptions = constructFieldsOptions(
metaFields,
i18n.translate('xpack.lens.indexPattern.metaFieldsLabel', {
defaultMessage: 'Meta fields',
})
);
return [
...fieldNamesToOptions(specialFields),
availableFieldsOptions,
emptyFieldsOptions,
metaFieldsOptions,
].filter(Boolean);
}, [
incompatibleSelectedOperationType,

5
x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx
View file

@ -184,7 +184,8 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) {
defaultMessage: 'Click for a field preview, or drag and drop to visualize.',
})
: i18n.translate('xpack.lens.indexPattern.fieldStatsButtonEmptyLabel', {
defaultMessage: "This field doesn't have data. Drag and drop to visualize.",
defaultMessage:
'This field doesn’t have any data but you can still drag and drop to visualize.',
})
}
type="iInCircle"
@ -307,7 +308,7 @@ function FieldItemPopoverContents(props: State & FieldItemProps) {
<EuiText size="s">
{i18n.translate('xpack.lens.indexPattern.fieldStatsNoData', {
defaultMessage:
'This field is empty because it doesn’t exist in the 500 sampled documents.',
'This field is empty because it doesn’t exist in the 500 sampled documents. Adding this field to the configuration may result in a blank chart.',
})}
</EuiText>
);

20
x-pack/plugins/lens/public/indexpattern_datasource/field_list.scss Normal file
View file

@ -0,0 +1,20 @@
/**
* 1. Don't cut off the shadow of the field items
*/
.lnsIndexPatternFieldList {
@include euiOverflowShadow;
@include euiScrollBar;
margin-left: -$euiSize; /* 1 */
position: relative;
flex-grow: 1;
overflow: auto;
}
.lnsIndexPatternFieldList__accordionContainer {
padding-top: $euiSizeS;
position: absolute;
top: 0;
left: $euiSize; /* 1 */
right: $euiSizeXS; /* 1 */
}

193
x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx Normal file
View file

@ -0,0 +1,193 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import './field_list.scss';
import { throttle } from 'lodash';
import React, { useState, Fragment, useCallback, useMemo, useEffect } from 'react';
import { EuiSpacer } from '@elastic/eui';
import { FieldItem } from './field_item';
import { NoFieldsCallout } from './no_fields_callout';
import { IndexPatternField } from './types';
import { FieldItemSharedProps, FieldsAccordion } from './fields_accordion';
const PAGINATION_SIZE = 50;
export interface FieldsGroup {
specialFields: IndexPatternField[];
availableFields: IndexPatternField[];
emptyFields: IndexPatternField[];
metaFields: IndexPatternField[];
}
export type FieldGroups = Record<
string,
{
fields: IndexPatternField[];
fieldCount: number;
showInAccordion: boolean;
isInitiallyOpen: boolean;
title: string;
isAffectedByGlobalFilter: boolean;
isAffectedByTimeFilter: boolean;
hideDetails?: boolean;
}
>;
function getDisplayedFieldsLength(
fieldGroups: FieldGroups,
accordionState: Partial<Record<string, boolean>>
) {
return Object.entries(fieldGroups)
.filter(([key]) => accordionState[key])
.reduce((allFieldCount, [, { fields }]) => allFieldCount + fields.length, 0);
}
export function FieldList({
exists,
fieldGroups,
existenceFetchFailed,
fieldProps,
hasSyncedExistingFields,
filter,
currentIndexPatternId,
}: {
exists: (field: IndexPatternField) => boolean;
fieldGroups: FieldGroups;
fieldProps: FieldItemSharedProps;
hasSyncedExistingFields: boolean;
existenceFetchFailed?: boolean;
filter: {
nameFilter: string;
typeFilter: string[];
};
currentIndexPatternId: string;
}) {
const [pageSize, setPageSize] = useState(PAGINATION_SIZE);
const [scrollContainer, setScrollContainer] = useState<Element | undefined>(undefined);
const [accordionState, setAccordionState] = useState<Partial<Record<string, boolean>>>(() =>
Object.fromEntries(
Object.entries(fieldGroups)
.filter(([, { showInAccordion }]) => showInAccordion)
.map(([key, { isInitiallyOpen }]) => [key, isInitiallyOpen])
)
);
const isAffectedByFieldFilter = !!(filter.typeFilter.length || filter.nameFilter.length);
useEffect(() => {
// Reset the scroll if we have made material changes to the field list
if (scrollContainer) {
scrollContainer.scrollTop = 0;
setPageSize(PAGINATION_SIZE);
}
}, [filter.nameFilter, filter.typeFilter, currentIndexPatternId, scrollContainer]);
const lazyScroll = useCallback(() => {
if (scrollContainer) {
const nearBottom =
scrollContainer.scrollTop + scrollContainer.clientHeight >
scrollContainer.scrollHeight * 0.9;
if (nearBottom) {
setPageSize(
Math.max(
PAGINATION_SIZE,
Math.min(
pageSize + PAGINATION_SIZE * 0.5,
getDisplayedFieldsLength(fieldGroups, accordionState)
)
)
);
}
}
}, [scrollContainer, pageSize, setPageSize, fieldGroups, accordionState]);
const paginatedFields = useMemo(() => {
let remainingItems = pageSize;
return Object.fromEntries(
Object.entries(fieldGroups)
.filter(([, { showInAccordion }]) => showInAccordion)
.map(([key, fieldGroup]) => {
if (!accordionState[key] || remainingItems <= 0) {
return [key, []];
}
const slicedFieldList = fieldGroup.fields.slice(0, remainingItems);
remainingItems = remainingItems - slicedFieldList.length;
return [key, slicedFieldList];
})
);
}, [pageSize, fieldGroups, accordionState]);
return (
<div
className="lnsIndexPatternFieldList"
ref={(el) => {
if (el && !el.dataset.dynamicScroll) {
el.dataset.dynamicScroll = 'true';
setScrollContainer(el);
}
}}
onScroll={throttle(lazyScroll, 100)}
>
<div className="lnsIndexPatternFieldList__accordionContainer">
{Object.entries(fieldGroups)
.filter(([, { showInAccordion }]) => !showInAccordion)
.flatMap(([, { fields }]) =>
fields.map((field) => (
<FieldItem
{...fieldProps}
exists={exists(field)}
field={field}
hideDetails={true}
key={field.name}
/>
))
)}
<EuiSpacer size="s" />
{Object.entries(fieldGroups)
.filter(([, { showInAccordion }]) => showInAccordion)
.map(([key, fieldGroup]) => (
<Fragment key={key}>
<FieldsAccordion
initialIsOpen={Boolean(accordionState[key])}
key={key}
id={`lnsIndexPattern${key}`}
label={fieldGroup.title}
exists={exists}
hideDetails={fieldGroup.hideDetails}
hasLoaded={!!hasSyncedExistingFields}
fieldsCount={fieldGroup.fields.length}
isFiltered={fieldGroup.fieldCount !== fieldGroup.fields.length}
paginatedFields={paginatedFields[key]}
fieldProps={fieldProps}
onToggle={(open) => {
setAccordionState((s) => ({
...s,
[key]: open,
}));
const displayedFieldLength = getDisplayedFieldsLength(fieldGroups, {
...accordionState,
[key]: open,
});
setPageSize(
Math.max(PAGINATION_SIZE, Math.min(pageSize * 1.5, displayedFieldLength))
);
}}
showExistenceFetchError={existenceFetchFailed}
renderCallout={
<NoFieldsCallout
isAffectedByGlobalFilter={fieldGroup.isAffectedByGlobalFilter}
isAffectedByFieldFilter={isAffectedByFieldFilter}
isAffectedByTimerange={fieldGroup.isAffectedByTimeFilter}
existFieldsInIndex={!!fieldGroup.fieldCount}
/>
}
/>
<EuiSpacer size="m" />
</Fragment>
))}
</div>
</div>
);
}

10
x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx
View file

@ -71,11 +71,19 @@ describe('Fields Accordion', () => {
paginatedFields: indexPattern.fields,
fieldProps,
renderCallout: <div id="lens-test-callout">Callout</div>,
exists: true,
exists: () => true,
};
});
it('renders correct number of Field Items', () => {
const wrapper = mountWithIntl(
<FieldsAccordion {...defaultProps} exists={(field) => field.name === 'timestamp'} />
);
expect(wrapper.find(FieldItem).at(0).prop('exists')).toEqual(true);
expect(wrapper.find(FieldItem).at(1).prop('exists')).toEqual(false);
});
it('passed correct exists flag to each field', () => {
const wrapper = mountWithIntl(<FieldsAccordion {...defaultProps} />);
expect(wrapper.find(FieldItem).length).toEqual(2);
});

4
x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx
View file

@ -45,7 +45,7 @@ export interface FieldsAccordionProps {
paginatedFields: IndexPatternField[];
fieldProps: FieldItemSharedProps;
renderCallout: JSX.Element;
exists: boolean;
exists: (field: IndexPatternField) => boolean;
showExistenceFetchError?: boolean;
hideDetails?: boolean;
}
@ -71,7 +71,7 @@ export const InnerFieldsAccordion = function InnerFieldsAccordion({
{...fieldProps}
key={field.name}
field={field}
exists={exists}
exists={exists(field)}
hideDetails={hideDetails}
/>
),

52
x-pack/plugins/lens/public/indexpattern_datasource/loader.test.ts
View file

@ -197,7 +197,7 @@ function mockClient() {
function mockIndexPatternsService() {
return ({
get: jest.fn(async (id: '1' | '2') => {
return sampleIndexPatternsFromService[id];
return { ...sampleIndexPatternsFromService[id], metaFields: [] };
}),
} as unknown) as Pick<IndexPatternsContract, 'get'>;
}
@ -248,6 +248,7 @@ describe('loader', () => {
get: jest.fn(async () => ({
id: 'foo',
title: 'Foo index',
metaFields: [],
typeMeta: {
aggs: {
date_histogram: {
@ -295,6 +296,55 @@ describe('loader', () => {
date_histogram: { agg: 'date_histogram', fixed_interval: 'm' },
});
});
it('should map meta flag', async () => {
const cache = await loadIndexPatterns({
cache: {},
patterns: ['foo'],
indexPatternsService: ({
get: jest.fn(async () => ({
id: 'foo',
title: 'Foo index',
metaFields: ['timestamp'],
typeMeta: {
aggs: {
date_histogram: {
timestamp: {
agg: 'date_histogram',
fixed_interval: 'm',
},
},
sum: {
bytes: {
agg: 'sum',
},
},
},
},
fields: [
{
name: 'timestamp',
displayName: 'timestampLabel',
type: 'date',
aggregatable: true,
searchable: true,
},
{
name: 'bytes',
displayName: 'bytes',
type: 'number',
aggregatable: true,
searchable: true,
},
],
})),
} as unknown) as Pick<IndexPatternsContract, 'get'>,
});
expect(cache.foo.fields.find((f: IndexPatternField) => f.name === 'timestamp')!.meta).toEqual(
true
);
});
});
describe('loadInitialState', () => {

1
x-pack/plugins/lens/public/indexpattern_datasource/loader.ts
View file

@ -63,6 +63,7 @@ export async function loadIndexPatterns({
type: field.type,
aggregatable: field.aggregatable,
searchable: field.searchable,
meta: indexPattern.metaFields.includes(field.name),
esTypes: field.esTypes,
scripted: field.scripted,
};

1
x-pack/plugins/lens/public/indexpattern_datasource/types.ts
View file

@ -26,6 +26,7 @@ export interface IndexPattern {
export type IndexPatternField = IFieldType & {
displayName: string;
aggregationRestrictions?: Partial<IndexPatternAggRestrictions>;
meta?: boolean;
};
export interface IndexPatternLayer {

32
x-pack/plugins/lens/server/routes/existing_fields.test.ts
View file

@ -15,6 +15,7 @@ describe('existingFields', () => {
name,
isScript: false,
isAlias: false,
isMeta: false,
path: name.split('.'),
...obj,
};
@ -101,6 +102,15 @@ describe('existingFields', () => {
expect(result).toEqual(['baz']);
});
it('supports meta fields', () => {
const result = existingFields(
[{ _mymeta: 'abc', ...indexPattern({}, { bar: 'scriptvalue' }) }],
[field({ name: '_mymeta', isMeta: true, path: ['_mymeta'] })]
);
expect(result).toEqual(['_mymeta']);
});
});
describe('buildFieldList', () => {
@ -116,6 +126,7 @@ describe('buildFieldList', () => {
{ name: 'bar' },
{ name: '@bar' },
{ name: 'baz' },
{ name: '_mymeta' },
]),
},
references: [],
@ -142,7 +153,7 @@ describe('buildFieldList', () => {
];
it('uses field descriptors to determine the path', () => {
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, []);
expect(fields.find((f) => f.name === 'baz')).toMatchObject({
isAlias: false,
isScript: false,
@ -152,7 +163,7 @@ describe('buildFieldList', () => {
});
it('uses aliases to determine the path', () => {
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, []);
expect(fields.find((f) => f.isAlias)).toMatchObject({
isAlias: true,
isScript: false,
@ -162,7 +173,7 @@ describe('buildFieldList', () => {
});
it('supports scripted fields', () => {
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, []);
expect(fields.find((f) => f.isScript)).toMatchObject({
isAlias: false,
isScript: true,
@ -173,13 +184,24 @@ describe('buildFieldList', () => {
});
});
it('supports meta fields', () => {
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, ['_mymeta']);
expect(fields.find((f) => f.isMeta)).toMatchObject({
isAlias: false,
isScript: false,
isMeta: true,
name: '_mymeta',
path: ['_mymeta'],
});
});
it('handles missing mappings', () => {
const fields = buildFieldList(indexPattern, {}, fieldDescriptors);
const fields = buildFieldList(indexPattern, {}, fieldDescriptors, []);
expect(fields.every((f) => f.isAlias === false)).toEqual(true);
});
it('handles empty fieldDescriptors by skipping multi-mappings', () => {
const fields = buildFieldList(indexPattern, mappings, []);
const fields = buildFieldList(indexPattern, mappings, [], []);
expect(fields.find((f) => f.name === 'baz')).toMatchObject({
isAlias: false,
isScript: false,

32
x-pack/plugins/lens/server/routes/existing_fields.ts
View file

@ -12,6 +12,7 @@ import { BASE_API_URL } from '../../common';
import {
IndexPatternsFetcher,
IndexPatternAttributes,
UI_SETTINGS,
} from '../../../../../src/plugins/data/server';
/**
@ -36,13 +37,12 @@ export interface Field {
name: string;
isScript: boolean;
isAlias: boolean;
isMeta: boolean;
path: string[];
lang?: string;
script?: string;
}
const metaFields = ['_source', '_type'];
export async function existingFieldsRoute(setup: CoreSetup) {
const router = setup.http.createRouter();
@ -104,14 +104,15 @@ async function fetchFieldExistence({
toDate?: string;
timeFieldName?: string;
}) {
const metaFields: string[] = await context.core.uiSettings.client.get(UI_SETTINGS.META_FIELDS);
const {
indexPattern,
indexPatternTitle,
mappings,
fieldDescriptors,
} = await fetchIndexPatternDefinition(indexPatternId, context);
} = await fetchIndexPatternDefinition(indexPatternId, context, metaFields);
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
const fields = buildFieldList(indexPattern, mappings, fieldDescriptors, metaFields);
const docs = await fetchIndexPatternStats({
fromDate,
toDate,
@ -128,7 +129,11 @@ async function fetchFieldExistence({
};
}
async function fetchIndexPatternDefinition(indexPatternId: string, context: RequestHandlerContext) {
async function fetchIndexPatternDefinition(
indexPatternId: string,
context: RequestHandlerContext,
metaFields: string[]
) {
const savedObjectsClient = context.core.savedObjects.client;
const requestClient = context.core.elasticsearch.legacy.client;
const indexPattern = await savedObjectsClient.get<IndexPatternAttributes>(
@ -178,7 +183,8 @@ async function fetchIndexPatternDefinition(indexPatternId: string, context: Requ
export function buildFieldList(
indexPattern: SavedObject<IndexPatternAttributes>,
mappings: MappingResult | {},
fieldDescriptors: FieldDescriptor[]
fieldDescriptors: FieldDescriptor[],
metaFields: string[]
): Field[] {
const aliasMap = Object.entries(Object.values(mappings)[0]?.mappings.properties ?? {})
.map(([name, v]) => ({ ...v, name }))
@ -204,6 +210,9 @@ export function buildFieldList(
path: path.split('.'),
lang: field.lang,
script: field.script,
// id is a special case - it doesn't show up in the meta field list,
// but as it's not part of source, it has to be handled separately.
isMeta: metaFields.includes(field.name) || field.name === '_id',
};
}
);
@ -312,7 +321,7 @@ function exists(obj: unknown, path: string[], i = 0): boolean {
* Exported only for unit tests.
*/
export function existingFields(
docs: Array<{ _source: unknown; fields: unknown }>,
docs: Array<{ _source: unknown; fields: unknown; [key: string]: unknown }>,
fields: Field[]
): string[] {
const missingFields = new Set(fields);
@ -323,7 +332,14 @@ export function existingFields(
}
missingFields.forEach((field) => {
if (exists(field.isScript ? doc.fields : doc._source, field.path)) {
let fieldStore = doc._source;
if (field.isScript) {
fieldStore = doc.fields;
}
if (field.isMeta) {
fieldStore = doc;
}
if (exists(fieldStore, field.path)) {
missingFields.delete(field);
}
});

9
x-pack/test/api_integration/apis/lens/existing_fields.ts
View file

@ -20,6 +20,9 @@ const fieldsWithData = [
'@tags',
'@tags.raw',
'@timestamp',
'_id',
'_index',
'_source',
'agent',
'agent.raw',
'bytes',
@ -96,6 +99,9 @@ const fieldsWithData = [
const metricBeatData = [
'@timestamp',
'_id',
'_index',
'_source',
'agent.ephemeral_id',
'agent.hostname',
'agent.id',
@ -185,6 +191,9 @@ export default ({ getService }: FtrProviderContext) => {
'@tags',
'@tags.raw',
'@timestamp',
'_id',
'_index',
'_source',
'agent',
'agent.raw',
'bytes',