maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[7.x] [Discover] Add support for unmapped fields using the fields API (#89074) (#90184)

Browse source 
* [Discover] Add support for unmapped fields using the fields API (#89074)

* Add search source to example plugin.

* Add uiSetting for fields API.

* Update SearchSource to support fields API.

* [PoC] reading from the fields API in Discover

* Add N fields as a default column

* Make fields column non-removeable

* Do not add 'fields' to state

* Remove fields from app state and read from source when needed

* Remove fields column if a new column is added

* Add search source to example plugin.

* Add uiSetting for fields API.

* Update SearchSource to support fields API.

* Improve error handling in search examples plugin.

* Add unit tests for legacy behavior.

* Remove uiSettings feature flag; add fieldsFromSource config.

* Rewrite flatten() based on final API design.

* Update example app based on final API design.

* Update maps app to use legacy fieldsFromSource.

* Update Discover to use legacy fieldsFromSource.

* Rename source filters to field filters.

* Address feedback.

* Update generated docs.

* Update maps functional test.

* Formatting fields column similar to _source

* Moving logic for using search API to updating search source

* Fix small merge error

* Move useSource switch to Discover section of advanced settings

* Do not use fields and source at the same time

* Remove unmapped fields switch

* Add basic support for grouping multifields

* Remove output.txt

* Fix some merge leftovers

* Fix some merge leftovers

* Fix merge errors

* Fix typescript errors and update nested fields logic

* Add a unit test

* Fixing field formats

* Fix multifield selection logic

* Request all fields from source

* Fix eslint

* Fix default columns when switching between _source and fields

* More unit tests

* Update API changes

* Add unit test for discover field details footer

* Remove unused file

* Remove fields formatting from index pattern

* Remove unnecessary check

* Addressing design comments

* Fixing fields column display and renaming it to Document

* Adding more unit tests

* Adding a missing check for useNewFieldsAPI; minor fixes

* Fixing typescript error

* Remove unnecessary console statement

* Add missing prop

* Fixing import order

* Adding functional test to test fields API

* [Functional test] Clean up in after

* Fixing context app

* Addressing PR comments

* Add support for unmapped fields

* Add data migration

* Add toggle unmapped fields logic

* Adding more unit tests

* Some cleanup

* More unit tests

* Fixing failing snapshot

* Add tooltip next to unmapped switch

* Add functional test for the feature

* Fixing a typo in a functional test

* Refetch data when unmapped fields value changes

* Updating mapping

* Support for fields API in search embeddable

* Addressing PR comments

* Fix failing unit test

* Updating the text

Co-authored-by: Luke Elmers <luke.elmers@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

* Fixing ES license

Co-authored-by: Luke Elmers <luke.elmers@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Maja Grubic 2021-02-04 14:13:32 +00:00 committed by GitHub
parent f5bd166b7c
commit b6038f30b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 1065 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/plugins/discover/public/application/angular/context_app.js vendored
View file

@ -50,14 +50,15 @@ getAngularModule().directive('contextApp', function ContextApp() {
function ContextAppController($scope, Private) {
const { filterManager, indexPatterns, uiSettings, navigation } = getServices();
const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
const queryParameterActions = getQueryParameterActions(filterManager, indexPatterns);
const queryActions = Private(QueryActionsProvider);
const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
this.state = createInitialState(
parseInt(uiSettings.get(CONTEXT_STEP_SETTING), 10),
getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING)),
useNewFieldsApi
);
this.state.useNewFieldsApi = useNewFieldsApi;
this.topNavMenu = navigation.ui.TopNavMenu;
this.actions = _.mapValues(

16
src/plugins/discover/public/application/angular/discover.js vendored
View file

@ -739,6 +739,21 @@ function discoverController($route, $scope, Promise) {
history.push('/');
};
const showUnmappedFieldsDefaultValue = $scope.useNewFieldsApi && !!$scope.opts.savedSearch.pre712;
let showUnmappedFields = showUnmappedFieldsDefaultValue;
const onChangeUnmappedFields = (value) => {
showUnmappedFields = value;
$scope.unmappedFieldsConfig.showUnmappedFields = value;
$scope.fetch();
};
$scope.unmappedFieldsConfig = {
showUnmappedFieldsDefaultValue,
showUnmappedFields,
onChangeUnmappedFields,
};
$scope.updateDataSource = () => {
const { indexPattern, searchSource, useNewFieldsApi } = $scope;
const { columns, sort } = $scope.state;
@ -748,6 +763,7 @@ function discoverController($route, $scope, Promise) {
sort,
columns,
useNewFieldsApi,
showUnmappedFields,
});
return Promise.resolve();
};

1
src/plugins/discover/public/application/angular/discover_legacy.html
View file

@ -30,6 +30,7 @@
update-query="handleRefresh"
update-saved-query-id="updateSavedQueryId"
use-new-fields-api="useNewFieldsApi"
unmapped-fields-config="unmappedFieldsConfig"
>
</discover>
</discover-app>

1
src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx
View file

@ -5,7 +5,6 @@
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { i18n } from '@kbn/i18n';
import { IndexPattern } from '../../../../../kibana_services';

1
src/plugins/discover/public/application/components/create_discover_directive.ts
View file

@ -40,5 +40,6 @@ export function createDiscoverDirective(reactDirective: any) {
['topNavMenu', { watchDepth: 'reference' }],
['updateQuery', { watchDepth: 'reference' }],
['updateSavedQueryId', { watchDepth: 'reference' }],
['unmappedFieldsConfig', { watchDepth: 'value' }],
]);
}

2
src/plugins/discover/public/application/components/discover.tsx
View file

@ -80,6 +80,7 @@ export function Discover({
topNavMenu,
updateQuery,
updateSavedQueryId,
unmappedFieldsConfig,
}: DiscoverProps) {
const scrollableDesktop = useRef<HTMLDivElement>(null);
const collapseIcon = useRef<HTMLButtonElement>(null);
@ -146,6 +147,7 @@ export function Discover({
setIndexPattern={setIndexPattern}
isClosed={isSidebarClosed}
trackUiMetric={trackUiMetric}
unmappedFieldsConfig={unmappedFieldsConfig}
useNewFieldsApi={useNewFieldsApi}
/>
</EuiFlexItem>

18
src/plugins/discover/public/application/components/sidebar/discover_field_search.test.tsx
View file

@ -136,4 +136,22 @@ describe('DiscoverFieldSearch', () => {
popover = component.find(EuiPopover);
expect(popover.prop('isOpen')).toBe(false);
});
test('unmapped fields', () => {
const onChangeUnmappedFields = jest.fn();
const componentProps = {
...defaultProps,
showUnmappedFields: true,
useNewFieldsApi: false,
onChangeUnmappedFields,
};
const component = mountComponent(componentProps);
const btn = findTestSubject(component, 'toggleFieldFilterButton');
btn.simulate('click');
const unmappedFieldsSwitch = findTestSubject(component, 'unmappedFieldsSwitch');
act(() => {
unmappedFieldsSwitch.simulate('click');
});
expect(onChangeUnmappedFields).toHaveBeenCalledWith(false);
});
});

93
src/plugins/discover/public/application/components/sidebar/discover_field_search.tsx
View file

@ -27,6 +27,8 @@ import {
EuiOutsideClickDetector,
EuiFilterButton,
EuiSpacer,
EuiIcon,
EuiToolTip,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
@ -35,6 +37,7 @@ export interface State {
aggregatable: string;
type: string;
missing: boolean;
unmappedFields: boolean;
[index: string]: string | boolean;
}
@ -53,13 +56,36 @@ export interface Props {
* types for the type filter
*/
types: string[];
/**
* use new fields api
*/
useNewFieldsApi?: boolean;
/**
* callback funtion to change the value of unmapped fields switch
* @param value new value to set
*/
onChangeUnmappedFields?: (value: boolean) => void;
/**
* should unmapped fields switch be rendered
*/
showUnmappedFields?: boolean;
}
/**
* Component is Discover's side bar to search of available fields
* Additionally there's a button displayed that allows the user to show/hide more filter fields
*/
export function DiscoverFieldSearch({ onChange, value, types }: Props) {
export function DiscoverFieldSearch({
onChange,
value,
types,
useNewFieldsApi,
showUnmappedFields,
onChangeUnmappedFields,
}: Props) {
const searchPlaceholder = i18n.translate('discover.fieldChooser.searchPlaceHolder', {
defaultMessage: 'Search field names',
});
@ -85,6 +111,7 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
aggregatable: 'any',
type: 'any',
missing: true,
unmappedFields: !!showUnmappedFields,
});
if (typeof value !== 'string') {
@ -154,6 +181,14 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
handleValueChange('missing', missingValue);
};
const handleUnmappedFieldsChange = (e: EuiSwitchEvent) => {
const unmappedFieldsValue = e.target.checked;
handleValueChange('unmappedFields', unmappedFieldsValue);
if (onChangeUnmappedFields) {
onChangeUnmappedFields(unmappedFieldsValue);
}
};
const buttonContent = (
<EuiFilterButton
aria-label={filterBtnAriaLabel}
@ -226,6 +261,51 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
);
};
const footer = () => {
if (!showUnmappedFields && useNewFieldsApi) {
return null;
}
return (
<EuiPopoverFooter>
{showUnmappedFields ? (
<EuiFlexGroup>
<EuiFlexItem component="span">
<EuiSwitch
label={i18n.translate('discover.fieldChooser.filter.showUnmappedFields', {
defaultMessage: 'Show unmapped fields',
})}
checked={values.unmappedFields}
onChange={handleUnmappedFieldsChange}
data-test-subj="unmappedFieldsSwitch"
/>
</EuiFlexItem>
<EuiFlexItem component="span" grow={false}>
<EuiToolTip
position="right"
content={i18n.translate('discover.fieldChooser.filter.unmappedFieldsWarning', {
defaultMessage:
'Unmapped fields will be deprecated and removed in a future release.',
})}
>
<EuiIcon type="alert" />
</EuiToolTip>
</EuiFlexItem>
</EuiFlexGroup>
) : null}
{useNewFieldsApi ? null : (
<EuiSwitch
label={i18n.translate('discover.fieldChooser.filter.hideMissingFieldsLabel', {
defaultMessage: 'Hide missing fields',
})}
checked={values.missing}
onChange={handleMissingChange}
data-test-subj="missingSwitch"
/>
)}
</EuiPopoverFooter>
);
};
const selectionPanel = (
<div className="dscFieldSearch__formWrapper">
<EuiForm data-test-subj="filterSelectionPanel">
@ -277,16 +357,7 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) {
})}
</EuiPopoverTitle>
{selectionPanel}
<EuiPopoverFooter>
<EuiSwitch
label={i18n.translate('discover.fieldChooser.filter.hideMissingFieldsLabel', {
defaultMessage: 'Hide missing fields',
})}
checked={values.missing}
onChange={handleMissingChange}
data-test-subj="missingSwitch"
/>
</EuiPopoverFooter>
{footer()}
</EuiPopover>
</EuiFilterGroup>
</EuiOutsideClickDetector>

47
src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx
View file

@ -104,6 +104,27 @@ export interface DiscoverSidebarProps {
* Shows index pattern and a button that displays the sidebar in a flyout
*/
useFlyout?: boolean;
/**
* an object containing properties for proper handling of unmapped fields in the UI
*/
unmappedFieldsConfig?: {
/**
* callback function to change the value of `showUnmappedFields` flag
* @param value new value to set
*/
onChangeUnmappedFields: (value: boolean) => void;
/**
* determines whether to display unmapped fields
* configurable through the switch in the UI
*/
showUnmappedFields: boolean;
/**
* determines if we should display an option to toggle showUnmappedFields value in the first place
* this value is not configurable through the UI
*/
showUnmappedFieldsDefaultValue: boolean;
};
}
export function DiscoverSidebar({
@ -123,6 +144,7 @@ export function DiscoverSidebar({
trackUiMetric,
useNewFieldsApi = false,
useFlyout = false,
unmappedFieldsConfig,
}: DiscoverSidebarProps) {
const [fields, setFields] = useState<IndexPatternField[] | null>(null);
@ -145,14 +167,30 @@ export function DiscoverSidebar({
);
const popularLimit = services.uiSettings.get(FIELDS_LIMIT_SETTING);
const {
selected: selectedFields,
popular: popularFields,
unpopular: unpopularFields,
} = useMemo(
() => groupFields(fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi),
[fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi]
() =>
groupFields(
fields,
columns,
popularLimit,
fieldCounts,
fieldFilter,
useNewFieldsApi,
!!unmappedFieldsConfig?.showUnmappedFields
),
[
fields,
columns,
popularLimit,
fieldCounts,
fieldFilter,
useNewFieldsApi,
unmappedFieldsConfig?.showUnmappedFields,
]
);
const fieldTypes = useMemo(() => {
@ -239,6 +277,9 @@ export function DiscoverSidebar({
onChange={onChangeFieldSearch}
value={fieldFilter.name}
types={fieldTypes}
useNewFieldsApi={useNewFieldsApi}
onChangeUnmappedFields={unmappedFieldsConfig?.onChangeUnmappedFields}
showUnmappedFields={unmappedFieldsConfig?.showUnmappedFieldsDefaultValue}
/>
</form>
</EuiFlexItem>

14
src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.test.tsx
View file

@ -15,7 +15,7 @@ import realHits from 'fixtures/real_hits.js';
import stubbedLogstashFields from 'fixtures/logstash_fields';
import { mountWithIntl } from '@kbn/test/jest';
import React from 'react';
import { DiscoverSidebarProps } from './discover_sidebar';
import { DiscoverSidebar, DiscoverSidebarProps } from './discover_sidebar';
import { coreMock } from '../../../../../../core/public/mocks';
import { IndexPatternAttributes } from '../../../../../data/common';
import { getStubIndexPattern } from '../../../../../data/public/test_utils';
@ -131,4 +131,16 @@ describe('discover responsive sidebar', function () {
findTestSubject(comp, 'plus-extension-gif').simulate('click');
expect(props.onAddFilter).toHaveBeenCalled();
});
it('renders sidebar with unmapped fields config', function () {
const unmappedFieldsConfig = {
onChangeUnmappedFields: jest.fn(),
showUnmappedFields: false,
showUnmappedFieldsDefaultValue: false,
};
const componentProps = { ...props, unmappedFieldsConfig };
const component = mountWithIntl(<DiscoverSidebarResponsive {...componentProps} />);
const discoverSidebar = component.find(DiscoverSidebar);
expect(discoverSidebar).toHaveLength(1);
expect(discoverSidebar.props().unmappedFieldsConfig).toEqual(unmappedFieldsConfig);
});
});

21
src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx
View file

@ -97,6 +97,27 @@ export interface DiscoverSidebarResponsiveProps {
* Read from the Fields API
*/
useNewFieldsApi?: boolean;
/**
* an object containing properties for proper handling of unmapped fields in the UI
*/
unmappedFieldsConfig?: {
/**
* callback function to change the value of `showUnmappedFields` flag
* @param value new value to set
*/
onChangeUnmappedFields: (value: boolean) => void;
/**
* determines whether to display unmapped fields
* configurable through the switch in the UI
*/
showUnmappedFields: boolean;
/**
* determines if we should display an option to toggle showUnmappedFields value in the first place
* this value is not configurable through the UI
*/
showUnmappedFieldsDefaultValue: boolean;
};
}
/**

27
src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts
View file

@ -178,6 +178,7 @@ describe('group_fields', function () {
fieldFilterState,
true
);
expect(actual.popular).toEqual([category]);
expect(actual.selected).toEqual([currency]);
expect(actual.unpopular).toEqual([]);
@ -214,4 +215,30 @@ describe('group_fields', function () {
'unknown',
]);
});
it('excludes unmapped fields if showUnmappedFields set to false', function () {
const fieldFilterState = getDefaultFieldFilter();
const fieldsWithUnmappedField = [...fields];
fieldsWithUnmappedField.push({
name: 'unknown_field',
type: 'unknown',
esTypes: ['unknown'],
count: 1,
scripted: false,
searchable: true,
aggregatable: true,
readFromDocValues: true,
});
const actual = groupFields(
fieldsWithUnmappedField as IndexPatternField[],
['customer_birth_date', 'currency', 'unknown'],
5,
fieldCounts,
fieldFilterState,
true,
false
);
expect(actual.unpopular).toEqual([]);
});
});

7
src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx
View file

@ -24,7 +24,8 @@ export function groupFields(
popularLimit: number,
fieldCounts: Record<string, number>,
fieldFilterState: FieldFilterState,
useNewFieldsApi: boolean
useNewFieldsApi: boolean,
showUnmappedFields = true
): GroupedFields {
const result: GroupedFields = {
selected: [],
@ -61,7 +62,9 @@ export function groupFields(
result.popular.push(field);
}
} else if (field.type !== '_source') {
if (!isSubfield) {
// do not show unmapped fields unless explicitly specified
// do not add subfields to this list
if ((field.type !== 'unknown' || showUnmappedFields) && !isSubfield) {
result.unpopular.push(field);
}
}

20
src/plugins/discover/public/application/components/types.ts
View file

@ -177,4 +177,24 @@ export interface DiscoverProps {
* Function to update the actual savedQuery id
*/
updateSavedQueryId: (savedQueryId?: string) => void;
/**
* An object containing properties for proper handling of unmapped fields in the UI
*/
unmappedFieldsConfig?: {
/**
* determines whether to display unmapped fields
* configurable through the switch in the UI
*/
showUnmappedFields: boolean;
/**
* determines if we should display an option to toggle showUnmappedFields value in the first place
* this value is not configurable through the UI
*/
showUnmappedFieldsDefaultValue: boolean;
/**
* callback function to change the value of `showUnmappedFields` flag
* @param value new value to set
*/
onChangeUnmappedFields: (value: boolean) => void;
};
}

32
src/plugins/discover/public/application/embeddable/search_embeddable.ts
View file

@ -38,7 +38,11 @@ import {
} from '../../kibana_services';
import { SEARCH_EMBEDDABLE_TYPE } from './constants';
import { SavedSearch } from '../..';
import { SAMPLE_SIZE_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common';
import {
SAMPLE_SIZE_SETTING,
SEARCH_FIELDS_FROM_SOURCE,
SORT_DEFAULT_ORDER_SETTING,
} from '../../../common';
import { DiscoverGridSettings } from '../components/discover_grid/types';
import { DiscoverServices } from '../../build_services';
import { ElasticSearchHit } from '../doc_views/doc_views_types';
@ -62,6 +66,7 @@ interface SearchScope extends ng.IScope {
totalHitCount?: number;
isLoading?: boolean;
showTimeCol?: boolean;
useNewFieldsApi?: boolean;
}
interface SearchEmbeddableConfig {
@ -220,11 +225,14 @@ export class SearchEmbeddable
this.updateInput({ sort });
};
const useNewFieldsApi = !getServices().uiSettings.get(SEARCH_FIELDS_FROM_SOURCE, false);
searchScope.useNewFieldsApi = useNewFieldsApi;
searchScope.addColumn = (columnName: string) => {
if (!searchScope.columns) {
return;
}
const columns = columnActions.addColumn(searchScope.columns, columnName);
const columns = columnActions.addColumn(searchScope.columns, columnName, useNewFieldsApi);
this.updateInput({ columns });
};
@ -232,7 +240,7 @@ export class SearchEmbeddable
if (!searchScope.columns) {
return;
}
const columns = columnActions.removeColumn(searchScope.columns, columnName);
const columns = columnActions.removeColumn(searchScope.columns, columnName, useNewFieldsApi);
this.updateInput({ columns });
};
@ -280,10 +288,10 @@ export class SearchEmbeddable
private fetch = async () => {
const searchSessionId = this.input.searchSessionId;
const useNewFieldsApi = !this.services.uiSettings.get(SEARCH_FIELDS_FROM_SOURCE, false);
if (!this.searchScope) return;
const { searchSource } = this.savedSearch;
const { searchSource, pre712 } = this.savedSearch;
// Abort any in-progress requests
if (this.abortController) this.abortController.abort();
@ -298,6 +306,20 @@ export class SearchEmbeddable
this.services.uiSettings.get(SORT_DEFAULT_ORDER_SETTING)
)
);
if (useNewFieldsApi) {
searchSource.removeField('fieldsFromSource');
const fields: Record<string, any> = { field: '*' };
if (pre712) {
fields.include_unmapped = true;
}
searchSource.setField('fields', [fields]);
} else {
searchSource.removeField('fields');
if (this.searchScope.indexPattern) {
const fieldNames = this.searchScope.indexPattern.fields.map((field) => field.name);
searchSource.setField('fieldsFromSource', fieldNames);
}
}
// Log request to inspector
this.inspectorAdapters.requests!.reset();

1
src/plugins/discover/public/application/embeddable/search_template.html
View file

@ -16,5 +16,6 @@
render-complete
sorting="sort"
total-hit-count="totalHitCount"
use-new-fields-api="useNewFieldsApi"
>
</doc-table>

56
src/plugins/discover/public/application/helpers/update_search_source.test.ts
View file

@ -63,7 +63,61 @@ describe('updateSearchSource', () => {
});
expect(result.getField('index')).toEqual(indexPatternMock);
expect(result.getField('size')).toEqual(sampleSize);
expect(result.getField('fields')).toEqual(['*']);
expect(result.getField('fields')).toEqual([{ field: '*' }]);
expect(result.getField('fieldsFromSource')).toBe(undefined);
});
test('requests unmapped fields when the flag is provided, using the new fields api', async () => {
const searchSourceMock = createSearchSourceMock({});
const sampleSize = 250;
const result = updateSearchSource(searchSourceMock, {
indexPattern: indexPatternMock,
services: ({
data: dataPluginMock.createStartContract(),
uiSettings: ({
get: (key: string) => {
if (key === SAMPLE_SIZE_SETTING) {
return sampleSize;
}
return false;
},
} as unknown) as IUiSettingsClient,
} as unknown) as DiscoverServices,
sort: [] as SortOrder[],
columns: [],
useNewFieldsApi: true,
showUnmappedFields: true,
});
expect(result.getField('index')).toEqual(indexPatternMock);
expect(result.getField('size')).toEqual(sampleSize);
expect(result.getField('fields')).toEqual([{ field: '*', include_unmapped: 'true' }]);
expect(result.getField('fieldsFromSource')).toBe(undefined);
});
test('updates a given search source when showUnmappedFields option is set to true', async () => {
const searchSourceMock = createSearchSourceMock({});
const sampleSize = 250;
const result = updateSearchSource(searchSourceMock, {
indexPattern: indexPatternMock,
services: ({
data: dataPluginMock.createStartContract(),
uiSettings: ({
get: (key: string) => {
if (key === SAMPLE_SIZE_SETTING) {
return sampleSize;
}
return false;
},
} as unknown) as IUiSettingsClient,
} as unknown) as DiscoverServices,
sort: [] as SortOrder[],
columns: [],
useNewFieldsApi: true,
showUnmappedFields: true,
});
expect(result.getField('index')).toEqual(indexPatternMock);
expect(result.getField('size')).toEqual(sampleSize);
expect(result.getField('fields')).toEqual([{ field: '*', include_unmapped: 'true' }]);
expect(result.getField('fieldsFromSource')).toBe(undefined);
});
});

8
src/plugins/discover/public/application/helpers/update_search_source.ts
View file

@ -23,12 +23,14 @@ export function updateSearchSource(
sort,
columns,
useNewFieldsApi,
showUnmappedFields,
}: {
indexPattern: IndexPattern;
services: DiscoverServices;
sort: SortOrder[];
columns: string[];
useNewFieldsApi: boolean;
showUnmappedFields?: boolean;
}
) {
const { uiSettings, data } = services;
@ -46,7 +48,11 @@ export function updateSearchSource(
.setField('filter', data.query.filterManager.getFilters());
if (useNewFieldsApi) {
searchSource.removeField('fieldsFromSource');
searchSource.setField('fields', ['*']);
const fields: Record<string, string> = { field: '*' };
if (showUnmappedFields) {
fields.include_unmapped = 'true';
}
searchSource.setField('fields', [fields]);
} else {
searchSource.removeField('fields');
const fieldNames = indexPattern.fields.map((field) => field.name);

3
src/plugins/discover/public/saved_searches/_saved_search.ts
View file

@ -19,6 +19,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) {
grid: 'object',
sort: 'keyword',
version: 'integer',
pre712: 'boolean',
};
// Order these fields to the top, the rest are alphabetical
public static fieldOrder = ['title', 'description'];
@ -39,6 +40,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) {
grid: 'object',
sort: 'keyword',
version: 'integer',
pre712: 'boolean',
},
searchSource: true,
defaults: {
@ -48,6 +50,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) {
hits: 0,
sort: [],
version: 1,
pre712: false,
},
});
this.showInRecentlyAccessed = true;

1
src/plugins/discover/public/saved_searches/types.ts
View file

@ -23,6 +23,7 @@ export interface SavedSearch {
save: (saveOptions: SavedObjectSaveOpts) => Promise<string>;
lastSavedTitle?: string;
copyOnSave?: boolean;
pre712?: boolean;
}
export interface SavedSearchLoader {
get: (id: string) => Promise<SavedSearch>;

1
src/plugins/discover/server/saved_objects/search.ts
View file

@ -44,6 +44,7 @@ export const searchSavedObjectType: SavedObjectsType = {
title: { type: 'text' },
grid: { type: 'object', enabled: false },
version: { type: 'integer' },
pre712: { type: 'boolean' },
},
},
migrations: searchMigrations as any,

37
src/plugins/discover/server/saved_objects/search_migrations.test.ts
View file

@ -350,4 +350,41 @@ Object {
testMigrateMatchAllQuery(migrationFn);
});
});
describe('7.12.0', () => {
const migrationFn = searchMigrations['7.12.0'];
describe('migrateExistingSavedSearch', () => {
it('should add a new flag to existing saved searches', () => {
const migratedDoc = migrationFn(
{
type: 'search',
attributes: {
kibanaSavedObjectMeta: {},
},
},
savedObjectMigrationContext
);
const migratedPre712Flag = migratedDoc.attributes.pre712;
expect(migratedPre712Flag).toEqual(true);
});
it('should not modify a flag if it already exists', () => {
const migratedDoc = migrationFn(
{
type: 'search',
attributes: {
kibanaSavedObjectMeta: {},
pre712: false,
},
},
savedObjectMigrationContext
);
const migratedPre712Flag = migratedDoc.attributes.pre712;
expect(migratedPre712Flag).toEqual(false);
});
});
});
});

19
src/plugins/discover/server/saved_objects/search_migrations.ts
View file

@ -117,9 +117,28 @@ const migrateSearchSortToNestedArray: SavedObjectMigrationFn<any, any> = (doc) =
};
};
const migrateExistingSavedSearch: SavedObjectMigrationFn<any, any> = (doc) => {
if (!doc.attributes) {
return doc;
}
const pre712 = doc.attributes.pre712;
// pre712 already has a value
if (pre712 !== undefined) {
return doc;
}
return {
...doc,
attributes: {
...doc.attributes,
pre712: true,
},
};
};
export const searchMigrations = {
'6.7.2': flow(migrateMatchAllQuery),
'7.0.0': flow(setNewReferences),
'7.4.0': flow(migrateSearchSortToNestedArray),
'7.9.3': flow(migrateMatchAllQuery),
'7.12.0': flow(migrateExistingSavedSearch),
};

86
test/functional/apps/discover/_indexpattern_with_unmapped_fields.ts Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const testSubjects = getService('testSubjects');
const log = getService('log');
const retry = getService('retry');
const PageObjects = getPageObjects(['common', 'timePicker', 'discover']);
describe('index pattern with unmapped fields', () => {
const unmappedFieldsSwitchSelector = 'unmappedFieldsSwitch';
before(async () => {
await esArchiver.loadIfNeeded('unmapped_fields');
await kibanaServer.uiSettings.replace({ defaultIndex: 'test-index-unmapped-fields' });
await kibanaServer.uiSettings.update({
'discover:searchFieldsFromSource': false,
});
log.debug('discover');
const fromTime = 'Jan 20, 2021 @ 00:00:00.000';
const toTime = 'Jan 25, 2021 @ 00:00:00.000';
await PageObjects.common.navigateToApp('discover');
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
});
after(async () => {
await esArchiver.unload('unmapped_fields');
});
it('unmapped fields do not exist on a new saved search', async () => {
const expectedHitCount = '4';
await retry.try(async function () {
expect(await PageObjects.discover.getHitCount()).to.be(expectedHitCount);
});
const allFields = await PageObjects.discover.getAllFieldNames();
// message is a mapped field
expect(allFields.includes('message')).to.be(true);
// sender is not a mapped field
expect(allFields.includes('sender')).to.be(false);
});
it('unmapped fields toggle does not exist on a new saved search', async () => {
await PageObjects.discover.openSidebarFieldFilter();
await testSubjects.existOrFail('filterSelectionPanel');
await testSubjects.missingOrFail('unmappedFieldsSwitch');
});
it('unmapped fields exist on an existing saved search', async () => {
await PageObjects.discover.loadSavedSearch('Existing Saved Search');
const expectedHitCount = '4';
await retry.try(async function () {
expect(await PageObjects.discover.getHitCount()).to.be(expectedHitCount);
});
const allFields = await PageObjects.discover.getAllFieldNames();
expect(allFields.includes('message')).to.be(true);
expect(allFields.includes('sender')).to.be(true);
expect(allFields.includes('receiver')).to.be(true);
});
it('unmapped fields toggle exists on an existing saved search', async () => {
await PageObjects.discover.openSidebarFieldFilter();
await testSubjects.existOrFail('filterSelectionPanel');
await testSubjects.existOrFail(unmappedFieldsSwitchSelector);
expect(await testSubjects.isEuiSwitchChecked(unmappedFieldsSwitchSelector)).to.be(true);
});
it('switching unmapped fields toggle off hides unmapped fields', async () => {
await testSubjects.setEuiSwitch(unmappedFieldsSwitchSelector, 'uncheck');
await PageObjects.discover.closeSidebarFieldFilter();
const allFields = await PageObjects.discover.getAllFieldNames();
expect(allFields.includes('message')).to.be(true);
expect(allFields.includes('sender')).to.be(false);
expect(allFields.includes('receiver')).to.be(false);
});
});
}

1
test/functional/apps/discover/index.ts
View file

@ -46,5 +46,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./_data_grid_field_data'));
loadTestFile(require.resolve('./_data_grid_doc_navigation'));
loadTestFile(require.resolve('./_data_grid_doc_table'));
loadTestFile(require.resolve('./_indexpattern_with_unmapped_fields'));
});
}

BIN
test/functional/fixtures/es_archiver/data/data.json.gz Normal file
View file

Binary file not shown.

450
test/functional/fixtures/es_archiver/data/mappings.json Normal file
View file

@ -0,0 +1,450 @@
{
"type": "index",
"value": {
"aliases": {
".kibana": {
}
},
"index": ".kibana_1",
"mappings": {
"_meta": {
"migrationMappingPropertyHashes": {
"application_usage_daily": "43b8830d5d0df85a6823d290885fc9fd",
"application_usage_totals": "3d1b76c39bfb2cc8296b024d73854724",
"application_usage_transactional": "3d1b76c39bfb2cc8296b024d73854724",
"config": "c63748b75f39d0c54de12d12c1ccbc20",
"core-usage-stats": "3d1b76c39bfb2cc8296b024d73854724",
"coreMigrationVersion": "2f4316de49999235636386fe51dc06c1",
"dashboard": "40554caf09725935e2c02e02563a2d07",
"index-pattern": "45915a1ad866812242df474eb0479052",
"kql-telemetry": "d12a98a6f19a2d273696597547e064ee",
"legacy-url-alias": "3d1b76c39bfb2cc8296b024d73854724",
"migrationVersion": "4a1746014a75ade3a714e1db5763276f",
"namespace": "2f4316de49999235636386fe51dc06c1",
"namespaces": "2f4316de49999235636386fe51dc06c1",
"originId": "2f4316de49999235636386fe51dc06c1",
"query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9",
"references": "7997cf5a56cc02bdc9c93361bde732b0",
"sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4",
"search": "e5b843b43566421ffa75fb499271dc34",
"search-telemetry": "3d1b76c39bfb2cc8296b024d73854724",
"telemetry": "36a616f7026dfa617d6655df850fe16d",
"timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf",
"type": "2f4316de49999235636386fe51dc06c1",
"ui-counter": "0d409297dc5ebe1e3a1da691c6ee32e3",
"ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3",
"updated_at": "00da57df13e94e9d98437d13ace4bfe0",
"url": "c7f66a0df8b1b52f17c28c4adb111105",
"visualization": "f819cf6636b75c9e76ba733a0c6ef355"
}
},
"dynamic": "strict",
"properties": {
"application_usage_daily": {
"dynamic": "false",
"properties": {
"timestamp": {
"type": "date"
}
}
},
"application_usage_totals": {
"dynamic": "false",
"type": "object"
},
"application_usage_transactional": {
"dynamic": "false",
"type": "object"
},
"config": {
"dynamic": "false",
"properties": {
"buildNum": {
"type": "keyword"
}
}
},
"core-usage-stats": {
"dynamic": "false",
"type": "object"
},
"coreMigrationVersion": {
"type": "keyword"
},
"dashboard": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"doc_values": false,
"index": false,
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"index": false,
"type": "text"
}
}
},
"optionsJSON": {
"index": false,
"type": "text"
},
"panelsJSON": {
"index": false,
"type": "text"
},
"refreshInterval": {
"properties": {
"display": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"pause": {
"doc_values": false,
"index": false,
"type": "boolean"
},
"section": {
"doc_values": false,
"index": false,
"type": "integer"
},
"value": {
"doc_values": false,
"index": false,
"type": "integer"
}
}
},
"timeFrom": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"timeRestore": {
"doc_values": false,
"index": false,
"type": "boolean"
},
"timeTo": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"index-pattern": {
"dynamic": "false",
"properties": {
"title": {
"type": "text"
},
"type": {
"type": "keyword"
}
}
},
"kql-telemetry": {
"properties": {
"optInCount": {
"type": "long"
},
"optOutCount": {
"type": "long"
}
}
},
"legacy-url-alias": {
"dynamic": "false",
"type": "object"
},
"migrationVersion": {
"dynamic": "true",
"properties": {
"config": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"namespace": {
"type": "keyword"
},
"namespaces": {
"type": "keyword"
},
"originId": {
"type": "keyword"
},
"query": {
"properties": {
"description": {
"type": "text"
},
"filters": {
"enabled": false,
"type": "object"
},
"query": {
"properties": {
"language": {
"type": "keyword"
},
"query": {
"index": false,
"type": "keyword"
}
}
},
"timefilter": {
"enabled": false,
"type": "object"
},
"title": {
"type": "text"
}
}
},
"references": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"type": {
"type": "keyword"
}
},
"type": "nested"
},
"sample-data-telemetry": {
"properties": {
"installCount": {
"type": "long"
},
"unInstallCount": {
"type": "long"
}
}
},
"search": {
"properties": {
"columns": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"description": {
"type": "text"
},
"grid": {
"enabled": false,
"type": "object"
},
"hits": {
"doc_values": false,
"index": false,
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"index": false,
"type": "text"
}
}
},
"pre712": {
"type": "boolean"
},
"sort": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"search-telemetry": {
"dynamic": "false",
"type": "object"
},
"telemetry": {
"properties": {
"allowChangingOptInStatus": {
"type": "boolean"
},
"enabled": {
"type": "boolean"
},
"lastReported": {
"type": "date"
},
"lastVersionChecked": {
"type": "keyword"
},
"reportFailureCount": {
"type": "integer"
},
"reportFailureVersion": {
"type": "keyword"
},
"sendUsageFrom": {
"type": "keyword"
},
"userHasSeenNotice": {
"type": "boolean"
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"type": "keyword"
},
"ui-counter": {
"properties": {
"count": {
"type": "integer"
}
}
},
"ui-metric": {
"properties": {
"count": {
"type": "integer"
}
}
},
"updated_at": {
"type": "date"
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"fields": {
"keyword": {
"ignore_above": 2048,
"type": "keyword"
}
},
"type": "text"
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"index": false,
"type": "text"
}
}
},
"savedSearchRefName": {
"doc_values": false,
"index": false,
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"index": false,
"type": "text"
},
"version": {
"type": "integer"
},
"visState": {
"index": false,
"type": "text"
}
}
}
}
},
"settings": {
"index": {
"auto_expand_replicas": "0-1",
"number_of_replicas": "0",
"number_of_shards": "1"
}
}
}
}

105
test/functional/fixtures/es_archiver/unmapped_fields/data.json Normal file
View file

@ -0,0 +1,105 @@
{
"type": "doc",
"value": {
"id": "search:cd43f5c2-h761-13f6-9486-733b1ac9221a",
"index": ".kibana",
"source": {
"search": {
"columns": [
"_source"
],
"description": "Existing Saved Search",
"hits": 4,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\n \"index\": \"test-index-unmapped-fields\",\n \"highlightAll\": true,\n \"filter\": [],\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n }\n}"
},
"sort": [
"@timestamp",
"desc"
],
"title": "Existing Saved Search",
"version": 1
},
"type": "search"
}
}
}
{
"type": "doc",
"value": {
"id": "index-pattern:test-index-unmapped-fields",
"index": ".kibana",
"source": {
"index-pattern": {
"fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":4,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
"timeFieldName": "timestamp",
"title": "test-index-unmapped-fields",
"fieldFormatMap": "{\"timestamp\":{\"id\":\"date\"}}"
},
"type": "index-pattern"
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "1",
"index": "test-index-unmapped-fields",
"source": {
"timestamp": "2021-01-21T12:00:00.000Z",
"message": "Something bad is coming",
"address": "Elm Street 1"
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "2",
"index": "test-index-unmapped-fields",
"source": {
"timestamp": "2021-01-22T12:00:00.000Z",
"message": "We have a new case",
"address": "221b Baker Street"
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "3",
"index": "test-index-unmapped-fields",
"source": {
"timestamp": "2021-01-23T12:00:00.000Z",
"message": "We have a new case",
"address": "221b Baker Street",
"sender": "John Doe",
"receiver": "Sherlock Holmes"
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "4",
"index": "test-index-unmapped-fields",
"source": {
"timestamp": "2021-01-24T12:00:00.000Z",
"message": "I am coming for you",
"address": "13 Elm Street",
"sender": "Freddy Krueger",
"receiver": "Nancy Thompson"
},
"type": "_doc"
}
}

20
test/functional/fixtures/es_archiver/unmapped_fields/mappings.json Normal file
View file

@ -0,0 +1,20 @@
{
"type": "index",
"value": {
"index": "test-index-unmapped-fields",
"mappings": {
"dynamic": "false",
"properties": {
"timestamp": {"type": "date"},
"message": { "type": "text" },
"address": { "type": "text" }
}
},
"settings": {
"index": {
"number_of_replicas": "1",
"number_of_shards": "1"
}
}
}
}