diff --git a/src/plugins/discover/public/application/angular/context_app.js b/src/plugins/discover/public/application/angular/context_app.js index 98820f137b7f..04406338f49a 100644 --- a/src/plugins/discover/public/application/angular/context_app.js +++ b/src/plugins/discover/public/application/angular/context_app.js @@ -50,14 +50,15 @@ getAngularModule().directive('contextApp', function ContextApp() { function ContextAppController($scope, Private) { const { filterManager, indexPatterns, uiSettings, navigation } = getServices(); + const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); const queryParameterActions = getQueryParameterActions(filterManager, indexPatterns); const queryActions = Private(QueryActionsProvider); - const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); this.state = createInitialState( parseInt(uiSettings.get(CONTEXT_STEP_SETTING), 10), getFirstSortableField(this.indexPattern, uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING)), useNewFieldsApi ); + this.state.useNewFieldsApi = useNewFieldsApi; this.topNavMenu = navigation.ui.TopNavMenu; this.actions = _.mapValues( diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 16d50aa874cb..fac5bb2d8de4 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -739,6 +739,21 @@ function discoverController($route, $scope, Promise) { history.push('/'); }; + const showUnmappedFieldsDefaultValue = $scope.useNewFieldsApi && !!$scope.opts.savedSearch.pre712; + let showUnmappedFields = showUnmappedFieldsDefaultValue; + + const onChangeUnmappedFields = (value) => { + showUnmappedFields = value; + $scope.unmappedFieldsConfig.showUnmappedFields = value; + $scope.fetch(); + }; + + $scope.unmappedFieldsConfig = { + showUnmappedFieldsDefaultValue, + showUnmappedFields, + onChangeUnmappedFields, + }; + $scope.updateDataSource = () => { const { indexPattern, searchSource, useNewFieldsApi } = $scope; const { columns, sort } = $scope.state; @@ -748,6 +763,7 @@ function discoverController($route, $scope, Promise) { sort, columns, useNewFieldsApi, + showUnmappedFields, }); return Promise.resolve(); }; diff --git a/src/plugins/discover/public/application/angular/discover_legacy.html b/src/plugins/discover/public/application/angular/discover_legacy.html index 76e5c568ffde..83a9cf23c85f 100644 --- a/src/plugins/discover/public/application/angular/discover_legacy.html +++ b/src/plugins/discover/public/application/angular/discover_legacy.html @@ -30,6 +30,7 @@ update-query="handleRefresh" update-saved-query-id="updateSavedQueryId" use-new-fields-api="useNewFieldsApi" + unmapped-fields-config="unmappedFieldsConfig" > diff --git a/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx b/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx index a75aea716973..13be90667470 100644 --- a/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx +++ b/src/plugins/discover/public/application/angular/doc_table/components/table_header/helpers.tsx @@ -5,7 +5,6 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - import { i18n } from '@kbn/i18n'; import { IndexPattern } from '../../../../../kibana_services'; diff --git a/src/plugins/discover/public/application/components/create_discover_directive.ts b/src/plugins/discover/public/application/components/create_discover_directive.ts index 5a58fe808f84..2a88c1b71313 100644 --- a/src/plugins/discover/public/application/components/create_discover_directive.ts +++ b/src/plugins/discover/public/application/components/create_discover_directive.ts @@ -40,5 +40,6 @@ export function createDiscoverDirective(reactDirective: any) { ['topNavMenu', { watchDepth: 'reference' }], ['updateQuery', { watchDepth: 'reference' }], ['updateSavedQueryId', { watchDepth: 'reference' }], + ['unmappedFieldsConfig', { watchDepth: 'value' }], ]); } diff --git a/src/plugins/discover/public/application/components/discover.tsx b/src/plugins/discover/public/application/components/discover.tsx index 038c33d254ca..e6c4524f81f5 100644 --- a/src/plugins/discover/public/application/components/discover.tsx +++ b/src/plugins/discover/public/application/components/discover.tsx @@ -80,6 +80,7 @@ export function Discover({ topNavMenu, updateQuery, updateSavedQueryId, + unmappedFieldsConfig, }: DiscoverProps) { const scrollableDesktop = useRef(null); const collapseIcon = useRef(null); @@ -146,6 +147,7 @@ export function Discover({ setIndexPattern={setIndexPattern} isClosed={isSidebarClosed} trackUiMetric={trackUiMetric} + unmappedFieldsConfig={unmappedFieldsConfig} useNewFieldsApi={useNewFieldsApi} /> diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_search.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_search.test.tsx index 04562cbd2652..797a6c9697c3 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field_search.test.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_search.test.tsx @@ -136,4 +136,22 @@ describe('DiscoverFieldSearch', () => { popover = component.find(EuiPopover); expect(popover.prop('isOpen')).toBe(false); }); + + test('unmapped fields', () => { + const onChangeUnmappedFields = jest.fn(); + const componentProps = { + ...defaultProps, + showUnmappedFields: true, + useNewFieldsApi: false, + onChangeUnmappedFields, + }; + const component = mountComponent(componentProps); + const btn = findTestSubject(component, 'toggleFieldFilterButton'); + btn.simulate('click'); + const unmappedFieldsSwitch = findTestSubject(component, 'unmappedFieldsSwitch'); + act(() => { + unmappedFieldsSwitch.simulate('click'); + }); + expect(onChangeUnmappedFields).toHaveBeenCalledWith(false); + }); }); diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_search.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_search.tsx index 0c29fd4f37cc..8fb90bfea3a9 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_field_search.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_search.tsx @@ -27,6 +27,8 @@ import { EuiOutsideClickDetector, EuiFilterButton, EuiSpacer, + EuiIcon, + EuiToolTip, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -35,6 +37,7 @@ export interface State { aggregatable: string; type: string; missing: boolean; + unmappedFields: boolean; [index: string]: string | boolean; } @@ -53,13 +56,36 @@ export interface Props { * types for the type filter */ types: string[]; + + /** + * use new fields api + */ + useNewFieldsApi?: boolean; + + /** + * callback funtion to change the value of unmapped fields switch + * @param value new value to set + */ + onChangeUnmappedFields?: (value: boolean) => void; + + /** + * should unmapped fields switch be rendered + */ + showUnmappedFields?: boolean; } /** * Component is Discover's side bar to search of available fields * Additionally there's a button displayed that allows the user to show/hide more filter fields */ -export function DiscoverFieldSearch({ onChange, value, types }: Props) { +export function DiscoverFieldSearch({ + onChange, + value, + types, + useNewFieldsApi, + showUnmappedFields, + onChangeUnmappedFields, +}: Props) { const searchPlaceholder = i18n.translate('discover.fieldChooser.searchPlaceHolder', { defaultMessage: 'Search field names', }); @@ -85,6 +111,7 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) { aggregatable: 'any', type: 'any', missing: true, + unmappedFields: !!showUnmappedFields, }); if (typeof value !== 'string') { @@ -154,6 +181,14 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) { handleValueChange('missing', missingValue); }; + const handleUnmappedFieldsChange = (e: EuiSwitchEvent) => { + const unmappedFieldsValue = e.target.checked; + handleValueChange('unmappedFields', unmappedFieldsValue); + if (onChangeUnmappedFields) { + onChangeUnmappedFields(unmappedFieldsValue); + } + }; + const buttonContent = ( { + if (!showUnmappedFields && useNewFieldsApi) { + return null; + } + return ( + + {showUnmappedFields ? ( + + + + + + + + + + + ) : null} + {useNewFieldsApi ? null : ( + + )} + + ); + }; + const selectionPanel = (
@@ -277,16 +357,7 @@ export function DiscoverFieldSearch({ onChange, value, types }: Props) { })} {selectionPanel} - - - + {footer()} diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx index b2d3df1f0a99..db5f40d8e13c 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx @@ -104,6 +104,27 @@ export interface DiscoverSidebarProps { * Shows index pattern and a button that displays the sidebar in a flyout */ useFlyout?: boolean; + + /** + * an object containing properties for proper handling of unmapped fields in the UI + */ + unmappedFieldsConfig?: { + /** + * callback function to change the value of `showUnmappedFields` flag + * @param value new value to set + */ + onChangeUnmappedFields: (value: boolean) => void; + /** + * determines whether to display unmapped fields + * configurable through the switch in the UI + */ + showUnmappedFields: boolean; + /** + * determines if we should display an option to toggle showUnmappedFields value in the first place + * this value is not configurable through the UI + */ + showUnmappedFieldsDefaultValue: boolean; + }; } export function DiscoverSidebar({ @@ -123,6 +144,7 @@ export function DiscoverSidebar({ trackUiMetric, useNewFieldsApi = false, useFlyout = false, + unmappedFieldsConfig, }: DiscoverSidebarProps) { const [fields, setFields] = useState(null); @@ -145,14 +167,30 @@ export function DiscoverSidebar({ ); const popularLimit = services.uiSettings.get(FIELDS_LIMIT_SETTING); - const { selected: selectedFields, popular: popularFields, unpopular: unpopularFields, } = useMemo( - () => groupFields(fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi), - [fields, columns, popularLimit, fieldCounts, fieldFilter, useNewFieldsApi] + () => + groupFields( + fields, + columns, + popularLimit, + fieldCounts, + fieldFilter, + useNewFieldsApi, + !!unmappedFieldsConfig?.showUnmappedFields + ), + [ + fields, + columns, + popularLimit, + fieldCounts, + fieldFilter, + useNewFieldsApi, + unmappedFieldsConfig?.showUnmappedFields, + ] ); const fieldTypes = useMemo(() => { @@ -239,6 +277,9 @@ export function DiscoverSidebar({ onChange={onChangeFieldSearch} value={fieldFilter.name} types={fieldTypes} + useNewFieldsApi={useNewFieldsApi} + onChangeUnmappedFields={unmappedFieldsConfig?.onChangeUnmappedFields} + showUnmappedFields={unmappedFieldsConfig?.showUnmappedFieldsDefaultValue} /> diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.test.tsx index dc6c57fb7bdb..7ee6cb56d99f 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.test.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.test.tsx @@ -15,7 +15,7 @@ import realHits from 'fixtures/real_hits.js'; import stubbedLogstashFields from 'fixtures/logstash_fields'; import { mountWithIntl } from '@kbn/test/jest'; import React from 'react'; -import { DiscoverSidebarProps } from './discover_sidebar'; +import { DiscoverSidebar, DiscoverSidebarProps } from './discover_sidebar'; import { coreMock } from '../../../../../../core/public/mocks'; import { IndexPatternAttributes } from '../../../../../data/common'; import { getStubIndexPattern } from '../../../../../data/public/test_utils'; @@ -131,4 +131,16 @@ describe('discover responsive sidebar', function () { findTestSubject(comp, 'plus-extension-gif').simulate('click'); expect(props.onAddFilter).toHaveBeenCalled(); }); + it('renders sidebar with unmapped fields config', function () { + const unmappedFieldsConfig = { + onChangeUnmappedFields: jest.fn(), + showUnmappedFields: false, + showUnmappedFieldsDefaultValue: false, + }; + const componentProps = { ...props, unmappedFieldsConfig }; + const component = mountWithIntl(); + const discoverSidebar = component.find(DiscoverSidebar); + expect(discoverSidebar).toHaveLength(1); + expect(discoverSidebar.props().unmappedFieldsConfig).toEqual(unmappedFieldsConfig); + }); }); diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx index 2585adf216dc..b8e8fd0679ba 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar_responsive.tsx @@ -97,6 +97,27 @@ export interface DiscoverSidebarResponsiveProps { * Read from the Fields API */ useNewFieldsApi?: boolean; + + /** + * an object containing properties for proper handling of unmapped fields in the UI + */ + unmappedFieldsConfig?: { + /** + * callback function to change the value of `showUnmappedFields` flag + * @param value new value to set + */ + onChangeUnmappedFields: (value: boolean) => void; + /** + * determines whether to display unmapped fields + * configurable through the switch in the UI + */ + showUnmappedFields: boolean; + /** + * determines if we should display an option to toggle showUnmappedFields value in the first place + * this value is not configurable through the UI + */ + showUnmappedFieldsDefaultValue: boolean; + }; } /** diff --git a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts index bb9e99ad4167..89980f7fd0f5 100644 --- a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts +++ b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.test.ts @@ -178,6 +178,7 @@ describe('group_fields', function () { fieldFilterState, true ); + expect(actual.popular).toEqual([category]); expect(actual.selected).toEqual([currency]); expect(actual.unpopular).toEqual([]); @@ -214,4 +215,30 @@ describe('group_fields', function () { 'unknown', ]); }); + + it('excludes unmapped fields if showUnmappedFields set to false', function () { + const fieldFilterState = getDefaultFieldFilter(); + const fieldsWithUnmappedField = [...fields]; + fieldsWithUnmappedField.push({ + name: 'unknown_field', + type: 'unknown', + esTypes: ['unknown'], + count: 1, + scripted: false, + searchable: true, + aggregatable: true, + readFromDocValues: true, + }); + + const actual = groupFields( + fieldsWithUnmappedField as IndexPatternField[], + ['customer_birth_date', 'currency', 'unknown'], + 5, + fieldCounts, + fieldFilterState, + true, + false + ); + expect(actual.unpopular).toEqual([]); + }); }); diff --git a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx index 64b175d4432e..c7242a8518b5 100644 --- a/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx +++ b/src/plugins/discover/public/application/components/sidebar/lib/group_fields.tsx @@ -24,7 +24,8 @@ export function groupFields( popularLimit: number, fieldCounts: Record, fieldFilterState: FieldFilterState, - useNewFieldsApi: boolean + useNewFieldsApi: boolean, + showUnmappedFields = true ): GroupedFields { const result: GroupedFields = { selected: [], @@ -61,7 +62,9 @@ export function groupFields( result.popular.push(field); } } else if (field.type !== '_source') { - if (!isSubfield) { + // do not show unmapped fields unless explicitly specified + // do not add subfields to this list + if ((field.type !== 'unknown' || showUnmappedFields) && !isSubfield) { result.unpopular.push(field); } } diff --git a/src/plugins/discover/public/application/components/types.ts b/src/plugins/discover/public/application/components/types.ts index d02b142dbea5..abc8086e7271 100644 --- a/src/plugins/discover/public/application/components/types.ts +++ b/src/plugins/discover/public/application/components/types.ts @@ -177,4 +177,24 @@ export interface DiscoverProps { * Function to update the actual savedQuery id */ updateSavedQueryId: (savedQueryId?: string) => void; + /** + * An object containing properties for proper handling of unmapped fields in the UI + */ + unmappedFieldsConfig?: { + /** + * determines whether to display unmapped fields + * configurable through the switch in the UI + */ + showUnmappedFields: boolean; + /** + * determines if we should display an option to toggle showUnmappedFields value in the first place + * this value is not configurable through the UI + */ + showUnmappedFieldsDefaultValue: boolean; + /** + * callback function to change the value of `showUnmappedFields` flag + * @param value new value to set + */ + onChangeUnmappedFields: (value: boolean) => void; + }; } diff --git a/src/plugins/discover/public/application/embeddable/search_embeddable.ts b/src/plugins/discover/public/application/embeddable/search_embeddable.ts index 140a246572d6..d04d482c7aad 100644 --- a/src/plugins/discover/public/application/embeddable/search_embeddable.ts +++ b/src/plugins/discover/public/application/embeddable/search_embeddable.ts @@ -38,7 +38,11 @@ import { } from '../../kibana_services'; import { SEARCH_EMBEDDABLE_TYPE } from './constants'; import { SavedSearch } from '../..'; -import { SAMPLE_SIZE_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common'; +import { + SAMPLE_SIZE_SETTING, + SEARCH_FIELDS_FROM_SOURCE, + SORT_DEFAULT_ORDER_SETTING, +} from '../../../common'; import { DiscoverGridSettings } from '../components/discover_grid/types'; import { DiscoverServices } from '../../build_services'; import { ElasticSearchHit } from '../doc_views/doc_views_types'; @@ -62,6 +66,7 @@ interface SearchScope extends ng.IScope { totalHitCount?: number; isLoading?: boolean; showTimeCol?: boolean; + useNewFieldsApi?: boolean; } interface SearchEmbeddableConfig { @@ -220,11 +225,14 @@ export class SearchEmbeddable this.updateInput({ sort }); }; + const useNewFieldsApi = !getServices().uiSettings.get(SEARCH_FIELDS_FROM_SOURCE, false); + searchScope.useNewFieldsApi = useNewFieldsApi; + searchScope.addColumn = (columnName: string) => { if (!searchScope.columns) { return; } - const columns = columnActions.addColumn(searchScope.columns, columnName); + const columns = columnActions.addColumn(searchScope.columns, columnName, useNewFieldsApi); this.updateInput({ columns }); }; @@ -232,7 +240,7 @@ export class SearchEmbeddable if (!searchScope.columns) { return; } - const columns = columnActions.removeColumn(searchScope.columns, columnName); + const columns = columnActions.removeColumn(searchScope.columns, columnName, useNewFieldsApi); this.updateInput({ columns }); }; @@ -280,10 +288,10 @@ export class SearchEmbeddable private fetch = async () => { const searchSessionId = this.input.searchSessionId; - + const useNewFieldsApi = !this.services.uiSettings.get(SEARCH_FIELDS_FROM_SOURCE, false); if (!this.searchScope) return; - const { searchSource } = this.savedSearch; + const { searchSource, pre712 } = this.savedSearch; // Abort any in-progress requests if (this.abortController) this.abortController.abort(); @@ -298,6 +306,20 @@ export class SearchEmbeddable this.services.uiSettings.get(SORT_DEFAULT_ORDER_SETTING) ) ); + if (useNewFieldsApi) { + searchSource.removeField('fieldsFromSource'); + const fields: Record = { field: '*' }; + if (pre712) { + fields.include_unmapped = true; + } + searchSource.setField('fields', [fields]); + } else { + searchSource.removeField('fields'); + if (this.searchScope.indexPattern) { + const fieldNames = this.searchScope.indexPattern.fields.map((field) => field.name); + searchSource.setField('fieldsFromSource', fieldNames); + } + } // Log request to inspector this.inspectorAdapters.requests!.reset(); diff --git a/src/plugins/discover/public/application/embeddable/search_template.html b/src/plugins/discover/public/application/embeddable/search_template.html index be2f5cceac08..3e37b3645650 100644 --- a/src/plugins/discover/public/application/embeddable/search_template.html +++ b/src/plugins/discover/public/application/embeddable/search_template.html @@ -16,5 +16,6 @@ render-complete sorting="sort" total-hit-count="totalHitCount" + use-new-fields-api="useNewFieldsApi" > diff --git a/src/plugins/discover/public/application/helpers/update_search_source.test.ts b/src/plugins/discover/public/application/helpers/update_search_source.test.ts index 9664ddf3f780..51586a6bccc2 100644 --- a/src/plugins/discover/public/application/helpers/update_search_source.test.ts +++ b/src/plugins/discover/public/application/helpers/update_search_source.test.ts @@ -63,7 +63,61 @@ describe('updateSearchSource', () => { }); expect(result.getField('index')).toEqual(indexPatternMock); expect(result.getField('size')).toEqual(sampleSize); - expect(result.getField('fields')).toEqual(['*']); + expect(result.getField('fields')).toEqual([{ field: '*' }]); + expect(result.getField('fieldsFromSource')).toBe(undefined); + }); + + test('requests unmapped fields when the flag is provided, using the new fields api', async () => { + const searchSourceMock = createSearchSourceMock({}); + const sampleSize = 250; + const result = updateSearchSource(searchSourceMock, { + indexPattern: indexPatternMock, + services: ({ + data: dataPluginMock.createStartContract(), + uiSettings: ({ + get: (key: string) => { + if (key === SAMPLE_SIZE_SETTING) { + return sampleSize; + } + return false; + }, + } as unknown) as IUiSettingsClient, + } as unknown) as DiscoverServices, + sort: [] as SortOrder[], + columns: [], + useNewFieldsApi: true, + showUnmappedFields: true, + }); + expect(result.getField('index')).toEqual(indexPatternMock); + expect(result.getField('size')).toEqual(sampleSize); + expect(result.getField('fields')).toEqual([{ field: '*', include_unmapped: 'true' }]); + expect(result.getField('fieldsFromSource')).toBe(undefined); + }); + + test('updates a given search source when showUnmappedFields option is set to true', async () => { + const searchSourceMock = createSearchSourceMock({}); + const sampleSize = 250; + const result = updateSearchSource(searchSourceMock, { + indexPattern: indexPatternMock, + services: ({ + data: dataPluginMock.createStartContract(), + uiSettings: ({ + get: (key: string) => { + if (key === SAMPLE_SIZE_SETTING) { + return sampleSize; + } + return false; + }, + } as unknown) as IUiSettingsClient, + } as unknown) as DiscoverServices, + sort: [] as SortOrder[], + columns: [], + useNewFieldsApi: true, + showUnmappedFields: true, + }); + expect(result.getField('index')).toEqual(indexPatternMock); + expect(result.getField('size')).toEqual(sampleSize); + expect(result.getField('fields')).toEqual([{ field: '*', include_unmapped: 'true' }]); expect(result.getField('fieldsFromSource')).toBe(undefined); }); }); diff --git a/src/plugins/discover/public/application/helpers/update_search_source.ts b/src/plugins/discover/public/application/helpers/update_search_source.ts index 34906814ed2d..55d2b05a29b6 100644 --- a/src/plugins/discover/public/application/helpers/update_search_source.ts +++ b/src/plugins/discover/public/application/helpers/update_search_source.ts @@ -23,12 +23,14 @@ export function updateSearchSource( sort, columns, useNewFieldsApi, + showUnmappedFields, }: { indexPattern: IndexPattern; services: DiscoverServices; sort: SortOrder[]; columns: string[]; useNewFieldsApi: boolean; + showUnmappedFields?: boolean; } ) { const { uiSettings, data } = services; @@ -46,7 +48,11 @@ export function updateSearchSource( .setField('filter', data.query.filterManager.getFilters()); if (useNewFieldsApi) { searchSource.removeField('fieldsFromSource'); - searchSource.setField('fields', ['*']); + const fields: Record = { field: '*' }; + if (showUnmappedFields) { + fields.include_unmapped = 'true'; + } + searchSource.setField('fields', [fields]); } else { searchSource.removeField('fields'); const fieldNames = indexPattern.fields.map((field) => field.name); diff --git a/src/plugins/discover/public/saved_searches/_saved_search.ts b/src/plugins/discover/public/saved_searches/_saved_search.ts index bf2caba69cef..d5bd3ea4011b 100644 --- a/src/plugins/discover/public/saved_searches/_saved_search.ts +++ b/src/plugins/discover/public/saved_searches/_saved_search.ts @@ -19,6 +19,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { grid: 'object', sort: 'keyword', version: 'integer', + pre712: 'boolean', }; // Order these fields to the top, the rest are alphabetical public static fieldOrder = ['title', 'description']; @@ -39,6 +40,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { grid: 'object', sort: 'keyword', version: 'integer', + pre712: 'boolean', }, searchSource: true, defaults: { @@ -48,6 +50,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { hits: 0, sort: [], version: 1, + pre712: false, }, }); this.showInRecentlyAccessed = true; diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index 62cdc3324a8e..24fbbcb61cb4 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -23,6 +23,7 @@ export interface SavedSearch { save: (saveOptions: SavedObjectSaveOpts) => Promise; lastSavedTitle?: string; copyOnSave?: boolean; + pre712?: boolean; } export interface SavedSearchLoader { get: (id: string) => Promise; diff --git a/src/plugins/discover/server/saved_objects/search.ts b/src/plugins/discover/server/saved_objects/search.ts index c7d706460aa1..43f107399ac3 100644 --- a/src/plugins/discover/server/saved_objects/search.ts +++ b/src/plugins/discover/server/saved_objects/search.ts @@ -44,6 +44,7 @@ export const searchSavedObjectType: SavedObjectsType = { title: { type: 'text' }, grid: { type: 'object', enabled: false }, version: { type: 'integer' }, + pre712: { type: 'boolean' }, }, }, migrations: searchMigrations as any, diff --git a/src/plugins/discover/server/saved_objects/search_migrations.test.ts b/src/plugins/discover/server/saved_objects/search_migrations.test.ts index fb608c0b6f3e..f1dc228a9ac0 100644 --- a/src/plugins/discover/server/saved_objects/search_migrations.test.ts +++ b/src/plugins/discover/server/saved_objects/search_migrations.test.ts @@ -350,4 +350,41 @@ Object { testMigrateMatchAllQuery(migrationFn); }); }); + + describe('7.12.0', () => { + const migrationFn = searchMigrations['7.12.0']; + + describe('migrateExistingSavedSearch', () => { + it('should add a new flag to existing saved searches', () => { + const migratedDoc = migrationFn( + { + type: 'search', + attributes: { + kibanaSavedObjectMeta: {}, + }, + }, + savedObjectMigrationContext + ); + const migratedPre712Flag = migratedDoc.attributes.pre712; + + expect(migratedPre712Flag).toEqual(true); + }); + + it('should not modify a flag if it already exists', () => { + const migratedDoc = migrationFn( + { + type: 'search', + attributes: { + kibanaSavedObjectMeta: {}, + pre712: false, + }, + }, + savedObjectMigrationContext + ); + const migratedPre712Flag = migratedDoc.attributes.pre712; + + expect(migratedPre712Flag).toEqual(false); + }); + }); + }); }); diff --git a/src/plugins/discover/server/saved_objects/search_migrations.ts b/src/plugins/discover/server/saved_objects/search_migrations.ts index feaf91409797..72749bfd2e9c 100644 --- a/src/plugins/discover/server/saved_objects/search_migrations.ts +++ b/src/plugins/discover/server/saved_objects/search_migrations.ts @@ -117,9 +117,28 @@ const migrateSearchSortToNestedArray: SavedObjectMigrationFn = (doc) = }; }; +const migrateExistingSavedSearch: SavedObjectMigrationFn = (doc) => { + if (!doc.attributes) { + return doc; + } + const pre712 = doc.attributes.pre712; + // pre712 already has a value + if (pre712 !== undefined) { + return doc; + } + return { + ...doc, + attributes: { + ...doc.attributes, + pre712: true, + }, + }; +}; + export const searchMigrations = { '6.7.2': flow(migrateMatchAllQuery), '7.0.0': flow(setNewReferences), '7.4.0': flow(migrateSearchSortToNestedArray), '7.9.3': flow(migrateMatchAllQuery), + '7.12.0': flow(migrateExistingSavedSearch), }; diff --git a/test/functional/apps/discover/_indexpattern_with_unmapped_fields.ts b/test/functional/apps/discover/_indexpattern_with_unmapped_fields.ts new file mode 100644 index 000000000000..bad7afacc124 --- /dev/null +++ b/test/functional/apps/discover/_indexpattern_with_unmapped_fields.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const testSubjects = getService('testSubjects'); + const log = getService('log'); + const retry = getService('retry'); + const PageObjects = getPageObjects(['common', 'timePicker', 'discover']); + + describe('index pattern with unmapped fields', () => { + const unmappedFieldsSwitchSelector = 'unmappedFieldsSwitch'; + + before(async () => { + await esArchiver.loadIfNeeded('unmapped_fields'); + await kibanaServer.uiSettings.replace({ defaultIndex: 'test-index-unmapped-fields' }); + await kibanaServer.uiSettings.update({ + 'discover:searchFieldsFromSource': false, + }); + log.debug('discover'); + const fromTime = 'Jan 20, 2021 @ 00:00:00.000'; + const toTime = 'Jan 25, 2021 @ 00:00:00.000'; + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + }); + + after(async () => { + await esArchiver.unload('unmapped_fields'); + }); + + it('unmapped fields do not exist on a new saved search', async () => { + const expectedHitCount = '4'; + await retry.try(async function () { + expect(await PageObjects.discover.getHitCount()).to.be(expectedHitCount); + }); + const allFields = await PageObjects.discover.getAllFieldNames(); + // message is a mapped field + expect(allFields.includes('message')).to.be(true); + // sender is not a mapped field + expect(allFields.includes('sender')).to.be(false); + }); + + it('unmapped fields toggle does not exist on a new saved search', async () => { + await PageObjects.discover.openSidebarFieldFilter(); + await testSubjects.existOrFail('filterSelectionPanel'); + await testSubjects.missingOrFail('unmappedFieldsSwitch'); + }); + + it('unmapped fields exist on an existing saved search', async () => { + await PageObjects.discover.loadSavedSearch('Existing Saved Search'); + const expectedHitCount = '4'; + await retry.try(async function () { + expect(await PageObjects.discover.getHitCount()).to.be(expectedHitCount); + }); + const allFields = await PageObjects.discover.getAllFieldNames(); + expect(allFields.includes('message')).to.be(true); + expect(allFields.includes('sender')).to.be(true); + expect(allFields.includes('receiver')).to.be(true); + }); + + it('unmapped fields toggle exists on an existing saved search', async () => { + await PageObjects.discover.openSidebarFieldFilter(); + await testSubjects.existOrFail('filterSelectionPanel'); + await testSubjects.existOrFail(unmappedFieldsSwitchSelector); + expect(await testSubjects.isEuiSwitchChecked(unmappedFieldsSwitchSelector)).to.be(true); + }); + + it('switching unmapped fields toggle off hides unmapped fields', async () => { + await testSubjects.setEuiSwitch(unmappedFieldsSwitchSelector, 'uncheck'); + await PageObjects.discover.closeSidebarFieldFilter(); + const allFields = await PageObjects.discover.getAllFieldNames(); + expect(allFields.includes('message')).to.be(true); + expect(allFields.includes('sender')).to.be(false); + expect(allFields.includes('receiver')).to.be(false); + }); + }); +} diff --git a/test/functional/apps/discover/index.ts b/test/functional/apps/discover/index.ts index 1d0d692d5269..5c319312c813 100644 --- a/test/functional/apps/discover/index.ts +++ b/test/functional/apps/discover/index.ts @@ -46,5 +46,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_data_grid_field_data')); loadTestFile(require.resolve('./_data_grid_doc_navigation')); loadTestFile(require.resolve('./_data_grid_doc_table')); + loadTestFile(require.resolve('./_indexpattern_with_unmapped_fields')); }); } diff --git a/test/functional/fixtures/es_archiver/data/data.json.gz b/test/functional/fixtures/es_archiver/data/data.json.gz new file mode 100644 index 000000000000..629276ccd186 Binary files /dev/null and b/test/functional/fixtures/es_archiver/data/data.json.gz differ diff --git a/test/functional/fixtures/es_archiver/data/mappings.json b/test/functional/fixtures/es_archiver/data/mappings.json new file mode 100644 index 000000000000..256978162b98 --- /dev/null +++ b/test/functional/fixtures/es_archiver/data/mappings.json @@ -0,0 +1,450 @@ +{ + "type": "index", + "value": { + "aliases": { + ".kibana": { + } + }, + "index": ".kibana_1", + "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "application_usage_daily": "43b8830d5d0df85a6823d290885fc9fd", + "application_usage_totals": "3d1b76c39bfb2cc8296b024d73854724", + "application_usage_transactional": "3d1b76c39bfb2cc8296b024d73854724", + "config": "c63748b75f39d0c54de12d12c1ccbc20", + "core-usage-stats": "3d1b76c39bfb2cc8296b024d73854724", + "coreMigrationVersion": "2f4316de49999235636386fe51dc06c1", + "dashboard": "40554caf09725935e2c02e02563a2d07", + "index-pattern": "45915a1ad866812242df474eb0479052", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "legacy-url-alias": "3d1b76c39bfb2cc8296b024d73854724", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "namespace": "2f4316de49999235636386fe51dc06c1", + "namespaces": "2f4316de49999235636386fe51dc06c1", + "originId": "2f4316de49999235636386fe51dc06c1", + "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "e5b843b43566421ffa75fb499271dc34", + "search-telemetry": "3d1b76c39bfb2cc8296b024d73854724", + "telemetry": "36a616f7026dfa617d6655df850fe16d", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "type": "2f4316de49999235636386fe51dc06c1", + "ui-counter": "0d409297dc5ebe1e3a1da691c6ee32e3", + "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "url": "c7f66a0df8b1b52f17c28c4adb111105", + "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + } + }, + "dynamic": "strict", + "properties": { + "application_usage_daily": { + "dynamic": "false", + "properties": { + "timestamp": { + "type": "date" + } + } + }, + "application_usage_totals": { + "dynamic": "false", + "type": "object" + }, + "application_usage_transactional": { + "dynamic": "false", + "type": "object" + }, + "config": { + "dynamic": "false", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "core-usage-stats": { + "dynamic": "false", + "type": "object" + }, + "coreMigrationVersion": { + "type": "keyword" + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "index": false, + "type": "text" + } + } + }, + "optionsJSON": { + "index": false, + "type": "text" + }, + "panelsJSON": { + "index": false, + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "pause": { + "doc_values": false, + "index": false, + "type": "boolean" + }, + "section": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "value": { + "doc_values": false, + "index": false, + "type": "integer" + } + } + }, + "timeFrom": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "timeRestore": { + "doc_values": false, + "index": false, + "type": "boolean" + }, + "timeTo": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "index-pattern": { + "dynamic": "false", + "properties": { + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "legacy-url-alias": { + "dynamic": "false", + "type": "object" + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "config": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "namespace": { + "type": "keyword" + }, + "namespaces": { + "type": "keyword" + }, + "originId": { + "type": "keyword" + }, + "query": { + "properties": { + "description": { + "type": "text" + }, + "filters": { + "enabled": false, + "type": "object" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "index": false, + "type": "keyword" + } + } + }, + "timefilter": { + "enabled": false, + "type": "object" + }, + "title": { + "type": "text" + } + } + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "description": { + "type": "text" + }, + "grid": { + "enabled": false, + "type": "object" + }, + "hits": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "index": false, + "type": "text" + } + } + }, + "pre712": { + "type": "boolean" + }, + "sort": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "search-telemetry": { + "dynamic": "false", + "type": "object" + }, + "telemetry": { + "properties": { + "allowChangingOptInStatus": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "lastReported": { + "type": "date" + }, + "lastVersionChecked": { + "type": "keyword" + }, + "reportFailureCount": { + "type": "integer" + }, + "reportFailureVersion": { + "type": "keyword" + }, + "sendUsageFrom": { + "type": "keyword" + }, + "userHasSeenNotice": { + "type": "boolean" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "type": { + "type": "keyword" + }, + "ui-counter": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "updated_at": { + "type": "date" + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "index": false, + "type": "text" + } + } + }, + "savedSearchRefName": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "index": false, + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "index": false, + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file diff --git a/test/functional/fixtures/es_archiver/unmapped_fields/data.json b/test/functional/fixtures/es_archiver/unmapped_fields/data.json new file mode 100644 index 000000000000..10c33280696b --- /dev/null +++ b/test/functional/fixtures/es_archiver/unmapped_fields/data.json @@ -0,0 +1,105 @@ +{ + "type": "doc", + "value": { + "id": "search:cd43f5c2-h761-13f6-9486-733b1ac9221a", + "index": ".kibana", + "source": { + "search": { + "columns": [ + "_source" + ], + "description": "Existing Saved Search", + "hits": 4, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"index\": \"test-index-unmapped-fields\",\n \"highlightAll\": true,\n \"filter\": [],\n \"query\": {\n \"query_string\": {\n \"query\": \"*\",\n \"analyze_wildcard\": true\n }\n }\n}" + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Existing Saved Search", + "version": 1 + }, + "type": "search" + } + } +} + +{ + "type": "doc", + "value": { + "id": "index-pattern:test-index-unmapped-fields", + "index": ".kibana", + "source": { + "index-pattern": { + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":4,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "timeFieldName": "timestamp", + "title": "test-index-unmapped-fields", + "fieldFormatMap": "{\"timestamp\":{\"id\":\"date\"}}" + }, + "type": "index-pattern" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "test-index-unmapped-fields", + "source": { + "timestamp": "2021-01-21T12:00:00.000Z", + "message": "Something bad is coming", + "address": "Elm Street 1" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "test-index-unmapped-fields", + "source": { + "timestamp": "2021-01-22T12:00:00.000Z", + "message": "We have a new case", + "address": "221b Baker Street" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "test-index-unmapped-fields", + "source": { + "timestamp": "2021-01-23T12:00:00.000Z", + "message": "We have a new case", + "address": "221b Baker Street", + "sender": "John Doe", + "receiver": "Sherlock Holmes" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "test-index-unmapped-fields", + "source": { + "timestamp": "2021-01-24T12:00:00.000Z", + "message": "I am coming for you", + "address": "13 Elm Street", + "sender": "Freddy Krueger", + "receiver": "Nancy Thompson" + }, + "type": "_doc" + } +} + diff --git a/test/functional/fixtures/es_archiver/unmapped_fields/mappings.json b/test/functional/fixtures/es_archiver/unmapped_fields/mappings.json new file mode 100644 index 000000000000..f92c2eff4888 --- /dev/null +++ b/test/functional/fixtures/es_archiver/unmapped_fields/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "test-index-unmapped-fields", + "mappings": { + "dynamic": "false", + "properties": { + "timestamp": {"type": "date"}, + "message": { "type": "text" }, + "address": { "type": "text" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}