diff --git a/src/plugins/discover/public/application/angular/context.js b/src/plugins/discover/public/application/angular/context.js index e4128b3e2624..01a28a5c174b 100644 --- a/src/plugins/discover/public/application/angular/context.js +++ b/src/plugins/discover/public/application/angular/context.js @@ -65,6 +65,7 @@ function ContextAppRouteController($routeParams, $scope, $route) { storeInSessionStorage: getServices().uiSettings.get('state:storeInSessionStorage'), history: getServices().history(), toasts: getServices().core.notifications.toasts, + uiSettings: getServices().core.uiSettings, }); this.state = { ...appState.getState() }; this.anchorId = $routeParams.id; diff --git a/src/plugins/discover/public/application/angular/context/api/context.ts b/src/plugins/discover/public/application/angular/context/api/context.ts index d57b54ca1ea1..43f6e83d286b 100644 --- a/src/plugins/discover/public/application/angular/context/api/context.ts +++ b/src/plugins/discover/public/application/angular/context/api/context.ts @@ -64,9 +64,9 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract, useNewFields const searchSource = await createSearchSource(indexPattern, filters); const sortDirToApply = type === 'successors' ? sortDir : reverseSortDir(sortDir); - const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor._source[timeField]) : ''; + const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor.fields[timeField][0]) : ''; const timeValueMillis = - nanos !== '' ? convertIsoToMillis(anchor._source[timeField]) : anchor.sort[0]; + nanos !== '' ? convertIsoToMillis(anchor.fields[timeField][0]) : anchor.sort[0]; const intervals = generateIntervals(LOOKUP_OFFSETS, timeValueMillis, type, sortDir); let documents: EsHitRecordList = []; diff --git a/src/plugins/discover/public/application/angular/context_state.test.ts b/src/plugins/discover/public/application/angular/context_state.test.ts index e7622a13394c..b49a6546a993 100644 --- a/src/plugins/discover/public/application/angular/context_state.test.ts +++ b/src/plugins/discover/public/application/angular/context_state.test.ts @@ -6,10 +6,12 @@ * Side Public License, v 1. */ +import { IUiSettingsClient } from 'kibana/public'; import { getState } from './context_state'; import { createBrowserHistory, History } from 'history'; import { FilterManager, Filter } from '../../../../data/public'; import { coreMock } from '../../../../../core/public/mocks'; +import { SEARCH_FIELDS_FROM_SOURCE } from '../../../common'; const setupMock = coreMock.createSetup(); describe('Test Discover Context State', () => { @@ -23,6 +25,10 @@ describe('Test Discover Context State', () => { defaultStepSize: '4', timeFieldName: 'time', history, + uiSettings: { + get: (key: string) => + ((key === SEARCH_FIELDS_FROM_SOURCE ? true : ['_source']) as unknown) as T, + } as IUiSettingsClient, }); state.startSync(); }); diff --git a/src/plugins/discover/public/application/angular/context_state.ts b/src/plugins/discover/public/application/angular/context_state.ts index d109badbdd16..0bae006ec1f6 100644 --- a/src/plugins/discover/public/application/angular/context_state.ts +++ b/src/plugins/discover/public/application/angular/context_state.ts @@ -8,7 +8,7 @@ import _ from 'lodash'; import { History } from 'history'; -import { NotificationsStart } from 'kibana/public'; +import { NotificationsStart, IUiSettingsClient } from 'kibana/public'; import { createStateContainer, createKbnUrlStateStorage, @@ -17,6 +17,7 @@ import { withNotifyOnErrors, } from '../../../../kibana_utils/public'; import { esFilters, FilterManager, Filter, Query } from '../../../../data/public'; +import { handleSourceColumnState } from './helpers'; export interface AppState { /** @@ -73,6 +74,11 @@ interface GetStateParams { * kbnUrlStateStorage will use it notifying about inner errors */ toasts?: NotificationsStart['toasts']; + + /** + * core ui settings service + */ + uiSettings: IUiSettingsClient; } interface GetStateReturn { @@ -123,6 +129,7 @@ export function getState({ storeInSessionStorage = false, history, toasts, + uiSettings, }: GetStateParams): GetStateReturn { const stateStorage = createKbnUrlStateStorage({ useHash: storeInSessionStorage, @@ -134,7 +141,12 @@ export function getState({ const globalStateContainer = createStateContainer(globalStateInitial); const appStateFromUrl = stateStorage.get(APP_STATE_URL_KEY) as AppState; - const appStateInitial = createInitialAppState(defaultStepSize, timeFieldName, appStateFromUrl); + const appStateInitial = createInitialAppState( + defaultStepSize, + timeFieldName, + appStateFromUrl, + uiSettings + ); const appStateContainer = createStateContainer(appStateInitial); const { start, stop } = syncStates([ @@ -257,7 +269,8 @@ function getFilters(state: AppState | GlobalState): Filter[] { function createInitialAppState( defaultSize: string, timeFieldName: string, - urlState: AppState + urlState: AppState, + uiSettings: IUiSettingsClient ): AppState { const defaultState = { columns: ['_source'], @@ -270,8 +283,11 @@ function createInitialAppState( return defaultState; } - return { - ...defaultState, - ...urlState, - }; + return handleSourceColumnState( + { + ...defaultState, + ...urlState, + }, + uiSettings + ); } diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index c2bbbf2e57a9..78ad40e48fd9 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -110,7 +110,7 @@ app.config(($routeProvider) => { const history = getHistory(); const savedSearchId = $route.current.params.id; return data.indexPatterns.ensureDefaultIndexPattern(history).then(() => { - const { appStateContainer } = getState({ history }); + const { appStateContainer } = getState({ history, uiSettings: config }); const { index } = appStateContainer.getState(); return Promise.props({ ip: loadIndexPattern(index, data.indexPatterns, config), @@ -195,6 +195,7 @@ function discoverController($route, $scope, Promise) { storeInSessionStorage: config.get('state:storeInSessionStorage'), history, toasts: core.notifications.toasts, + uiSettings: config, }); const { diff --git a/src/plugins/discover/public/application/angular/discover_state.test.ts b/src/plugins/discover/public/application/angular/discover_state.test.ts index d3b7ed53e421..e7322a858863 100644 --- a/src/plugins/discover/public/application/angular/discover_state.test.ts +++ b/src/plugins/discover/public/application/angular/discover_state.test.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { IUiSettingsClient } from 'kibana/public'; import { getState, GetStateReturn, @@ -14,11 +15,17 @@ import { import { createBrowserHistory, History } from 'history'; import { dataPluginMock } from '../../../../data/public/mocks'; import { SavedSearch } from '../../saved_searches'; +import { SEARCH_FIELDS_FROM_SOURCE } from '../../../common'; let history: History; let state: GetStateReturn; const getCurrentUrl = () => history.createHref(history.location); +const uiSettingsMock = { + get: (key: string) => + ((key === SEARCH_FIELDS_FROM_SOURCE ? true : ['_source']) as unknown) as T, +} as IUiSettingsClient; + describe('Test discover state', () => { beforeEach(async () => { history = createBrowserHistory(); @@ -26,6 +33,7 @@ describe('Test discover state', () => { state = getState({ getStateDefaults: () => ({ index: 'test' }), history, + uiSettings: uiSettingsMock, }); await state.replaceUrlAppState({}); await state.startSync(); @@ -81,6 +89,7 @@ describe('Test discover state with legacy migration', () => { state = getState({ getStateDefaults: () => ({ index: 'test' }), history, + uiSettings: uiSettingsMock, }); expect(state.appStateContainer.getState()).toMatchInlineSnapshot(` Object { @@ -106,6 +115,7 @@ describe('createSearchSessionRestorationDataProvider', () => { data: mockDataPlugin, appStateContainer: getState({ history: createBrowserHistory(), + uiSettings: uiSettingsMock, }).appStateContainer, getSavedSearch: () => mockSavedSearch, }); diff --git a/src/plugins/discover/public/application/angular/discover_state.ts b/src/plugins/discover/public/application/angular/discover_state.ts index 93fc49b65cbc..e7d5ed469525 100644 --- a/src/plugins/discover/public/application/angular/discover_state.ts +++ b/src/plugins/discover/public/application/angular/discover_state.ts @@ -9,7 +9,7 @@ import { isEqual } from 'lodash'; import { i18n } from '@kbn/i18n'; import { History } from 'history'; -import { NotificationsStart } from 'kibana/public'; +import { NotificationsStart, IUiSettingsClient } from 'kibana/public'; import { createKbnUrlStateStorage, createStateContainer, @@ -30,6 +30,7 @@ import { migrateLegacyQuery } from '../helpers/migrate_legacy_query'; import { DiscoverGridSettings } from '../components/discover_grid/types'; import { DISCOVER_APP_URL_GENERATOR, DiscoverUrlGeneratorState } from '../../url_generator'; import { SavedSearch } from '../../saved_searches'; +import { handleSourceColumnState } from './helpers'; export interface AppState { /** @@ -90,6 +91,11 @@ interface GetStateParams { * kbnUrlStateStorage will use it notifying about inner errors */ toasts?: NotificationsStart['toasts']; + + /** + * core ui settings service + */ + uiSettings: IUiSettingsClient; } export interface GetStateReturn { @@ -149,6 +155,7 @@ export function getState({ storeInSessionStorage = false, history, toasts, + uiSettings, }: GetStateParams): GetStateReturn { const defaultAppState = getStateDefaults ? getStateDefaults() : {}; const stateStorage = createKbnUrlStateStorage({ @@ -163,10 +170,14 @@ export function getState({ appStateFromUrl.query = migrateLegacyQuery(appStateFromUrl.query); } - let initialAppState = { - ...defaultAppState, - ...appStateFromUrl, - }; + let initialAppState = handleSourceColumnState( + { + ...defaultAppState, + ...appStateFromUrl, + }, + uiSettings + ); + // todo filter source depending on fields fetchinbg flag (if no columns remain and source fetching is enabled, use default columns) let previousAppState: AppState; const appStateContainer = createStateContainer(initialAppState); diff --git a/src/plugins/discover/public/application/angular/helpers/index.ts b/src/plugins/discover/public/application/angular/helpers/index.ts index a4ab9e575e42..3d4893268fde 100644 --- a/src/plugins/discover/public/application/angular/helpers/index.ts +++ b/src/plugins/discover/public/application/angular/helpers/index.ts @@ -8,3 +8,4 @@ export { buildPointSeriesData } from './point_series'; export { formatRow } from './row_formatter'; +export { handleSourceColumnState } from './state_helpers'; diff --git a/src/plugins/discover/public/application/angular/helpers/state_helpers.ts b/src/plugins/discover/public/application/angular/helpers/state_helpers.ts new file mode 100644 index 000000000000..ec5009dcf483 --- /dev/null +++ b/src/plugins/discover/public/application/angular/helpers/state_helpers.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IUiSettingsClient } from 'src/core/public'; +import { SEARCH_FIELDS_FROM_SOURCE, DEFAULT_COLUMNS_SETTING } from '../../../../common'; + +/** + * Makes sure the current state is not referencing the source column when using the fields api + * @param state + * @param uiSettings + */ +export function handleSourceColumnState( + state: TState, + uiSettings: IUiSettingsClient +): TState { + if (!state.columns) { + return state; + } + const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); + const defaultColumns = uiSettings.get(DEFAULT_COLUMNS_SETTING); + if (useNewFieldsApi) { + // if fields API is used, filter out the source column + return { + ...state, + columns: state.columns.filter((column) => column !== '_source'), + }; + } else if (state.columns.length === 0) { + // if _source fetching is used and there are no column, switch back to default columns + // this can happen if the fields API was previously used + return { + ...state, + columns: [...defaultColumns], + }; + } + + return state; +} diff --git a/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx b/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx index a4e59a50a2f6..fcaea63f2a77 100644 --- a/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx @@ -315,7 +315,8 @@ export const DiscoverGrid = ({ = { [SEARCH_FIELDS_FROM_SOURCE]: { name: 'Read fields from _source', description: `When enabled will load documents directly from \`_source\`. This is soon going to be deprecated. When disabled, will retrieve fields via the new Fields API in the high-level search service.`, - value: true, + value: false, category: ['discover'], schema: schema.boolean(), }, diff --git a/test/functional/apps/context/index.js b/test/functional/apps/context/index.js index bc6b892d8aa0..245f88a337dc 100644 --- a/test/functional/apps/context/index.js +++ b/test/functional/apps/context/index.js @@ -32,5 +32,6 @@ export default function ({ getService, getPageObjects, loadTestFile }) { loadTestFile(require.resolve('./_filters')); loadTestFile(require.resolve('./_size')); loadTestFile(require.resolve('./_date_nanos')); + loadTestFile(require.resolve('./_date_nanos_custom_timestamp')); }); } diff --git a/test/functional/apps/discover/_shared_links.ts b/test/functional/apps/discover/_shared_links.ts index 076a53a618c4..2893102367b0 100644 --- a/test/functional/apps/discover/_shared_links.ts +++ b/test/functional/apps/discover/_shared_links.ts @@ -77,7 +77,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { '/app/discover?_t=1453775307251#' + '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time' + ":(from:'2015-09-19T06:31:44.000Z',to:'2015-09" + - "-23T18:31:44.000Z'))&_a=(columns:!(_source),filters:!(),index:'logstash-" + + "-23T18:31:44.000Z'))&_a=(columns:!(),filters:!(),index:'logstash-" + "*',interval:auto,query:(language:kuery,query:'')" + ",sort:!(!('@timestamp',desc)))"; const actualUrl = await PageObjects.share.getSharedUrl(); diff --git a/test/functional/fixtures/es_archiver/date_nanos_custom/data.json b/test/functional/fixtures/es_archiver/date_nanos_custom/data.json new file mode 100644 index 000000000000..73cba70a8b93 --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_custom/data.json @@ -0,0 +1,56 @@ +{ + "type": "doc", + "value": { + "id": "index-pattern:date_nanos_custom_timestamp", + "index": ".kibana", + "source": { + "index-pattern": { + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"test\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"test.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"test\"}}},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date_nanos\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "timeFieldName": "timestamp", + "title": "date_nanos_custom_timestamp" + }, + "references": [ + ], + "type": "index-pattern", + "updated_at": "2020-01-09T21:43:20.283Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "date_nanos_custom_timestamp", + "source": { + "test": "1", + "timestamp": "2019-10-21 00:30:04.828740" + } + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "date_nanos_custom_timestamp", + "source": { + "test": "1", + "timestamp": "2019-10-21 08:30:04.828733" + } + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "date_nanos_custom_timestamp", + "source": { + "test": "1", + "timestamp": "2019-10-21 00:30:04.828723" + } + } +} + + diff --git a/test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json b/test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json new file mode 100644 index 000000000000..98af509c4e6f --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json @@ -0,0 +1,31 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "date_nanos_custom_timestamp", + "mappings": { + "properties": { + "test": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "timestamp": { + "format": "yyyy-MM-dd HH:mm:ss.SSSSSS", + "type": "date_nanos" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}