[Index Management] Add data streams functionality to indices tab (#67940)

* First iteration of data streams in index management

- Added the data streams column
- Moved state down to component (using withRouter)
- Removed previous middleware for syncing url hash param
- call history.push to update the url in the component

* Updated jest tests

* refactor: includeHidden -> includeHiddenIndices

* Fix types

* Fix jest test and remove getting filter param from parent

* Small refactor to read url params in render function

* Clean up old data streams code

* Fix sorting on data stream field in table

* dataStream -> data_stream

* qs > * as qs

x-pack/plugins/index_management/__jest__/client_integration/helpers/index.ts

@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import './mocks';
+
 export { nextTick, getRandomString, findTestSubject, TestBed } from '../../../../../test_utils';
 
 export { setupEnvironment, WithAppDependencies, services } from './setup_environment';

x-pack/plugins/index_management/__jest__/client_integration/helpers/mocks.ts

@@ -4,4 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { syncUrlHashQueryParam } from './sync_url_hash_query_param.js';
+(window as any).Worker = class Worker {
+  onmessage() {}
+  postMessage() {}
+};

x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.helpers.ts

@@ -24,7 +24,7 @@ import { WithAppDependencies, services, TestSubjects } from '../helpers';
 const testBedConfig: TestBedConfig = {
   store: () => indexManagementStore(services as any),
   memoryRouter: {
-    initialEntries: [`/indices?includeHidden=true`],
+    initialEntries: [`/indices`],
     componentRoutePath: `/:section(indices|templates)`,
   },
   doMountAsync: true,

x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.helpers.ts

@@ -14,7 +14,7 @@ import { WithAppDependencies, services, TestSubjects } from '../helpers';
 const testBedConfig: TestBedConfig = {
   store: () => indexManagementStore(services as any),
   memoryRouter: {
-    initialEntries: [`/indices?includeHidden=true`],
+    initialEntries: [`/indices?includeHiddenIndices=true`],
     componentRoutePath: `/:section(indices|templates)`,
   },
   doMountAsync: true,

x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.ts

@@ -33,16 +33,14 @@ describe('<IndexManagementHome />', () => {
       });
     });
 
-    test('sets the hash query param base on include hidden indices toggle', () => {
+    test('toggles the include hidden button through URL hash correctly', () => {
       const { actions } = testBed;
       expect(actions.getIncludeHiddenIndicesToggleStatus()).toBe(true);
-      expect(window.location.hash.includes('includeHidden=true')).toBe(true);
       actions.clickIncludeHiddenIndicesToggle();
-      expect(window.location.hash.includes('includeHidden=true')).toBe(false);
+      expect(actions.getIncludeHiddenIndicesToggleStatus()).toBe(false);
       // Note: this test modifies the shared location.hash state, we put it back the way it was
       actions.clickIncludeHiddenIndicesToggle();
       expect(actions.getIncludeHiddenIndicesToggleStatus()).toBe(true);
-      expect(window.location.hash.includes('includeHidden=true')).toBe(true);
     });
   });
 

x-pack/plugins/index_management/public/application/sections/home/index_list/index_list.js

@@ -8,15 +8,10 @@ import React from 'react';
 import { DetailPanel } from './detail_panel';
 import { IndexTable } from './index_table';
 
-export function IndexList({
-  match: {
-    params: { filter },
-  },
-  location,
-}) {
+export function IndexList() {
   return (
     <div className="im-snapshotTestSubject" data-test-subj="indicesList">
-      <IndexTable filterFromURI={filter} location={location} />
+      <IndexTable />
       <DetailPanel />
     </div>
   );

x-pack/plugins/index_management/public/application/sections/home/index_list/index_table/index_table.container.js

@@ -5,13 +5,13 @@
  */
 
 import { connect } from 'react-redux';
+import { withRouter } from 'react-router-dom';
 import {
   getDetailPanelIndexName,
   getPageOfIndices,
   getPager,
   getFilter,
   isDetailPanelOpen,
-  showHiddenIndices,
   getSortField,
   isSortAscending,
   getIndicesAsArray,
@@ -26,7 +26,6 @@ import {
   pageChanged,
   pageSizeChanged,
   sortChanged,
-  showHiddenIndicesChanged,
   loadIndices,
   reloadIndices,
   toggleChanged,
@@ -34,15 +33,14 @@ import {
 
 import { IndexTable as PresentationComponent } from './index_table';
 
-const mapStateToProps = (state) => {
+const mapStateToProps = (state, props) => {
   return {
     allIndices: getIndicesAsArray(state),
     isDetailPanelOpen: isDetailPanelOpen(state),
     detailPanelIndexName: getDetailPanelIndexName(state),
-    indices: getPageOfIndices(state),
-    pager: getPager(state),
+    indices: getPageOfIndices(state, props),
+    pager: getPager(state, props),
     filter: getFilter(state),
-    showHiddenIndices: showHiddenIndices(state),
     sortField: getSortField(state),
     isSortAscending: isSortAscending(state),
     indicesLoading: indicesLoading(state),
@@ -65,9 +63,6 @@ const mapDispatchToProps = (dispatch) => {
     sortChanged: (sortField, isSortAscending) => {
       dispatch(sortChanged({ sortField, isSortAscending }));
     },
-    showHiddenIndicesChanged: (showHiddenIndices) => {
-      dispatch(showHiddenIndicesChanged({ showHiddenIndices }));
-    },
     toggleChanged: (toggleName, toggleValue) => {
       dispatch(toggleChanged({ toggleName, toggleValue }));
     },
@@ -80,10 +75,12 @@ const mapDispatchToProps = (dispatch) => {
     loadIndices: () => {
       dispatch(loadIndices());
     },
-    reloadIndices: () => {
-      dispatch(reloadIndices());
+    reloadIndices: (indexNames) => {
+      dispatch(reloadIndices(indexNames));
     },
   };
 };
 
-export const IndexTable = connect(mapStateToProps, mapDispatchToProps)(PresentationComponent);
+export const IndexTable = withRouter(
+  connect(mapStateToProps, mapDispatchToProps)(PresentationComponent)
+);

x-pack/plugins/index_management/public/application/sections/home/index_list/index_table/index_table.js

@@ -8,7 +8,7 @@ import React, { Component, Fragment } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { Route } from 'react-router-dom';
-import { parse } from 'query-string';
+import qs from 'query-string';
 
 import {
   EuiButton,
@@ -66,6 +66,9 @@ const HEADERS = {
   size: i18n.translate('xpack.idxMgmt.indexTable.headers.storageSizeHeader', {
     defaultMessage: 'Storage size',
   }),
+  data_stream: i18n.translate('xpack.idxMgmt.indexTable.headers.dataStreamHeader', {
+    defaultMessage: 'Data stream',
+  }),
 };
 
 export class IndexTable extends Component {
@@ -97,17 +100,14 @@ export class IndexTable extends Component {
 
   componentDidMount() {
     this.props.loadIndices();
-    this.interval = setInterval(this.props.reloadIndices, REFRESH_RATE_INDEX_LIST);
-    const {
-      filterChanged,
-      filterFromURI,
-      showHiddenIndicesChanged,
-      showHiddenIndices,
-      location,
-    } = this.props;
-
-    if (filterFromURI) {
-      const decodedFilter = decodeURIComponent(filterFromURI);
+    this.interval = setInterval(
+      () => this.props.reloadIndices(this.props.indices.map((i) => i.name)),
+      REFRESH_RATE_INDEX_LIST
+    );
+    const { location, filterChanged } = this.props;
+    const { filter } = qs.parse((location && location.search) || '');
+    if (filter) {
+      const decodedFilter = decodeURIComponent(filter);
 
       try {
         const filter = EuiSearchBar.Query.parse(decodedFilter);
@@ -116,17 +116,30 @@ export class IndexTable extends Component {
         this.setState({ filterError: e });
       }
     }
-
-    // Check if the we have the includeHidden query param
-    const { includeHidden } = parse((location && location.search) || '');
-    const nextValue = includeHidden === 'true';
-    if (nextValue !== showHiddenIndices) {
-      showHiddenIndicesChanged(nextValue);
-    }
   }
   componentWillUnmount() {
     clearInterval(this.interval);
   }
+
+  readURLParams() {
+    const { location } = this.props;
+    const { includeHiddenIndices } = qs.parse((location && location.search) || '');
+    return {
+      includeHiddenIndices: includeHiddenIndices === 'true',
+    };
+  }
+
+  setIncludeHiddenParam(hidden) {
+    const { pathname, search } = this.props.location;
+    const params = qs.parse(search);
+    if (hidden) {
+      params.includeHiddenIndices = 'true';
+    } else {
+      delete params.includeHiddenIndices;
+    }
+    this.props.history.push(pathname + '?' + qs.stringify(params));
+  }
+
   onSort = (column) => {
     const { sortField, isSortAscending, sortChanged } = this.props;
 
@@ -416,8 +429,6 @@ export class IndexTable extends Component {
   render() {
     const {
       filter,
-      showHiddenIndices,
-      showHiddenIndicesChanged,
       indices,
       loadIndices,
       indicesLoading,
@@ -426,6 +437,8 @@ export class IndexTable extends Component {
       pager,
     } = this.props;
 
+    const { includeHiddenIndices } = this.readURLParams();
+
     let emptyState;
 
     if (indicesLoading) {
@@ -477,8 +490,8 @@ export class IndexTable extends Component {
                         <EuiSwitch
                           id="checkboxShowHiddenIndices"
                           data-test-subj="indexTableIncludeHiddenIndicesToggle"
-                          checked={showHiddenIndices}
-                          onChange={(event) => showHiddenIndicesChanged(event.target.checked)}
+                          checked={includeHiddenIndices}
+                          onChange={(event) => this.setIncludeHiddenParam(event.target.checked)}
                           label={
                             <FormattedMessage
                               id="xpack.idxMgmt.indexTable.hiddenIndicesSwitchLabel"

x-pack/plugins/index_management/public/application/services/sort_table.ts

@@ -14,7 +14,8 @@ type SortField =
   | 'replica'
   | 'documents'
   | 'size'
-  | 'primary_size';
+  | 'primary_size'
+  | 'data_stream';
 
 type Unit = 'kb' | 'mb' | 'gb' | 'tb' | 'pb';
 
@@ -55,6 +56,7 @@ const sorters = {
   documents: numericSort('documents'),
   size: byteSort('size'),
   primary_size: byteSort('primary_size'),
+  data_stream: stringSort('data_stream'),
 };
 
 export const sortTable = (array = [], sortField: SortField, isSortAscending: boolean) => {

x-pack/plugins/index_management/public/application/store/actions/reload_indices.js

@@ -6,15 +6,13 @@
 
 import { createAction } from 'redux-actions';
 import { i18n } from '@kbn/i18n';
-import { getIndexNamesForCurrentPage } from '../selectors';
 import { reloadIndices as request } from '../../services';
 import { loadIndices } from './load_indices';
 import { notificationService } from '../../services/notification';
 
 export const reloadIndicesSuccess = createAction('INDEX_MANAGEMENT_RELOAD_INDICES_SUCCESS');
-export const reloadIndices = (indexNames) => async (dispatch, getState) => {
+export const reloadIndices = (indexNames) => async (dispatch) => {
   let indices;
-  indexNames = indexNames || getIndexNamesForCurrentPage(getState());
   try {
     indices = await request(indexNames);
   } catch (error) {

x-pack/plugins/index_management/public/application/store/actions/table_state.js

@@ -17,8 +17,4 @@ export const pageSizeChanged = createAction('INDEX_MANAGEMENT_PAGE_SIZE_CHANGED'
 
 export const sortChanged = createAction('INDEX_MANAGEMENT_SORT_CHANGED');
 
-export const showHiddenIndicesChanged = createAction(
-  'INDEX_MANAGEMENT_SHOW_HIDDEN_INDICES_CHANGED'
-);
-
 export const toggleChanged = createAction('INDEX_MANAGEMENT_TOGGLE_CHANGED');

x-pack/plugins/index_management/public/application/store/middlewares/sync_url_hash_query_param.js.ts

@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-import * as q from 'query-string';
-import { Middleware } from 'redux';
-// @ts-ignore
-import { showHiddenIndicesChanged } from '../actions';
-
-export const syncUrlHashQueryParam: Middleware = () => (next) => (action) => {
-  if (action.type === String(showHiddenIndicesChanged)) {
-    const { url, query } = q.parseUrl(window.location.hash);
-    if (action.payload.showHiddenIndices) {
-      query.includeHidden = 'true';
-    } else {
-      delete query.includeHidden;
-    }
-    window.location.hash = url + '?' + q.stringify(query);
-  }
-  next(action);
-};

x-pack/plugins/index_management/public/application/store/reducers/table_state.js

@@ -10,7 +10,6 @@ import {
   pageChanged,
   pageSizeChanged,
   sortChanged,
-  showHiddenIndicesChanged,
   toggleChanged,
 } from '../actions';
 
@@ -20,7 +19,6 @@ export const defaultTableState = {
   currentPage: 0,
   sortField: 'index.name',
   isSortAscending: true,
-  showHiddenIndices: false,
 };
 
 export const tableState = handleActions(
@@ -33,14 +31,6 @@ export const tableState = handleActions(
         currentPage: 0,
       };
     },
-    [showHiddenIndicesChanged](state, action) {
-      const { showHiddenIndices } = action.payload;
-
-      return {
-        ...state,
-        showHiddenIndices,
-      };
-    },
     [toggleChanged](state, action) {
       const { toggleName, toggleValue } = action.payload;
       const toggleNameToVisibleMap = { ...state.toggleNameToVisibleMap };

x-pack/plugins/index_management/public/application/store/selectors/index.d.ts

@@ -6,3 +6,5 @@
 import { ExtensionsService } from '../../../services';
 
 export declare function setExtensionsService(extensionsService: ExtensionsService): any;
+
+export const getFilteredIndices: (state: any, props: any) => any;

x-pack/plugins/index_management/public/application/store/selectors/index.js

@@ -6,6 +6,7 @@
 import { Pager, EuiSearchBar } from '@elastic/eui';
 
 import { createSelector } from 'reselect';
+import * as qs from 'query-string';
 import { indexStatusLabels } from '../../lib/index_status_labels';
 import { sortTable } from '../../services';
 
@@ -35,6 +36,7 @@ export const getIndexByIndexName = (state, name) => getIndices(state)[name];
 export const getFilteredIds = (state) => state.indices.filteredIds;
 export const getRowStatuses = (state) => state.rowStatus;
 export const getTableState = (state) => state.tableState;
+export const getTableLocationProp = (_, props) => props.location;
 export const getAllIds = (state) => state.indices.allIds;
 export const getIndexStatusByIndexName = (state, indexName) => {
   const indices = getIndices(state);
@@ -79,18 +81,24 @@ const filterByToggles = (indices, toggleNameToVisibleMap) => {
     });
   });
 };
-const getFilteredIndices = createSelector(
+
+export const getFilteredIndices = createSelector(
   getIndices,
   getAllIds,
   getTableState,
-  (indices, allIds, tableState) => {
+  getTableLocationProp,
+  (indices, allIds, tableState, tableLocation) => {
     let indexArray = allIds.map((indexName) => indices[indexName]);
     indexArray = filterByToggles(indexArray, tableState.toggleNameToVisibleMap);
-    const systemFilteredIndexes = tableState.showHiddenIndices
+    const { includeHiddenIndices: includeHiddenParam } = qs.parse(tableLocation.search);
+    const includeHidden = includeHiddenParam === 'true';
+    const filteredIndices = includeHidden
       ? indexArray
-      : indexArray.filter((index) => !(index.name + '').startsWith('.') && !index.hidden);
+      : indexArray.filter((index) => {
+          return !(index.name + '').startsWith('.') && !index.hidden;
+        });
     const filter = tableState.filter || EuiSearchBar.Query.MATCH_ALL;
-    return EuiSearchBar.Query.execute(filter, systemFilteredIndexes, {
+    return EuiSearchBar.Query.execute(filter, filteredIndices, {
       defaultFields: defaultFilterFields,
     });
   }
@@ -133,29 +141,8 @@ export const getPageOfIndices = createSelector(
   }
 );
 
-export const getIndexNamesForCurrentPage = createSelector(getPageOfIndices, (pageOfIndices) => {
-  return pageOfIndices.map((index) => index.name);
-});
-
-export const getHasNextPage = createSelector(getPager, (pager) => {
-  return pager.hasNextPage;
-});
-
-export const getHasPreviousPage = createSelector(getPager, (pager) => {
-  return pager.hasPreviousPage;
-});
-
-export const getCurrentPage = createSelector(getPager, (pager) => {
-  return pager.currentPage;
-});
-
 export const getFilter = createSelector(getTableState, ({ filter }) => filter);
 
-export const showHiddenIndices = createSelector(
-  getTableState,
-  ({ showHiddenIndices }) => showHiddenIndices
-);
-
 export const isSortAscending = createSelector(
   getTableState,
   ({ isSortAscending }) => isSortAscending

x-pack/plugins/index_management/public/application/store/selectors/indices_filter.test.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ExtensionsService } from '../../../services';
+import { getFilteredIndices, setExtensionsService } from '.';
+// @ts-ignore
+import { defaultTableState } from '../reducers/table_state';
+
+describe('getFilteredIndices selector', () => {
+  let extensionService: ExtensionsService;
+  beforeAll(() => {
+    extensionService = new ExtensionsService();
+    extensionService.setup();
+    setExtensionsService(extensionService);
+  });
+
+  const state = {
+    tableState: { ...defaultTableState },
+    indices: {
+      byId: {
+        test: { name: 'index1', hidden: true },
+        anotherTest: { name: 'index2', hidden: false },
+        aTest: { name: 'index3' },
+        aFinalTest: { name: '.index4' },
+      },
+      allIds: ['test', 'anotherTest', 'aTest', 'aFinalTest'],
+    },
+  };
+
+  it('filters out hidden indices', () => {
+    expect(getFilteredIndices(state, { location: { search: '' } })).toEqual([
+      { name: 'index2', hidden: false },
+      { name: 'index3' },
+    ]);
+  });
+
+  it('includes hidden indices', () => {
+    expect(
+      getFilteredIndices(state, { location: { search: '?includeHiddenIndices=true' } })
+    ).toEqual([
+      { name: 'index1', hidden: true },
+      { name: 'index2', hidden: false },
+      { name: 'index3' },
+      { name: '.index4' },
+    ]);
+  });
+});

x-pack/plugins/index_management/public/application/store/store.js

@@ -9,7 +9,6 @@ import thunk from 'redux-thunk';
 import { defaultTableState } from './reducers/table_state';
 
 import { getReducer } from './reducers/';
-import { syncUrlHashQueryParam } from './middlewares';
 
 export function indexManagementStore(services) {
   const toggleNameToVisibleMap = {};
@@ -17,7 +16,7 @@ export function indexManagementStore(services) {
     toggleNameToVisibleMap[toggleExtension.name] = false;
   });
   const initialState = { tableState: { ...defaultTableState, toggleNameToVisibleMap } };
-  const enhancers = [applyMiddleware(thunk, syncUrlHashQueryParam)];
+  const enhancers = [applyMiddleware(thunk)];
 
   window.__REDUX_DEVTOOLS_EXTENSION__ && enhancers.push(window.__REDUX_DEVTOOLS_EXTENSION__());
   return createStore(getReducer(services), initialState, compose(...enhancers));

x-pack/plugins/index_management/server/lib/fetch_indices.ts

@@ -23,6 +23,7 @@ interface Hit {
 interface IndexInfo {
   aliases: { [aliasName: string]: unknown };
   mappings: unknown;
+  data_stream?: string;
   settings: {
     index: {
       hidden: 'true' | 'false';
@@ -87,6 +88,7 @@ async function fetchIndicesCall(
       isFrozen: hit.sth === 'true', // sth value coming back as a string from ES
       aliases: aliases.length ? aliases : 'none',
       hidden: index.settings.index.hidden === 'true',
+      data_stream: index.data_stream,
     };
   });
 }

x-pack/plugins/index_management/server/types.ts

@@ -32,6 +32,7 @@ export interface Index {
   size: any;
   isFrozen: boolean;
   aliases: string | string[];
+  data_stream?: string;
   [key: string]: any;
 }