[Ingest] Data streams list page (#64134)

* Clean up fleet setup request/response typings

* Add data stream model and list route, handler, and request/response types

* Initial pass at data streams list

* Table styling fixes

* Fix types, fix field names

* Change forEach to map

x-pack/plugins/ingest_manager/common/constants/routes.ts

@@ -5,9 +5,10 @@
  */
 // Base API paths
 export const API_ROOT = `/api/ingest_manager`;
+export const EPM_API_ROOT = `${API_ROOT}/epm`;
+export const DATA_STREAM_API_ROOT = `${API_ROOT}/data_streams`;
 export const DATASOURCE_API_ROOT = `${API_ROOT}/datasources`;
 export const AGENT_CONFIG_API_ROOT = `${API_ROOT}/agent_configs`;
-export const EPM_API_ROOT = `${API_ROOT}/epm`;
 export const FLEET_API_ROOT = `${API_ROOT}/fleet`;
 
 // EPM API routes
@@ -23,6 +24,11 @@ export const EPM_API_ROUTES = {
   CATEGORIES_PATTERN: `${EPM_API_ROOT}/categories`,
 };
 
+// Data stream API routes
+export const DATA_STREAM_API_ROUTES = {
+  LIST_PATTERN: `${DATA_STREAM_API_ROOT}`,
+};
+
 // Datasource API routes
 export const DATASOURCE_API_ROUTES = {
   LIST_PATTERN: `${DATASOURCE_API_ROOT}`,

x-pack/plugins/ingest_manager/common/services/routes.ts

@@ -8,6 +8,7 @@ import {
   EPM_API_ROUTES,
   DATASOURCE_API_ROUTES,
   AGENT_CONFIG_API_ROUTES,
+  DATA_STREAM_API_ROUTES,
   FLEET_SETUP_API_ROUTES,
   AGENT_API_ROUTES,
   ENROLLMENT_API_KEY_ROUTES,
@@ -88,6 +89,12 @@ export const agentConfigRouteService = {
   },
 };
 
+export const dataStreamRouteService = {
+  getListPath: () => {
+    return DATA_STREAM_API_ROUTES.LIST_PATTERN;
+  },
+};
+
 export const fleetSetupRouteService = {
   getFleetSetupPath: () => FLEET_SETUP_API_ROUTES.INFO_PATTERN,
   postFleetSetupPath: () => FLEET_SETUP_API_ROUTES.CREATE_PATTERN,

x-pack/plugins/ingest_manager/common/types/models/data_stream.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface DataStream {
+  index: string;
+  dataset: string;
+  namespace: string;
+  type: string;
+  package: string;
+  last_activity: string;
+  size_in_bytes: number;
+}

x-pack/plugins/ingest_manager/common/types/models/index.ts

@@ -7,6 +7,7 @@
 export * from './agent';
 export * from './agent_config';
 export * from './datasource';
+export * from './data_stream';
 export * from './output';
 export * from './epm';
 export * from './enrollment_api_key';

x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts

@@ -3,11 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { DataStream } from '../models';
 
-export const GetFleetSetupRequestSchema = {};
-
-export const CreateFleetSetupRequestSchema = {};
-
-export interface CreateFleetSetupResponse {
-  isInitialized: boolean;
+export interface GetDataStreamsResponse {
+  data_streams: DataStream[];
 }

x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts

@@ -4,16 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface GetFleetSetupRequest {}
-
-export interface CreateFleetSetupRequest {
-  body: {
-    fleet_enroll_username: string;
-    fleet_enroll_password: string;
-  };
-}
-
 export interface CreateFleetSetupResponse {
   isInitialized: boolean;
 }

x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts

@@ -5,6 +5,7 @@
  */
 export * from './common';
 export * from './datasource';
+export * from './data_stream';
 export * from './agent';
 export * from './agent_config';
 export * from './fleet_setup';

x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts

@@ -12,6 +12,7 @@ export const EPM_LIST_INSTALLED_PACKAGES_PATH = `${EPM_PATH}/installed`;
 export const EPM_DETAIL_VIEW_PATH = `${EPM_PATH}/detail/:pkgkey/:panel?`;
 export const AGENT_CONFIG_PATH = '/configs';
 export const AGENT_CONFIG_DETAILS_PATH = `${AGENT_CONFIG_PATH}/`;
+export const DATA_STREAM_PATH = '/data-streams';
 export const FLEET_PATH = '/fleet';
 export const FLEET_AGENTS_PATH = `${FLEET_PATH}/agents`;
 export const FLEET_AGENT_DETAIL_PATH = `${FLEET_AGENTS_PATH}/`;

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { useRequest } from './use_request';
+import { dataStreamRouteService } from '../../services';
+import { GetDataStreamsResponse } from '../../types';
+
+export const useGetDataStreams = () => {
+  return useRequest<GetDataStreamsResponse>({
+    path: dataStreamRouteService.getListPath(),
+    method: 'get',
+  });
+};

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts

@@ -6,6 +6,7 @@
 export { setHttpClient, sendRequest, useRequest } from './use_request';
 export * from './agent_config';
 export * from './datasource';
+export * from './data_stream';
 export * from './agents';
 export * from './enrollment_api_keys';
 export * from './epm';

x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx

@@ -16,10 +16,10 @@ import {
   IngestManagerConfigType,
   IngestManagerStartDeps,
 } from '../../plugin';
-import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from './constants';
+import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from './constants';
 import { DefaultLayout, WithoutHeaderLayout } from './layouts';
 import { Loading, Error } from './components';
-import { IngestManagerOverview, EPMApp, AgentConfigApp, FleetApp } from './sections';
+import { IngestManagerOverview, EPMApp, AgentConfigApp, FleetApp, DataStreamApp } from './sections';
 import { CoreContext, DepsContext, ConfigContext, setHttpClient, useConfig } from './hooks';
 import { PackageInstallProvider } from './sections/epm/hooks';
 import { sendSetup } from './hooks/use_request/setup';
@@ -98,6 +98,11 @@ const IngestManagerRoutes = ({ ...rest }) => {
               <AgentConfigApp />
             </DefaultLayout>
           </Route>
+          <Route path={DATA_STREAM_PATH}>
+            <DefaultLayout section="data_stream">
+              <DataStreamApp />
+            </DefaultLayout>
+          </Route>
           <ProtectedRoute path={FLEET_PATH} isAllowed={fleet.enabled}>
             <DefaultLayout section="fleet">
               <FleetApp />

x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx

@@ -10,7 +10,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { Section } from '../sections';
 import { AlphaMessaging } from '../components';
 import { useLink, useConfig } from '../hooks';
-import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from '../constants';
+import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from '../constants';
 
 interface Props {
   section?: Section;
@@ -76,6 +76,12 @@ export const DefaultLayout: React.FunctionComponent<Props> = ({ section, childre
                   defaultMessage="Fleet"
                 />
               </EuiTab>
+              <EuiTab isSelected={section === 'data_stream'} href={useLink(DATA_STREAM_PATH)}>
+                <FormattedMessage
+                  id="xpack.ingestManager.appNavigation.dataStreamsLinkText"
+                  defaultMessage="Data streams"
+                />
+              </EuiTab>
             </EuiTabs>
           </EuiFlexItem>
         </EuiFlexGroup>

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx

@@ -191,7 +191,6 @@ export const AgentConfigListPage: React.FunctionComponent<{}> = () => {
           defaultMessage: 'Name',
         }),
         width: '20%',
-        // FIXME: use version once available - see: https://github.com/elastic/kibana/issues/56750
         render: (name: string, agentConfig: AgentConfig) => (
           <EuiFlexGroup gutterSize="s" alignItems="baseline" style={{ minWidth: 0 }}>
             <EuiFlexItem grow={false} style={NO_WRAP_TRUNCATE_STYLE}>

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+import { HashRouter as Router, Route, Switch } from 'react-router-dom';
+import { DataStreamListPage } from './list_page';
+
+export const DataStreamApp: React.FunctionComponent = () => {
+  return (
+    <Router>
+      <Switch>
+        <Route path="/data-streams">
+          <DataStreamListPage />
+        </Route>
+      </Switch>
+    </Router>
+  );
+};

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx

@@ -0,0 +1,283 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React, { useMemo } from 'react';
+import {
+  EuiBadge,
+  EuiButton,
+  EuiText,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiEmptyPrompt,
+  EuiInMemoryTable,
+  EuiTableActionsColumnType,
+  EuiTableFieldDataColumnType,
+} from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage, FormattedDate } from '@kbn/i18n/react';
+import { DataStream } from '../../../types';
+import { WithHeaderLayout } from '../../../layouts';
+import { useGetDataStreams, useStartDeps, usePagination } from '../../../hooks';
+
+const DataStreamListPageLayout: React.FunctionComponent = ({ children }) => (
+  <WithHeaderLayout
+    leftColumn={
+      <EuiFlexGroup direction="column" gutterSize="m">
+        <EuiFlexItem>
+          <EuiText>
+            <h1>
+              <FormattedMessage
+                id="xpack.ingestManager.dataStreamList.pageTitle"
+                defaultMessage="Data streams"
+              />
+            </h1>
+          </EuiText>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiText color="subdued">
+            <p>
+              <FormattedMessage
+                id="xpack.ingestManager.dataStreamList.pageSubtitle"
+                defaultMessage="Manage the data created by your agents."
+              />
+            </p>
+          </EuiText>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    }
+  >
+    {children}
+  </WithHeaderLayout>
+);
+
+export const DataStreamListPage: React.FunctionComponent<{}> = () => {
+  const {
+    data: { fieldFormats },
+  } = useStartDeps();
+
+  const { pagination, pageSizeOptions } = usePagination();
+
+  // Fetch agent configs
+  const { isLoading, data: dataStreamsData, sendRequest } = useGetDataStreams();
+
+  // Some configs retrieved, set up table props
+  const columns = useMemo(() => {
+    const cols: Array<
+      EuiTableFieldDataColumnType<DataStream> | EuiTableActionsColumnType<DataStream>
+    > = [
+      {
+        field: 'dataset',
+        sortable: true,
+        width: '25%',
+        truncateText: true,
+        name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', {
+          defaultMessage: 'Dataset',
+        }),
+      },
+      {
+        field: 'type',
+        sortable: true,
+        truncateText: true,
+        name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', {
+          defaultMessage: 'Type',
+        }),
+      },
+      {
+        field: 'namespace',
+        sortable: true,
+        truncateText: true,
+        name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', {
+          defaultMessage: 'Namespace',
+        }),
+        render: (namespace: string) => {
+          return namespace ? <EuiBadge color="hollow">{namespace}</EuiBadge> : '';
+        },
+      },
+      {
+        field: 'package',
+        sortable: true,
+        truncateText: true,
+        name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', {
+          defaultMessage: 'Integration',
+        }),
+      },
+      {
+        field: 'last_activity',
+        sortable: true,
+        width: '25%',
+        dataType: 'date',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.lastActivityColumnTitle', {
+          defaultMessage: 'Last activity',
+        }),
+        render: (date: DataStream['last_activity']) => {
+          try {
+            const formatter = fieldFormats.getInstance('date');
+            return formatter.convert(date);
+          } catch (e) {
+            return <FormattedDate value={date} year="numeric" month="short" day="2-digit" />;
+          }
+        },
+      },
+      {
+        field: 'size_in_bytes',
+        sortable: true,
+        name: i18n.translate('xpack.ingestManager.dataStreamList.sizeColumnTitle', {
+          defaultMessage: 'Size',
+        }),
+        render: (size: DataStream['size_in_bytes']) => {
+          try {
+            const formatter = fieldFormats.getInstance('bytes');
+            return formatter.convert(size);
+          } catch (e) {
+            return `${size}b`;
+          }
+        },
+      },
+    ];
+    return cols;
+  }, [fieldFormats]);
+
+  const emptyPrompt = useMemo(
+    () => (
+      <EuiEmptyPrompt
+        title={
+          <h2>
+            <FormattedMessage
+              id="xpack.ingestManager.dataStreamList.noDataStreamsPrompt"
+              defaultMessage="No data streams"
+            />
+          </h2>
+        }
+      />
+    ),
+    []
+  );
+
+  const filterOptions: { [key: string]: string[] } = {
+    dataset: [],
+    type: [],
+    namespace: [],
+    package: [],
+  };
+
+  if (dataStreamsData && dataStreamsData.data_streams.length) {
+    dataStreamsData.data_streams.forEach(stream => {
+      const { dataset, type, namespace, package: pkg } = stream;
+      if (!filterOptions.dataset.includes(dataset)) {
+        filterOptions.dataset.push(dataset);
+      }
+      if (!filterOptions.type.includes(type)) {
+        filterOptions.type.push(type);
+      }
+      if (!filterOptions.namespace.includes(namespace)) {
+        filterOptions.namespace.push(namespace);
+      }
+      if (!filterOptions.package.includes(pkg)) {
+        filterOptions.package.push(pkg);
+      }
+    });
+  }
+
+  return (
+    <DataStreamListPageLayout>
+      <EuiInMemoryTable
+        loading={isLoading}
+        hasActions={true}
+        message={
+          isLoading ? (
+            <FormattedMessage
+              id="xpack.ingestManager.dataStreamList.loadingDataStreamsMessage"
+              defaultMessage="Loading data streams…"
+            />
+          ) : dataStreamsData && !dataStreamsData.data_streams.length ? (
+            emptyPrompt
+          ) : (
+            <FormattedMessage
+              id="xpack.ingestManager.dataStreamList.noFilteredDataStreamsMessage"
+              defaultMessage="No matching data streams found"
+            />
+          )
+        }
+        items={dataStreamsData ? dataStreamsData.data_streams : []}
+        itemId="index"
+        columns={columns}
+        pagination={{
+          initialPageSize: pagination.pageSize,
+          pageSizeOptions,
+        }}
+        sorting={true}
+        search={{
+          toolsRight: [
+            <EuiButton color="primary" iconType="refresh" onClick={() => sendRequest()}>
+              <FormattedMessage
+                id="xpack.ingestManager.dataStreamList.reloadDataStreamsButtonText"
+                defaultMessage="Reload"
+              />
+            </EuiButton>,
+          ],
+          box: {
+            placeholder: i18n.translate(
+              'xpack.ingestManager.dataStreamList.searchPlaceholderTitle',
+              {
+                defaultMessage: 'Filter data streams',
+              }
+            ),
+            incremental: true,
+          },
+          filters: [
+            {
+              type: 'field_value_selection',
+              field: 'dataset',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', {
+                defaultMessage: 'Dataset',
+              }),
+              multiSelect: 'or',
+              options: filterOptions.dataset.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+            {
+              type: 'field_value_selection',
+              field: 'type',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', {
+                defaultMessage: 'Type',
+              }),
+              multiSelect: 'or',
+              options: filterOptions.type.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+            {
+              type: 'field_value_selection',
+              field: 'namespace',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', {
+                defaultMessage: 'Namespace',
+              }),
+              multiSelect: 'or',
+              options: filterOptions.namespace.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+            {
+              type: 'field_value_selection',
+              field: 'package',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', {
+                defaultMessage: 'Integration',
+              }),
+              multiSelect: 'or',
+              options: filterOptions.package.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+          ],
+        }}
+      />
+    </DataStreamListPageLayout>
+  );
+};

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx

@@ -6,6 +6,7 @@
 export { IngestManagerOverview } from './overview';
 export { EPMApp } from './epm';
 export { AgentConfigApp } from './agent_config';
+export { DataStreamApp } from './data_stream';
 export { FleetApp } from './fleet';
 
-export type Section = 'overview' | 'epm' | 'agent_config' | 'fleet';
+export type Section = 'overview' | 'epm' | 'agent_config' | 'fleet' | 'data_stream';

x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts

@@ -9,6 +9,7 @@ export { getFlattenedObject } from '../../../../../../../src/core/utils';
 export {
   agentConfigRouteService,
   datasourceRouteService,
+  dataStreamRouteService,
   fleetSetupRouteService,
   agentRouteService,
   enrollmentAPIKeyRouteService,

x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts

@@ -17,6 +17,7 @@ export {
   DatasourceInput,
   DatasourceInputStream,
   DatasourceConfigRecordEntry,
+  DataStream,
   // API schemas - Agent Config
   GetAgentConfigsResponse,
   GetAgentConfigsResponseItem,
@@ -30,6 +31,8 @@ export {
   // API schemas - Datasource
   CreateDatasourceRequest,
   CreateDatasourceResponse,
+  // API schemas - Data Streams
+  GetDataStreamsResponse,
   // API schemas - Agents
   GetAgentsResponse,
   GetAgentsRequest,

x-pack/plugins/ingest_manager/server/constants/index.ts

@@ -12,6 +12,7 @@ export {
   // Routes
   PLUGIN_ID,
   EPM_API_ROUTES,
+  DATA_STREAM_API_ROUTES,
   DATASOURCE_API_ROUTES,
   AGENT_API_ROUTES,
   AGENT_CONFIG_API_ROUTES,

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -33,6 +33,7 @@ import { registerEncryptedSavedObjects } from './saved_objects';
 import {
   registerEPMRoutes,
   registerDatasourceRoutes,
+  registerDataStreamRoutes,
   registerAgentConfigRoutes,
   registerSetupRoutes,
   registerAgentRoutes,
@@ -141,6 +142,7 @@ export class IngestManagerPlugin
     // Register routes
     registerAgentConfigRoutes(router);
     registerDatasourceRoutes(router);
+    registerDataStreamRoutes(router);
 
     // Conditional routes
     if (config.epm.enabled) {

x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { RequestHandler } from 'src/core/server';
+import { DataStream } from '../../types';
+import { GetDataStreamsResponse } from '../../../common';
+
+const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*';
+
+export const getListHandler: RequestHandler = async (context, request, response) => {
+  const callCluster = context.core.elasticsearch.dataClient.callAsCurrentUser;
+
+  try {
+    // Get stats (size on disk) of all potentially matching indices
+    const { indices: indexStats } = await callCluster('indices.stats', {
+      index: DATA_STREAM_INDEX_PATTERN,
+      metric: ['store'],
+    });
+
+    // Get all matching indices and info about each
+    // This returns the top 100,000 indices (as buckets) by last activity
+    const {
+      aggregations: {
+        index: { buckets: indexResults },
+      },
+    } = await callCluster('search', {
+      index: DATA_STREAM_INDEX_PATTERN,
+      body: {
+        size: 0,
+        query: {
+          bool: {
+            must: [
+              {
+                exists: {
+                  field: 'stream.namespace',
+                },
+              },
+              {
+                exists: {
+                  field: 'stream.dataset',
+                },
+              },
+            ],
+          },
+        },
+        aggs: {
+          index: {
+            terms: {
+              field: '_index',
+              size: 100000,
+              order: {
+                last_activity: 'desc',
+              },
+            },
+            aggs: {
+              dataset: {
+                terms: {
+                  field: 'stream.dataset',
+                  size: 1,
+                },
+              },
+              namespace: {
+                terms: {
+                  field: 'stream.namespace',
+                  size: 1,
+                },
+              },
+              type: {
+                terms: {
+                  field: 'stream.type',
+                  size: 1,
+                },
+              },
+              package: {
+                terms: {
+                  field: 'event.module',
+                  size: 1,
+                },
+              },
+              last_activity: {
+                max: {
+                  field: '@timestamp',
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    const dataStreams: DataStream[] = (indexResults as any[]).map(result => {
+      const {
+        key: indexName,
+        dataset: { buckets: datasetBuckets },
+        namespace: { buckets: namespaceBuckets },
+        type: { buckets: typeBuckets },
+        package: { buckets: packageBuckets },
+        last_activity: { value_as_string: lastActivity },
+      } = result;
+      return {
+        index: indexName,
+        dataset: datasetBuckets.length ? datasetBuckets[0].key : '',
+        namespace: namespaceBuckets.length ? namespaceBuckets[0].key : '',
+        type: typeBuckets.length ? typeBuckets[0].key : '',
+        package: packageBuckets.length ? packageBuckets[0].key : '',
+        last_activity: lastActivity,
+        size_in_bytes: indexStats[indexName] ? indexStats[indexName].total.store.size_in_bytes : 0,
+      };
+    });
+
+    const body: GetDataStreamsResponse = {
+      data_streams: dataStreams,
+    };
+    return response.ok({
+      body,
+    });
+  } catch (e) {
+    return response.customError({
+      statusCode: 500,
+      body: { message: e.message },
+    });
+  }
+};

x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { IRouter } from 'src/core/server';
+import { PLUGIN_ID, DATA_STREAM_API_ROUTES } from '../../constants';
+import { getListHandler } from './handlers';
+
+export const registerRoutes = (router: IRouter) => {
+  // List of data streams
+  router.get(
+    {
+      path: DATA_STREAM_API_ROUTES.LIST_PATTERN,
+      validate: false,
+      options: { tags: [`access:${PLUGIN_ID}-read`] },
+    },
+    getListHandler
+  );
+};

x-pack/plugins/ingest_manager/server/routes/index.ts

@@ -5,6 +5,7 @@
  */
 export { registerRoutes as registerAgentConfigRoutes } from './agent_config';
 export { registerRoutes as registerDatasourceRoutes } from './datasource';
+export { registerRoutes as registerDataStreamRoutes } from './data_streams';
 export { registerRoutes as registerEPMRoutes } from './epm';
 export { registerRoutes as registerSetupRoutes } from './setup';
 export { registerRoutes as registerAgentRoutes } from './agent';

x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts

@@ -5,7 +5,7 @@
  */
 import { RequestHandler } from 'src/core/server';
 import { outputService } from '../../services';
-import { CreateFleetSetupResponse } from '../../types';
+import { CreateFleetSetupResponse } from '../../../common';
 import { setupIngestManager, setupFleet } from '../../services/setup';
 
 export const getFleetSetupHandler: RequestHandler = async (context, request, response) => {

x-pack/plugins/ingest_manager/server/routes/setup/index.ts

@@ -5,7 +5,6 @@
  */
 import { IRouter } from 'src/core/server';
 import { PLUGIN_ID, FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../constants';
-import { GetFleetSetupRequestSchema, CreateFleetSetupRequestSchema } from '../../types';
 import {
   getFleetSetupHandler,
   createFleetSetupHandler,
@@ -28,7 +27,7 @@ export const registerRoutes = (router: IRouter) => {
   router.get(
     {
       path: FLEET_SETUP_API_ROUTES.INFO_PATTERN,
-      validate: GetFleetSetupRequestSchema,
+      validate: false,
       options: { tags: [`access:${PLUGIN_ID}-read`] },
     },
     getFleetSetupHandler
@@ -38,7 +37,7 @@ export const registerRoutes = (router: IRouter) => {
   router.post(
     {
       path: FLEET_SETUP_API_ROUTES.CREATE_PATTERN,
-      validate: CreateFleetSetupRequestSchema,
+      validate: false,
       options: { tags: [`access:${PLUGIN_ID}-all`] },
     },
     createFleetSetupHandler

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -22,6 +22,7 @@ export {
   AgentConfig,
   NewAgentConfig,
   AgentConfigStatus,
+  DataStream,
   Output,
   NewOutput,
   OutputType,

x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts

@@ -9,5 +9,4 @@ export * from './agent';
 export * from './datasource';
 export * from './epm';
 export * from './enrollment_api_key';
-export * from './fleet_setup';
 export * from './install_script';