Small cleanup

This reverts commit be170e1e0c, reversing
changes made to cdcf869fae.

docs/api/dashboard-api.asciidoc

@@ -1,7 +1,7 @@
 [[dashboard-api]]
 == Import and export dashboard APIs
 
-deprecated::[7.15.0,These experimental APIs have been deprecated in favor of <<saved-objects-api-import>> and <<saved-objects-api-export>>.]
+deprecated::[7.15.0,Both of these APIs have been deprecated in favor of <<saved-objects-api-import>> and <<saved-objects-api-export>>.]
 
 Import and export dashboards with the corresponding saved objects, such as visualizations, saved
 searches, and index patterns.

docs/api/dashboard/export-dashboard.asciidoc

@@ -6,7 +6,7 @@
 
 deprecated::[7.15.0,Use <<saved-objects-api-export>> instead.]
 
-experimental[] Export dashboards and corresponding saved objects.
+Export dashboards and corresponding saved objects.
 
 [[dashboard-api-export-request]]
 ==== Request

docs/api/dashboard/import-dashboard.asciidoc

@@ -6,7 +6,7 @@
 
 deprecated::[7.15.0,Use <<saved-objects-api-import>> instead.]
 
-experimental[] Import dashboards and corresponding saved objects.
+Import dashboards and corresponding saved objects.
 
 [[dashboard-api-import-request]]
 ==== Request

packages/kbn-securitysolution-autocomplete/src/field/index.tsx

@@ -8,7 +8,7 @@
 
 import React, { useCallback, useMemo, useState } from 'react';
 import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
-import { IndexPatternBase, IndexPatternFieldBase } from '@kbn/es-query';
+import { DataViewBase, DataViewFieldBase } from '@kbn/es-query';
 
 import {
   getGenericComboBoxProps,
@@ -20,14 +20,14 @@ const AS_PLAIN_TEXT = { asPlainText: true };
 interface OperatorProps {
   fieldInputWidth?: number;
   fieldTypeFilter?: string[];
-  indexPattern: IndexPatternBase | undefined;
+  indexPattern: DataViewBase | undefined;
   isClearable: boolean;
   isDisabled: boolean;
   isLoading: boolean;
   isRequired?: boolean;
-  onChange: (a: IndexPatternFieldBase[]) => void;
+  onChange: (a: DataViewFieldBase[]) => void;
   placeholder: string;
-  selectedField: IndexPatternFieldBase | undefined;
+  selectedField: DataViewFieldBase | undefined;
 }
 
 export const FieldComponent: React.FC<OperatorProps> = ({
@@ -56,7 +56,7 @@ export const FieldComponent: React.FC<OperatorProps> = ({
 
   const handleValuesChange = useCallback(
     (newOptions: EuiComboBoxOptionOption[]): void => {
-      const newValues: IndexPatternFieldBase[] = newOptions.map(
+      const newValues: DataViewFieldBase[] = newOptions.map(
         ({ label }) => availableFields[labels.indexOf(label)]
       );
       onChange(newValues);
@@ -94,13 +94,13 @@ export const FieldComponent: React.FC<OperatorProps> = ({
 FieldComponent.displayName = 'Field';
 
 interface ComboBoxFields {
-  availableFields: IndexPatternFieldBase[];
-  selectedFields: IndexPatternFieldBase[];
+  availableFields: DataViewFieldBase[];
+  selectedFields: DataViewFieldBase[];
 }
 
 const getComboBoxFields = (
-  indexPattern: IndexPatternBase | undefined,
-  selectedField: IndexPatternFieldBase | undefined,
+  indexPattern: DataViewBase | undefined,
+  selectedField: DataViewFieldBase | undefined,
   fieldTypeFilter: string[]
 ): ComboBoxFields => {
   const existingFields = getExistingFields(indexPattern);
@@ -113,29 +113,27 @@ const getComboBoxFields = (
 const getComboBoxProps = (fields: ComboBoxFields): GetGenericComboBoxPropsReturn => {
   const { availableFields, selectedFields } = fields;
 
-  return getGenericComboBoxProps<IndexPatternFieldBase>({
+  return getGenericComboBoxProps<DataViewFieldBase>({
     getLabel: (field) => field.name,
     options: availableFields,
     selectedOptions: selectedFields,
   });
 };
 
-const getExistingFields = (indexPattern: IndexPatternBase | undefined): IndexPatternFieldBase[] => {
+const getExistingFields = (indexPattern: DataViewBase | undefined): DataViewFieldBase[] => {
   return indexPattern != null ? indexPattern.fields : [];
 };
 
-const getSelectedFields = (
-  selectedField: IndexPatternFieldBase | undefined
-): IndexPatternFieldBase[] => {
+const getSelectedFields = (selectedField: DataViewFieldBase | undefined): DataViewFieldBase[] => {
   return selectedField ? [selectedField] : [];
 };
 
 const getAvailableFields = (
-  existingFields: IndexPatternFieldBase[],
-  selectedFields: IndexPatternFieldBase[],
+  existingFields: DataViewFieldBase[],
+  selectedFields: DataViewFieldBase[],
   fieldTypeFilter: string[]
-): IndexPatternFieldBase[] => {
-  const fieldsByName = new Map<string, IndexPatternFieldBase>();
+): DataViewFieldBase[] => {
+  const fieldsByName = new Map<string, DataViewFieldBase>();
 
   existingFields.forEach((f) => fieldsByName.set(f.name, f));
   selectedFields.forEach((f) => fieldsByName.set(f.name, f));

src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts

@@ -5,7 +5,6 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
 import { IndexPatternsFetcher } from '.';
 import { ElasticsearchClient } from 'kibana/server';
 import * as indexNotFoundException from './index_not_found_exception.json';
@@ -15,36 +14,36 @@ describe('Index Pattern Fetcher - server', () => {
   let esClient: ElasticsearchClient;
   const emptyResponse = {
     body: {
-      count: 0,
+      indices: [],
     },
   };
   const response = {
     body: {
-      count: 1115,
+      indices: ['b'],
+      fields: [{ name: 'foo' }, { name: 'bar' }, { name: 'baz' }],
     },
   };
   const patternList = ['a', 'b', 'c'];
   beforeEach(() => {
+    jest.clearAllMocks();
     esClient = {
-      count: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
+      fieldCaps: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
     } as unknown as ElasticsearchClient;
     indexPatterns = new IndexPatternsFetcher(esClient);
   });
-
   it('Removes pattern without matching indices', async () => {
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual(['b', 'c']);
   });
-
   it('Returns all patterns when all match indices', async () => {
     esClient = {
-      count: jest.fn().mockResolvedValue(response),
+      fieldCaps: jest.fn().mockResolvedValue(response),
     } as unknown as ElasticsearchClient;
     indexPatterns = new IndexPatternsFetcher(esClient);
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual(patternList);
   });
-  it('Removes pattern when "index_not_found_exception" error is thrown', async () => {
+  it('Removes pattern when error is thrown', async () => {
     class ServerError extends Error {
       public body?: Record<string, any>;
       constructor(
@@ -56,9 +55,8 @@ describe('Index Pattern Fetcher - server', () => {
         this.body = errBody;
       }
     }
-
     esClient = {
-      count: jest
+      fieldCaps: jest
         .fn()
         .mockResolvedValueOnce(response)
         .mockRejectedValue(
@@ -69,4 +67,22 @@ describe('Index Pattern Fetcher - server', () => {
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual([patternList[0]]);
   });
+  it('When allowNoIndices is false, run validatePatternListActive', async () => {
+    const fieldCapsMock = jest.fn();
+    esClient = {
+      fieldCaps: fieldCapsMock.mockResolvedValue(response),
+    } as unknown as ElasticsearchClient;
+    indexPatterns = new IndexPatternsFetcher(esClient);
+    await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+    expect(fieldCapsMock.mock.calls).toHaveLength(4);
+  });
+  it('When allowNoIndices is true, do not run validatePatternListActive', async () => {
+    const fieldCapsMock = jest.fn();
+    esClient = {
+      fieldCaps: fieldCapsMock.mockResolvedValue(response),
+    } as unknown as ElasticsearchClient;
+    indexPatterns = new IndexPatternsFetcher(esClient, true);
+    await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+    expect(fieldCapsMock.mock.calls).toHaveLength(1);
+  });
 });

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -36,12 +36,10 @@ interface FieldSubType {
 export class IndexPatternsFetcher {
   private elasticsearchClient: ElasticsearchClient;
   private allowNoIndices: boolean;
-
   constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) {
     this.elasticsearchClient = elasticsearchClient;
     this.allowNoIndices = allowNoIndices;
   }
-
   /**
    *  Get a list of field objects for an index pattern that may contain wildcards
    *
@@ -60,23 +58,22 @@ export class IndexPatternsFetcher {
   }): Promise<FieldDescriptor[]> {
     const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
     const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
+    const allowNoIndices = fieldCapsOptions
+      ? fieldCapsOptions.allow_no_indices
+      : this.allowNoIndices;
     let patternListActive: string[] = patternList;
     // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless
-    if (patternList.length > 1) {
+    if (patternList.length > 1 && !allowNoIndices) {
       patternListActive = await this.validatePatternListActive(patternList);
     }
     const fieldCapsResponse = await getFieldCapabilities(
       this.elasticsearchClient,
-      // if none of the patterns are active, pass the original list to get an error
-      patternListActive.length > 0 ? patternListActive : patternList,
+      patternListActive,
       metaFields,
       {
-        allow_no_indices: fieldCapsOptions
-          ? fieldCapsOptions.allow_no_indices
-          : this.allowNoIndices,
+        allow_no_indices: allowNoIndices,
       }
     );
-
     if (type === 'rollup' && rollupIndex) {
       const rollupFields: FieldDescriptor[] = [];
       const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@@ -87,13 +84,11 @@ export class IndexPatternsFetcher {
         ).body
       )[rollupIndex].aggs;
       const fieldCapsResponseObj = keyBy(fieldCapsResponse, 'name');
-
       // Keep meta fields
       metaFields!.forEach(
         (field: string) =>
           fieldCapsResponseObj[field] && rollupFields.push(fieldCapsResponseObj[field])
       );
-
       return mergeCapabilitiesWithFields(
         rollupIndexCapabilities,
         fieldCapsResponseObj,
@@ -137,23 +132,20 @@ export class IndexPatternsFetcher {
   async validatePatternListActive(patternList: string[]) {
     const result = await Promise.all(
       patternList
-        .map((pattern) =>
-          this.elasticsearchClient.count({
-            index: pattern,
-          })
-        )
-        .map((p) =>
-          p.catch((e) => {
-            if (e.body.error.type === 'index_not_found_exception') {
-              return { body: { count: 0 } };
-            }
-            throw e;
-          })
-        )
+        .map(async (index) => {
+          const searchResponse = await this.elasticsearchClient.fieldCaps({
+            index,
+            fields: '_id',
+            ignore_unavailable: true,
+            allow_no_indices: false,
+          });
+          return searchResponse.body.indices.length > 0;
+        })
+        .map((p) => p.catch(() => false))
     );
     return result.reduce(
-      (acc: string[], { body: { count } }, patternListIndex) =>
-        count > 0 ? [...acc, patternList[patternListIndex]] : acc,
+      (acc: string[], isValid, patternListIndex) =>
+        isValid ? [...acc, patternList[patternListIndex]] : acc,
       []
     );
   }

x-pack/plugins/observability/public/pages/alerts/alerts_table_t_grid.tsx

@@ -412,6 +412,8 @@ export function AlertsTableTGrid(props: AlertsTableTGridProps) {
       },
       renderCellValue: getRenderCellValue({ setFlyoutAlert }),
       rowRenderers: NO_ROW_RENDER,
+      // TODO: implement Kibana data view runtime fields in observability
+      runtimeMappings: {},
       start: rangeFrom,
       setRefetch,
       sort: [

x-pack/plugins/security_solution/common/constants.ts

@@ -11,8 +11,16 @@ import type { TransformConfigSchema } from './transforms/types';
 import { ENABLE_CASE_CONNECTOR } from '../../cases/common';
 import { METADATA_TRANSFORMS_PATTERN } from './endpoint/constants';
 
+/**
+ * as const
+ *
+ * The const assertion ensures that type widening does not occur
+ * https://mariusschulz.com/blog/literal-type-widening-in-typescript
+ * Please follow this convention when adding to this file
+ */
+
 export const APP_ID = 'securitySolution' as const;
-export const APP_UI_ID = 'securitySolutionUI';
+export const APP_UI_ID = 'securitySolutionUI' as const;
 export const CASES_FEATURE_ID = 'securitySolutionCases' as const;
 export const SERVER_APP_ID = 'siem' as const;
 export const APP_NAME = 'Security' as const;
@@ -26,6 +34,8 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz' as const;
 export const DEFAULT_DARK_MODE = 'theme:darkMode' as const;
 export const DEFAULT_INDEX_KEY = 'securitySolution:defaultIndex' as const;
 export const DEFAULT_NUMBER_FORMAT = 'format:number:defaultPattern' as const;
+export const DEFAULT_DATA_VIEW_ID = 'security-solution' as const;
+export const DEFAULT_TIME_FIELD = '@timestamp' as const;
 export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults' as const;
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults' as const;
 export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const;
@@ -51,7 +61,6 @@ export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges' as const
 export const DEFAULT_TRANSFORMS = 'securitySolution:transforms' as const;
 export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled' as const;
 export const GLOBAL_HEADER_HEIGHT = 96 as const; // px
-export const GLOBAL_HEADER_HEIGHT_WITH_GLOBAL_BANNER = 128 as const; // px
 export const FILTERS_GLOBAL_HEIGHT = 109 as const; // px
 export const FULL_SCREEN_TOGGLED_CLASS_NAME = 'fullScreenToggled' as const;
 export const NO_ALERT_INDEX = 'no-alert-index-049FC71A-4C2C-446F-9901-37XMC5024C51' as const;
@@ -268,6 +277,7 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged` as const;
 
 export const NOTE_URL = '/api/note' as const;
 export const PINNED_EVENT_URL = '/api/pinned_event' as const;
+export const SOURCERER_API_URL = '/api/sourcerer' as const;
 
 /**
  * Default signals index key for kibana.dev.yml
@@ -355,7 +365,7 @@ export const ELASTIC_NAME = 'estc' as const;
 
 export const METADATA_TRANSFORM_STATS_URL = `/api/transform/transforms/${METADATA_TRANSFORMS_PATTERN}/_stats`;
 
-export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_';
+export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_' as const;
 
 export const TRANSFORM_STATES = {
   ABORTING: 'aborting',

x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts

@@ -11,7 +11,7 @@ import type {
   CreateExceptionListItemSchema,
 } from '@kbn/securitysolution-io-ts-list-types';
 import { buildExceptionFilter } from '@kbn/securitysolution-list-utils';
-import { Filter, EsQueryConfig, IndexPatternBase, buildEsQuery } from '@kbn/es-query';
+import { Filter, EsQueryConfig, DataViewBase, buildEsQuery } from '@kbn/es-query';
 
 import { ESBoolQuery } from '../typed_json';
 import { Query, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
@@ -24,7 +24,7 @@ export const getQueryFilter = (
   lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>,
   excludeExceptions: boolean = true
 ): ESBoolQuery => {
-  const indexPattern: IndexPatternBase = {
+  const indexPattern: DataViewBase = {
     fields: [],
     title: index.join(),
   };

x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts

@@ -5,79 +5,15 @@
  * 2.0.
  */
 
-import type { IFieldSubType } from '@kbn/es-query';
-
-import type {
-  IEsSearchRequest,
-  IEsSearchResponse,
-  IIndexPattern,
-} from '../../../../../../src/plugins/data/common';
-import type { DocValueFields, Maybe } from '../common';
-
-interface FieldInfo {
-  category: string;
-  description?: string;
-  example?: string | number;
-  format?: string;
-  name: string;
-  type?: string;
-}
-
-export interface IndexField {
-  /** Where the field belong */
-  category: string;
-  /** Example of field's value */
-  example?: Maybe<string | number>;
-  /** whether the field's belong to an alias index */
-  indexes: Array<Maybe<string>>;
-  /** The name of the field */
-  name: string;
-  /** The type of the field's values as recognized by Kibana */
-  type: string;
-  /** Whether the field's values can be efficiently searched for */
-  searchable: boolean;
-  /** Whether the field's values can be aggregated */
-  aggregatable: boolean;
-  /** Description of the field */
-  description?: Maybe<string>;
-  format?: Maybe<string>;
-  /** the elastic type as mapped in the index */
-  esTypes?: string[];
-  subType?: IFieldSubType;
-  readFromDocValues: boolean;
-}
-
-export type BeatFields = Record<string, FieldInfo>;
-
-export interface IndexFieldsStrategyRequest extends IEsSearchRequest {
-  indices: string[];
-  onlyCheckIfIndicesExist: boolean;
-}
-
-export interface IndexFieldsStrategyResponse extends IEsSearchResponse {
-  indexFields: IndexField[];
-  indicesExist: string[];
-}
-
-export interface BrowserField {
-  aggregatable: boolean;
-  category: string;
-  description: string | null;
-  example: string | number | null;
-  fields: Readonly<Record<string, Partial<BrowserField>>>;
-  format: string;
-  indexes: string[];
-  name: string;
-  searchable: boolean;
-  type: string;
-  subType?: IFieldSubType;
-}
-
-export type BrowserFields = Readonly<Record<string, Partial<BrowserField>>>;
-
-export const EMPTY_BROWSER_FIELDS = {};
-export const EMPTY_DOCVALUE_FIELD: DocValueFields[] = [];
-export const EMPTY_INDEX_PATTERN: IIndexPattern = {
-  fields: [],
-  title: '',
-};
+export {
+  FieldInfo,
+  IndexField,
+  BeatFields,
+  IndexFieldsStrategyRequest,
+  IndexFieldsStrategyResponse,
+  BrowserField,
+  BrowserFields,
+  EMPTY_BROWSER_FIELDS,
+  EMPTY_DOCVALUE_FIELD,
+  EMPTY_INDEX_FIELDS,
+} from '../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/events/last_event_time/index.ts

@@ -10,6 +10,7 @@ export { LastEventIndexKey } from '../../../../../../timelines/common';
 export type {
   LastTimeDetails,
   TimelineEventsLastEventTimeStrategyResponse,
+  TimelineKpiStrategyRequest,
   TimelineKpiStrategyResponse,
   TimelineEventsLastEventTimeRequestOptions,
 } from '../../../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { IEsSearchRequest } from '../../../../../../src/plugins/data/common';
 import { ESQuery } from '../../typed_json';
 import {
@@ -41,6 +42,7 @@ export interface TimelineRequestBasicOptions extends IEsSearchRequest {
   defaultIndex: string[];
   docValueFields?: DocValueFields[];
   factoryQueryType?: TimelineFactoryQueryTypes;
+  runtimeMappings: MappingRuntimeFields;
 }
 
 export interface TimelineRequestSortField<Field = string> extends SortField<Field> {
@@ -171,6 +173,7 @@ export interface SortTimelineInput {
 export interface TimelineInput {
   columns?: Maybe<ColumnHeaderInput[]>;
   dataProviders?: Maybe<DataProviderInput[]>;
+  dataViewId?: Maybe<string>;
   description?: Maybe<string>;
   eqlOptions?: Maybe<EqlOptionsInput>;
   eventType?: Maybe<string>;

x-pack/plugins/security_solution/common/test/index.ts

@@ -7,12 +7,12 @@
 
 // For the source of these roles please consult the PR these were introduced https://github.com/elastic/kibana/pull/81866#issue-511165754
 export enum ROLES {
+  soc_manager = 'soc_manager',
   reader = 'reader',
   t1_analyst = 't1_analyst',
   t2_analyst = 't2_analyst',
   hunter = 'hunter',
   rule_author = 'rule_author',
-  soc_manager = 'soc_manager',
   platform_engineer = 'platform_engineer',
   detections_admin = 'detections_admin',
 }

x-pack/plugins/security_solution/common/types/timeline/index.ts

@@ -272,6 +272,7 @@ export type TimelineTypeLiteralWithNull = runtimeTypes.TypeOf<typeof TimelineTyp
 export const SavedTimelineRuntimeType = runtimeTypes.partial({
   columns: unionWithNullType(runtimeTypes.array(SavedColumnHeaderRuntimeType)),
   dataProviders: unionWithNullType(runtimeTypes.array(SavedDataProviderRuntimeType)),
+  dataViewId: unionWithNullType(runtimeTypes.string),
   description: unionWithNullType(runtimeTypes.string),
   eqlOptions: unionWithNullType(EqlOptionsRuntimeType),
   eventType: unionWithNullType(runtimeTypes.string),
@@ -305,7 +306,7 @@ export type SavedTimelineNote = runtimeTypes.TypeOf<typeof SavedTimelineRuntimeT
  * This type represents a timeline type stored in a saved object that does not include any fields that reference
  * other saved objects.
  */
-export type TimelineWithoutExternalRefs = Omit<SavedTimeline, 'savedQueryId'>;
+export type TimelineWithoutExternalRefs = Omit<SavedTimeline, 'dataViewId' | 'savedQueryId'>;
 
 /*
  *  Timeline IDs
@@ -719,6 +720,7 @@ export interface TimelineResult {
   created?: Maybe<number>;
   createdBy?: Maybe<string>;
   dataProviders?: Maybe<DataProviderResult[]>;
+  dataViewId?: Maybe<string>;
   dateRange?: Maybe<DateRangePickerResult>;
   description?: Maybe<string>;
   eqlOptions?: Maybe<EqlOptionsResult>;

x-pack/plugins/security_solution/common/types/timeline/store.ts

@@ -38,19 +38,20 @@ export interface SortColumnTimeline {
 }
 
 export interface TimelinePersistInput {
-  id: string;
+  columns: ColumnHeaderOptions[];
   dataProviders?: DataProvider[];
+  dataViewId: string;
   dateRange?: {
     start: string;
     end: string;
   };
+  defaultColumns?: ColumnHeaderOptions[];
   excludedRowRendererIds?: RowRendererId[];
   expandedDetail?: TimelineExpandedDetail;
   filters?: Filter[];
-  columns: ColumnHeaderOptions[];
-  defaultColumns?: ColumnHeaderOptions[];
-  itemsPerPage?: number;
+  id: string;
   indexNames: string[];
+  itemsPerPage?: number;
   kqlQuery?: {
     filterQuery: SerializedFilterQuery | null;
   };

x-pack/plugins/security_solution/cypress/cypress.json

@@ -12,5 +12,10 @@
   "video": false,
   "videosFolder": "../../../target/kibana-security-solution/cypress/videos",
   "viewportHeight": 900,
-  "viewportWidth": 1440
+  "viewportWidth": 1440,
+  "env": {
+    "protocol": "http",
+    "hostname": "localhost",
+    "configport": "5601"
+  }
 }

x-pack/plugins/security_solution/cypress/downloads/timelines_export.ndjson

@@ -0,0 +1 @@
+{"savedObjectId":"46cca0e0-2580-11ec-8e56-9dafa0b0343b","version":"WzIyNjIzNCwxXQ==","columns":[{"id":"@timestamp"},{"id":"user.name"},{"id":"event.category"},{"id":"event.action"},{"id":"host.name"}],"kqlMode":"filter","kqlQuery":{"filterQuery":{"kuery":{"expression":"host.name: *","kind":"kuery"}}},"dateRange":{"start":"1514809376000","end":"1577881376000"},"description":"This is the best timeline","title":"Security Timeline","created":1633399341550,"createdBy":"elastic","updated":1633399341550,"updatedBy":"elastic","savedQueryId":null,"dataViewId":null,"timelineType":"default","sort":[],"eventNotes":[],"globalNotes":[],"pinnedEventIds":[]}

x-pack/plugins/security_solution/cypress/integration/cases/privileges.spec.ts

@@ -18,184 +18,28 @@ import {
   filterStatusOpen,
 } from '../../tasks/create_new_case';
 import {
-  constructUrlWithUser,
-  getEnvAuth,
+  loginAndWaitForHostDetailsPage,
   loginWithUserAndWaitForPageWithoutDateRange,
+  logout,
 } from '../../tasks/login';
+import {
+  createUsersAndRoles,
+  deleteUsersAndRoles,
+  secAll,
+  secAllUser,
+  secReadCasesAllUser,
+  secReadCasesAll,
+} from '../../tasks/privileges';
 
 import { CASES_URL } from '../../urls/navigation';
-
-interface User {
-  username: string;
-  password: string;
-  description?: string;
-  roles: string[];
-}
-
-interface UserInfo {
-  username: string;
-  full_name: string;
-  email: string;
-}
-
-interface FeaturesPrivileges {
-  [featureId: string]: string[];
-}
-
-interface ElasticsearchIndices {
-  names: string[];
-  privileges: string[];
-}
-
-interface ElasticSearchPrivilege {
-  cluster?: string[];
-  indices?: ElasticsearchIndices[];
-}
-
-interface KibanaPrivilege {
-  spaces: string[];
-  base?: string[];
-  feature?: FeaturesPrivileges;
-}
-
-interface Role {
-  name: string;
-  privileges: {
-    elasticsearch?: ElasticSearchPrivilege;
-    kibana?: KibanaPrivilege[];
-  };
-}
-
-const secAll: Role = {
-  name: 'sec_all_role',
-  privileges: {
-    elasticsearch: {
-      indices: [
-        {
-          names: ['*'],
-          privileges: ['all'],
-        },
-      ],
-    },
-    kibana: [
-      {
-        feature: {
-          siem: ['all'],
-          securitySolutionCases: ['all'],
-          actions: ['all'],
-          actionsSimulators: ['all'],
-        },
-        spaces: ['*'],
-      },
-    ],
-  },
-};
-
-const secAllUser: User = {
-  username: 'sec_all_user',
-  password: 'password',
-  roles: [secAll.name],
-};
-
-const secReadCasesAll: Role = {
-  name: 'sec_read_cases_all_role',
-  privileges: {
-    elasticsearch: {
-      indices: [
-        {
-          names: ['*'],
-          privileges: ['all'],
-        },
-      ],
-    },
-    kibana: [
-      {
-        feature: {
-          siem: ['read'],
-          securitySolutionCases: ['all'],
-          actions: ['all'],
-          actionsSimulators: ['all'],
-        },
-        spaces: ['*'],
-      },
-    ],
-  },
-};
-
-const secReadCasesAllUser: User = {
-  username: 'sec_read_cases_all_user',
-  password: 'password',
-  roles: [secReadCasesAll.name],
-};
-
+import { openSourcerer } from '../../tasks/sourcerer';
 const usersToCreate = [secAllUser, secReadCasesAllUser];
 const rolesToCreate = [secAll, secReadCasesAll];
-
-const getUserInfo = (user: User): UserInfo => ({
-  username: user.username,
-  full_name: user.username.replace('_', ' '),
-  email: `${user.username}@elastic.co`,
-});
-
-const createUsersAndRoles = (users: User[], roles: Role[]) => {
-  const envUser = getEnvAuth();
-  for (const role of roles) {
-    cy.log(`Creating role: ${JSON.stringify(role)}`);
-    cy.request({
-      body: role.privileges,
-      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
-      method: 'PUT',
-      url: constructUrlWithUser(envUser, `/api/security/role/${role.name}`),
-    })
-      .its('status')
-      .should('eql', 204);
-  }
-
-  for (const user of users) {
-    const userInfo = getUserInfo(user);
-    cy.log(`Creating user: ${JSON.stringify(user)}`);
-    cy.request({
-      body: {
-        username: user.username,
-        password: user.password,
-        roles: user.roles,
-        full_name: userInfo.full_name,
-        email: userInfo.email,
-      },
-      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
-      method: 'POST',
-      url: constructUrlWithUser(envUser, `/internal/security/users/${user.username}`),
-    })
-      .its('status')
-      .should('eql', 200);
-  }
-};
-
-const deleteUsersAndRoles = (users: User[], roles: Role[]) => {
-  const envUser = getEnvAuth();
-  for (const user of users) {
-    cy.log(`Deleting user: ${JSON.stringify(user)}`);
-    cy.request({
-      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
-      method: 'DELETE',
-      url: constructUrlWithUser(envUser, `/internal/security/users/${user.username}`),
-      failOnStatusCode: false,
-    })
-      .its('status')
-      .should('oneOf', [204, 404]);
-  }
-
-  for (const role of roles) {
-    cy.log(`Deleting role: ${JSON.stringify(role)}`);
-    cy.request({
-      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
-      method: 'DELETE',
-      url: constructUrlWithUser(envUser, `/api/security/role/${role.name}`),
-      failOnStatusCode: false,
-    })
-      .its('status')
-      .should('oneOf', [204, 404]);
-  }
+// needed to generate index pattern
+const visitSecuritySolution = () => {
+  loginAndWaitForHostDetailsPage();
+  openSourcerer();
+  logout();
 };
 
 const testCase: TestCaseWithoutTimeline = {
@@ -205,11 +49,11 @@ const testCase: TestCaseWithoutTimeline = {
   reporter: 'elastic',
   owner: 'securitySolution',
 };
-
 describe('Cases privileges', () => {
   before(() => {
     cleanKibana();
     createUsersAndRoles(usersToCreate, rolesToCreate);
+    visitSecuritySolution();
   });
 
   after(() => {

x-pack/plugins/security_solution/cypress/integration/data_sources/sourcerer.spec.ts

@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-import { loginAndWaitForPage } from '../../tasks/login';
+import {
+  loginAndWaitForPage,
+  loginWithUserAndWaitForPageWithoutDateRange,
+} from '../../tasks/login';
 
 import { HOSTS_URL } from '../../urls/navigation';
 import { waitForAllHostsToBeLoaded } from '../../tasks/hosts/all_hosts';
@@ -28,20 +31,34 @@ import { openTimelineUsingToggle } from '../../tasks/security_main';
 import { populateTimeline } from '../../tasks/timeline';
 import { SERVER_SIDE_EVENT_COUNT } from '../../screens/timeline';
 import { cleanKibana } from '../../tasks/common';
+import { createUsersAndRoles, secReadCasesAll, secReadCasesAllUser } from '../../tasks/privileges';
+import { TOASTER } from '../../screens/configure_cases';
 
+const usersToCreate = [secReadCasesAllUser];
+const rolesToCreate = [secReadCasesAll];
 // Skipped at the moment as this has flake due to click handler issues. This has been raised with team members
 // and the code is being re-worked and then these tests will be unskipped
-describe.skip('Sourcerer', () => {
-  before(() => {
+describe('Sourcerer', () => {
+  beforeEach(() => {
     cleanKibana();
   });
-
-  beforeEach(() => {
-    cy.clearLocalStorage();
-    loginAndWaitForPage(HOSTS_URL);
+  describe('permissions', () => {
+    before(() => {
+      createUsersAndRoles(usersToCreate, rolesToCreate);
+    });
+    it(`role(s) ${secReadCasesAllUser.roles.join()} shows error when user does not have permissions`, () => {
+      loginWithUserAndWaitForPageWithoutDateRange(HOSTS_URL, secReadCasesAllUser);
+      cy.get(TOASTER).should('have.text', 'Write role required to generate data');
+    });
   });
+  // Originially written in December 2020, flakey from day1
+  // has always been skipped with intentions to fix, see note at top of file
+  describe.skip('Default scope', () => {
+    beforeEach(() => {
+      cy.clearLocalStorage();
+      loginAndWaitForPage(HOSTS_URL);
+    });
 
-  describe('Default scope', () => {
     it('has SIEM index patterns selected on initial load', () => {
       openSourcerer();
       isSourcererSelection(`auditbeat-*`);
@@ -52,7 +69,7 @@ describe.skip('Sourcerer', () => {
       isSourcererOptions([`metrics-*`, `logs-*`]);
     });
 
-    it('selected KIP gets added to sourcerer', () => {
+    it('selected DATA_VIEW gets added to sourcerer', () => {
       setSourcererOption(`metrics-*`);
       openSourcerer();
       isSourcererSelection(`metrics-*`);
@@ -75,8 +92,14 @@ describe.skip('Sourcerer', () => {
       isNotSourcererSelection(`metrics-*`);
     });
   });
+  // Originially written in December 2020, flakey from day1
+  // has always been skipped with intentions to fix
+  describe.skip('Timeline scope', () => {
+    beforeEach(() => {
+      cy.clearLocalStorage();
+      loginAndWaitForPage(HOSTS_URL);
+    });
 
-  describe('Timeline scope', () => {
     const alertPatterns = ['.siem-signals-default'];
     const rawPatterns = ['auditbeat-*'];
     const allPatterns = [...alertPatterns, ...rawPatterns];

x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts

@@ -9,6 +9,7 @@ import { ALERT_FLYOUT, CELL_TEXT, JSON_TEXT, TABLE_ROWS } from '../../screens/al
 
 import {
   expandFirstAlert,
+  refreshAlerts,
   waitForAlertsIndexToBeCreated,
   waitForAlertsPanelToBeLoaded,
 } from '../../tasks/alerts';
@@ -32,6 +33,7 @@ describe('Alert details with unmapped fields', () => {
     createCustomRuleActivated(getUnmappedRule());
     loginAndWaitForPageWithoutDateRange(ALERTS_URL);
     waitForAlertsPanelToBeLoaded();
+    refreshAlerts();
     expandFirstAlert();
   });
 

x-pack/plugins/security_solution/cypress/integration/detection_rules/event_correlation_rule.spec.ts

@@ -70,7 +70,7 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login';
 
 import { ALERTS_URL } from '../../urls/navigation';
 
-describe.skip('Detection rules, EQL', () => {
+describe('Detection rules, EQL', () => {
   const expectedUrls = getEqlRule().referenceUrls.join('');
   const expectedFalsePositives = getEqlRule().falsePositivesExamples.join('');
   const expectedTags = getEqlRule().tags.join('');
@@ -169,7 +169,7 @@ describe.skip('Detection rules, EQL', () => {
   });
 });
 
-describe.skip('Detection rules, sequence EQL', () => {
+describe('Detection rules, sequence EQL', () => {
   const expectedNumberOfRules = 1;
   const expectedNumberOfSequenceAlerts = '1 alert';
 

x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts

@@ -114,7 +114,7 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login';
 import { goBackToAllRulesTable } from '../../tasks/rule_details';
 
 import { ALERTS_URL, RULE_CREATION } from '../../urls/navigation';
-import { DEFAULT_THREAT_MATCH_QUERY } from '../../../common/constants';
+const DEFAULT_THREAT_MATCH_QUERY = '@timestamp >= "now-30d"';
 
 describe('indicator match', () => {
   describe('Detection rules, Indicator Match', () => {

x-pack/plugins/security_solution/cypress/integration/detection_rules/sorting.spec.ts

@@ -34,7 +34,6 @@ import {
   waitForRuleToChangeStatus,
 } from '../../tasks/alerts_detection_rules';
 import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login';
-import { DEFAULT_RULE_REFRESH_INTERVAL_VALUE } from '../../../common/constants';
 
 import { ALERTS_URL } from '../../urls/navigation';
 import { createCustomRule } from '../../tasks/api_calls/rules';
@@ -46,6 +45,8 @@ import {
   getNewThresholdRule,
 } from '../../objects/rule';
 
+const DEFAULT_RULE_REFRESH_INTERVAL_VALUE = 60000;
+
 describe('Alerts detection rules', () => {
   beforeEach(() => {
     cleanKibana();

x-pack/plugins/security_solution/cypress/integration/ml/ml_conditional_links.spec.ts

@@ -98,7 +98,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkSingleIpNullKqlQuery);
     cy.url().should(
       'include',
-      'app/security/network/ip/127.0.0.1/source?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      'app/security/network/ip/127.0.0.1/source?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -106,7 +106,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkSingleIpKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/network/ip/127.0.0.1/source?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/network/ip/127.0.0.1/source?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -114,7 +114,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkMultipleIpNullKqlQuery);
     cy.url().should(
       'include',
-      'app/security/network/flows?query=(language:kuery,query:%27((source.ip:%20%22127.0.0.1%22%20or%20destination.ip:%20%22127.0.0.1%22)%20or%20(source.ip:%20%22127.0.0.2%22%20or%20destination.ip:%20%22127.0.0.2%22))%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      'app/security/network/flows?query=(language:kuery,query:%27((source.ip:%20%22127.0.0.1%22%20or%20destination.ip:%20%22127.0.0.1%22)%20or%20(source.ip:%20%22127.0.0.2%22%20or%20destination.ip:%20%22127.0.0.2%22))%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -122,7 +122,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkMultipleIpKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/network/flows?query=(language:kuery,query:%27((source.ip:%20%22127.0.0.1%22%20or%20destination.ip:%20%22127.0.0.1%22)%20or%20(source.ip:%20%22127.0.0.2%22%20or%20destination.ip:%20%22127.0.0.2%22))%20and%20((process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22))%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/network/flows?query=(language:kuery,query:%27((source.ip:%20%22127.0.0.1%22%20or%20destination.ip:%20%22127.0.0.1%22)%20or%20(source.ip:%20%22127.0.0.2%22%20or%20destination.ip:%20%22127.0.0.2%22))%20and%20((process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22))%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -130,15 +130,16 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkNullKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/network/flows?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/network/flows?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
   it('redirects from a $ip$ with a value for the query', () => {
     loginAndWaitForPageWithoutDateRange(mlNetworkKqlQuery);
+
     cy.url().should(
       'include',
-      '/app/security/network/flows?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      `/app/security/network/flows?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-08-28T11:00:00.000Z%27,kind:absolute,to:%272019-08-28T13:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))`
     );
   });
 
@@ -146,7 +147,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostSingleHostNullKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/hosts/siem-windows/anomalies?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/siem-windows/anomalies?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -154,7 +155,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostSingleHostKqlQueryVariable);
     cy.url().should(
       'include',
-      '/app/security/hosts/siem-windows/anomalies?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/siem-windows/anomalies?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -162,7 +163,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostSingleHostKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/hosts/siem-windows/anomalies?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/siem-windows/anomalies?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -170,7 +171,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostMultiHostNullKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/hosts/anomalies?query=(language:kuery,query:%27(host.name:%20%22siem-windows%22%20or%20host.name:%20%22siem-suricata%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/anomalies?query=(language:kuery,query:%27(host.name:%20%22siem-windows%22%20or%20host.name:%20%22siem-suricata%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -178,7 +179,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostMultiHostKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/hosts/anomalies?query=(language:kuery,query:%27(host.name:%20%22siem-windows%22%20or%20host.name:%20%22siem-suricata%22)%20and%20((process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22))%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/anomalies?query=(language:kuery,query:%27(host.name:%20%22siem-windows%22%20or%20host.name:%20%22siem-suricata%22)%20and%20((process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22))%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -186,7 +187,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostVariableHostNullKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/hosts/anomalies?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/anomalies?timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 
@@ -194,7 +195,7 @@ describe('ml conditional links', () => {
     loginAndWaitForPageWithoutDateRange(mlHostVariableHostKqlQuery);
     cy.url().should(
       'include',
-      '/app/security/hosts/anomalies?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:!(%27auditbeat-*%27))'
+      '/app/security/hosts/anomalies?query=(language:kuery,query:%27(process.name:%20%22conhost.exe%22%20or%20process.name:%20%22sc.exe%22)%27)&timerange=(global:(linkTo:!(timeline),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)),timeline:(linkTo:!(global),timerange:(from:%272019-06-06T06:00:00.000Z%27,kind:absolute,to:%272019-06-07T05:59:59.999Z%27)))&sourcerer=(default:(id:security-solution-default,selectedPatterns:!(%27auditbeat-*%27)))'
     );
   });
 });

x-pack/plugins/security_solution/cypress/integration/timelines/creation.spec.ts

@@ -121,7 +121,6 @@ describe('Create a timeline from a template', () => {
     loginAndWaitForPageWithoutDateRange(TIMELINE_TEMPLATES_URL);
     waitForTimelinesPanelToBeLoaded();
   });
-
   it('Should have the same query and open the timeline modal', () => {
     selectCustomTemplates();
     cy.wait('@timeline', { timeout: 100000 });
@@ -132,5 +131,6 @@ describe('Create a timeline from a template', () => {
     cy.get(TIMELINE_FLYOUT_WRAPPER).should('have.css', 'visibility', 'visible');
     cy.get(TIMELINE_DESCRIPTION).should('have.text', getTimeline().description);
     cy.get(TIMELINE_QUERY).should('have.text', getTimeline().query);
+    closeTimeline();
   });
 });

x-pack/plugins/security_solution/cypress/integration/urls/state.spec.ts

@@ -182,11 +182,10 @@ describe('url state', () => {
     loginAndWaitForPageWithoutDateRange(ABSOLUTE_DATE_RANGE.url);
     kqlSearch('source.ip: "10.142.0.9" {enter}');
     navigateFromHeaderTo(HOSTS);
-
     cy.get(NETWORK).should(
       'have.attr',
       'href',
-      `/app/security/network?query=(language:kuery,query:'source.ip:%20%2210.142.0.9%22%20')&sourcerer=(default:!(\'auditbeat-*\'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2019-08-01T20:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2019-08-01T20:33:29.186Z')))`
+      `/app/security/network?query=(language:kuery,query:'source.ip:%20%2210.142.0.9%22%20')&sourcerer=(default:(id:security-solution-default,selectedPatterns:!('auditbeat-*')))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2019-08-01T20:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2019-08-01T20:33:29.186Z')))`
     );
   });
 
@@ -199,12 +198,12 @@ describe('url state', () => {
     cy.get(HOSTS).should(
       'have.attr',
       'href',
-      `/app/security/hosts?query=(language:kuery,query:'host.name:%20%22siem-kibana%22%20')&sourcerer=(default:!(\'auditbeat-*\'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
+      `/app/security/hosts?query=(language:kuery,query:'host.name:%20%22siem-kibana%22%20')&sourcerer=(default:(id:security-solution-default,selectedPatterns:!('auditbeat-*')))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
     );
     cy.get(NETWORK).should(
       'have.attr',
       'href',
-      `/app/security/network?query=(language:kuery,query:'host.name:%20%22siem-kibana%22%20')&sourcerer=(default:!(\'auditbeat-*\'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
+      `/app/security/network?query=(language:kuery,query:'host.name:%20%22siem-kibana%22%20')&sourcerer=(default:(id:security-solution-default,selectedPatterns:!('auditbeat-*')))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
     );
     cy.get(HOSTS_NAMES).first().should('have.text', 'siem-kibana');
 
@@ -215,21 +214,21 @@ describe('url state', () => {
     cy.get(ANOMALIES_TAB).should(
       'have.attr',
       'href',
-      "/app/security/hosts/siem-kibana/anomalies?sourcerer=(default:!('auditbeat-*'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))&query=(language:kuery,query:'agent.type:%20%22auditbeat%22%20')"
+      "/app/security/hosts/siem-kibana/anomalies?sourcerer=(default:(id:security-solution-default,selectedPatterns:!('auditbeat-*')))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))&query=(language:kuery,query:'agent.type:%20%22auditbeat%22%20')"
     );
     cy.get(BREADCRUMBS)
       .eq(1)
       .should(
         'have.attr',
         'href',
-        `/app/security/hosts?query=(language:kuery,query:'agent.type:%20%22auditbeat%22%20')&sourcerer=(default:!(\'auditbeat-*\'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
+        `/app/security/hosts?query=(language:kuery,query:'agent.type:%20%22auditbeat%22%20')&sourcerer=(default:(id:security-solution-default,selectedPatterns:!('auditbeat-*')))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
       );
     cy.get(BREADCRUMBS)
       .eq(2)
       .should(
         'have.attr',
         'href',
-        `/app/security/hosts/siem-kibana?query=(language:kuery,query:'agent.type:%20%22auditbeat%22%20')&sourcerer=(default:!(\'auditbeat-*\'))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
+        `/app/security/hosts/siem-kibana?query=(language:kuery,query:'agent.type:%20%22auditbeat%22%20')&sourcerer=(default:(id:security-solution-default,selectedPatterns:!('auditbeat-*')))&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')),timeline:(linkTo:!(global),timerange:(from:'2019-08-01T20:03:29.186Z',kind:absolute,to:'2020-01-01T21:33:29.186Z')))`
       );
   });
 

x-pack/plugins/security_solution/cypress/objects/timeline.ts

@@ -87,6 +87,7 @@ export const expectedExportedTimelineTemplate = (
         },
       },
     },
+    dataViewId: timelineTemplateBody.dataViewId,
     dateRange: {
       start: timelineTemplateBody.dateRange?.start,
       end: timelineTemplateBody.dateRange?.end,
@@ -127,6 +128,7 @@ export const expectedExportedTimeline = (timelineResponse: Cypress.Response<Time
       },
     },
     dateRange: { start: timelineBody.dateRange?.start, end: timelineBody.dateRange?.end },
+    dataViewId: timelineBody.dataViewId,
     description: timelineBody.description,
     title: timelineBody.title,
     created: timelineBody.created,

x-pack/plugins/security_solution/cypress/screens/sourcerer.ts

@@ -7,10 +7,10 @@
 
 export const SOURCERER_TRIGGER = '[data-test-subj="sourcerer-trigger"]';
 export const SOURCERER_INPUT =
-  '[data-test-subj="indexPattern-switcher"] [data-test-subj="comboBoxInput"]';
+  '[data-test-subj="sourcerer-combo-box"] [data-test-subj="comboBoxInput"]';
 export const SOURCERER_OPTIONS =
-  '[data-test-subj="comboBoxOptionsList indexPattern-switcher-optionsList"]';
-export const SOURCERER_SAVE_BUTTON = 'button[data-test-subj="add-index"]';
+  '[data-test-subj="comboBoxOptionsList sourcerer-combo-box-optionsList"]';
+export const SOURCERER_SAVE_BUTTON = 'button[data-test-subj="sourcerer-save"]';
 export const SOURCERER_RESET_BUTTON = 'button[data-test-subj="sourcerer-reset"]';
 export const SOURCERER_POPOVER_TITLE = '.euiPopoverTitle';
 export const HOSTS_STAT = '[data-test-subj="stat-hosts"] [data-test-subj="stat-title"]';

x-pack/plugins/security_solution/cypress/tasks/alerts.ts

@@ -102,6 +102,12 @@ export const goToOpenedAlerts = () => {
   cy.get(LOADING_INDICATOR).should('not.exist');
 };
 
+export const refreshAlerts = () => {
+  // ensure we've refetched fields the first time index is defined
+  cy.get(REFRESH_BUTTON).should('have.text', 'Refresh');
+  cy.get(REFRESH_BUTTON).first().click({ force: true });
+};
+
 export const openFirstAlert = () => {
   cy.get(TIMELINE_CONTEXT_MENU_BTN).first().click({ force: true });
   cy.get(OPEN_ALERT_BTN).click();

x-pack/plugins/security_solution/cypress/tasks/login.ts

@@ -10,7 +10,7 @@ import Url, { UrlObject } from 'url';
 
 import { ROLES } from '../../common/test';
 import { TIMELINE_FLYOUT_BODY } from '../screens/timeline';
-import { hostDetailsUrl } from '../urls/navigation';
+import { hostDetailsUrl, LOGOUT_URL } from '../urls/navigation';
 
 /**
  * Credentials in the `kibana.dev.yml` config file will be used to authenticate
@@ -326,3 +326,7 @@ export const waitForPageWithoutDateRange = (url: string, role?: ROLES) => {
   cy.visit(role ? getUrlWithRoute(role, url) : url);
   cy.get('[data-test-subj="headerGlobalNav"]', { timeout: 120000 });
 };
+
+export const logout = () => {
+  cy.visit(LOGOUT_URL);
+};

x-pack/plugins/security_solution/cypress/tasks/privileges.ts

@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { constructUrlWithUser, getEnvAuth } from './login';
+
+interface User {
+  username: string;
+  password: string;
+  description?: string;
+  roles: string[];
+}
+
+interface UserInfo {
+  username: string;
+  full_name: string;
+  email: string;
+}
+
+interface FeaturesPrivileges {
+  [featureId: string]: string[];
+}
+
+interface ElasticsearchIndices {
+  names: string[];
+  privileges: string[];
+}
+
+interface ElasticSearchPrivilege {
+  cluster?: string[];
+  indices?: ElasticsearchIndices[];
+}
+
+interface KibanaPrivilege {
+  spaces: string[];
+  base?: string[];
+  feature?: FeaturesPrivileges;
+}
+
+interface Role {
+  name: string;
+  privileges: {
+    elasticsearch?: ElasticSearchPrivilege;
+    kibana?: KibanaPrivilege[];
+  };
+}
+
+export const secAll: Role = {
+  name: 'sec_all_role',
+  privileges: {
+    elasticsearch: {
+      indices: [
+        {
+          names: ['*'],
+          privileges: ['all'],
+        },
+      ],
+    },
+    kibana: [
+      {
+        feature: {
+          siem: ['all'],
+          securitySolutionCases: ['all'],
+          actions: ['all'],
+          actionsSimulators: ['all'],
+          // TODO: Steph/sourcerer remove once we have our internal saved object client
+          // https://github.com/elastic/security-team/issues/1978
+          indexPatterns: ['read'],
+          savedObjectsManagement: ['read'],
+        },
+        spaces: ['*'],
+      },
+    ],
+  },
+};
+
+export const secAllUser: User = {
+  username: 'sec_all_user',
+  password: 'password',
+  roles: [secAll.name],
+};
+
+export const secReadCasesAll: Role = {
+  name: 'sec_read_cases_all_role',
+  privileges: {
+    elasticsearch: {
+      indices: [
+        {
+          names: ['*'],
+          privileges: ['all'],
+        },
+      ],
+    },
+    kibana: [
+      {
+        feature: {
+          siem: ['read'],
+          securitySolutionCases: ['all'],
+          actions: ['all'],
+          actionsSimulators: ['all'],
+          // TODO: Steph/sourcerer remove once we have our internal saved object client
+          // https://github.com/elastic/security-team/issues/1978
+          indexPatterns: ['read'],
+          savedObjectsManagement: ['read'],
+        },
+        spaces: ['*'],
+      },
+    ],
+  },
+};
+
+export const secReadCasesAllUser: User = {
+  username: 'sec_read_cases_all_user',
+  password: 'password',
+  roles: [secReadCasesAll.name],
+};
+
+const getUserInfo = (user: User): UserInfo => ({
+  username: user.username,
+  full_name: user.username.replace('_', ' '),
+  email: `${user.username}@elastic.co`,
+});
+
+export const createUsersAndRoles = (users: User[], roles: Role[]) => {
+  const envUser = getEnvAuth();
+  for (const role of roles) {
+    cy.log(`Creating role: ${JSON.stringify(role)}`);
+    cy.request({
+      body: role.privileges,
+      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
+      method: 'PUT',
+      url: constructUrlWithUser(envUser, `/api/security/role/${role.name}`),
+    })
+      .its('status')
+      .should('eql', 204);
+  }
+
+  for (const user of users) {
+    const userInfo = getUserInfo(user);
+    cy.log(`Creating user: ${JSON.stringify(user)}`);
+    cy.request({
+      body: {
+        username: user.username,
+        password: user.password,
+        roles: user.roles,
+        full_name: userInfo.full_name,
+        email: userInfo.email,
+      },
+      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
+      method: 'POST',
+      url: constructUrlWithUser(envUser, `/internal/security/users/${user.username}`),
+    })
+      .its('status')
+      .should('eql', 200);
+  }
+};
+
+export const deleteUsersAndRoles = (users: User[], roles: Role[]) => {
+  const envUser = getEnvAuth();
+  for (const user of users) {
+    cy.log(`Deleting user: ${JSON.stringify(user)}`);
+    cy.request({
+      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
+      method: 'DELETE',
+      url: constructUrlWithUser(envUser, `/internal/security/users/${user.username}`),
+      failOnStatusCode: false,
+    })
+      .its('status')
+      .should('oneOf', [204, 404]);
+  }
+
+  for (const role of roles) {
+    cy.log(`Deleting role: ${JSON.stringify(role)}`);
+    cy.request({
+      headers: { 'kbn-xsrf': 'cypress-creds-via-config' },
+      method: 'DELETE',
+      url: constructUrlWithUser(envUser, `/api/security/role/${role.name}`),
+      failOnStatusCode: false,
+    })
+      .its('status')
+      .should('oneOf', [204, 404]);
+  }
+};

x-pack/plugins/security_solution/cypress/urls/navigation.ts

@@ -41,3 +41,4 @@ export const OVERVIEW_URL = '/app/security/overview';
 export const RULE_CREATION = 'app/security/rules/create';
 export const TIMELINES_URL = '/app/security/timelines';
 export const TIMELINE_TEMPLATES_URL = '/app/security/timelines/template';
+export const LOGOUT_URL = '/logout';

x-pack/plugins/security_solution/public/app/home/index.tsx

@@ -16,8 +16,8 @@ import { UseUrlState } from '../../common/components/url_state';
 import { navTabs } from './home_navigations';
 import {
   useInitSourcerer,
-  useSourcererScope,
   getScopeFromPath,
+  useSourcererDataView,
 } from '../../common/containers/sourcerer';
 import { useUpgradeSecurityPackages } from '../../common/hooks/use_upgrade_security_packages';
 import { GlobalHeader } from './global_header';
@@ -38,8 +38,7 @@ const HomePageComponent: React.FC<HomePageProps> = ({
 
   useInitSourcerer(getScopeFromPath(pathname));
 
-  const { browserFields, indexPattern } = useSourcererScope(getScopeFromPath(pathname));
-
+  const { browserFields, indexPattern } = useSourcererDataView(getScopeFromPath(pathname));
   // side effect: this will attempt to upgrade the endpoint package if it is not up to date
   // this will run when a user navigates to the Security Solution app and when they navigate between
   // tabs in the app. This is useful for keeping the endpoint package as up to date as possible until

x-pack/plugins/security_solution/public/app/home/template_wrapper/bottom_bar/index.tsx

@@ -8,25 +8,23 @@
 /* eslint-disable react/display-name */
 
 import React from 'react';
-import { useLocation } from 'react-router-dom';
 import { KibanaPageTemplateProps } from '../../../../../../../../src/plugins/kibana_react/public';
 import { AppLeaveHandler } from '../../../../../../../../src/core/public';
 import { useShowTimeline } from '../../../../common/utils/timeline/use_show_timeline';
-import { useSourcererScope, getScopeFromPath } from '../../../../common/containers/sourcerer';
+import { useSourcererDataView } from '../../../../common/containers/sourcerer';
 import { TimelineId } from '../../../../../common/types/timeline';
 import { AutoSaveWarningMsg } from '../../../../timelines/components/timeline/auto_save_warning';
 import { Flyout } from '../../../../timelines/components/flyout';
 import { useResolveRedirect } from '../../../../common/hooks/use_resolve_redirect';
+import { SourcererScopeName } from '../../../../common/store/sourcerer/model';
 
 export const BOTTOM_BAR_CLASSNAME = 'timeline-bottom-bar';
 
 export const SecuritySolutionBottomBar = React.memo(
   ({ onAppLeave }: { onAppLeave: (handler: AppLeaveHandler) => void }) => {
-    const { pathname } = useLocation();
-
     const [showTimeline] = useShowTimeline();
 
-    const { indicesExist } = useSourcererScope(getScopeFromPath(pathname));
+    const { indicesExist } = useSourcererDataView(SourcererScopeName.timeline);
     useResolveRedirect();
 
     return indicesExist && showTimeline ? (

x-pack/plugins/security_solution/public/cases/components/case_view/helpers.ts

@@ -7,7 +7,7 @@
 
 import { isObject, get, isString, isNumber } from 'lodash';
 import { useMemo } from 'react';
-import { useSourcererScope } from '../../../common/containers/sourcerer';
+import { useSourcererDataView } from '../../../common/containers/sourcerer';
 import { SourcererScopeName } from '../../../common/store/sourcerer/model';
 import { useQueryAlerts } from '../../../detections/containers/detection_engine/alerts/use_query';
 import { Ecs } from '../../../../../cases/common';
@@ -102,7 +102,7 @@ export interface Alert {
   [key: string]: unknown;
 }
 export const useFetchAlertData = (alertIds: string[]): [boolean, Record<string, Ecs>] => {
-  const { selectedPatterns } = useSourcererScope(SourcererScopeName.detections);
+  const { selectedPatterns } = useSourcererDataView(SourcererScopeName.detections);
   const alertsQuery = useMemo(() => buildAlertsQuery(alertIds), [alertIds]);
 
   const { loading: isLoadingAlerts, data: alertsData } = useQueryAlerts<SignalHit, unknown>({

x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx

@@ -21,7 +21,7 @@ import { SecurityPageName } from '../../../app/types';
 import { useKibana } from '../../../common/lib/kibana';
 import { APP_UI_ID } from '../../../../common/constants';
 import { timelineActions } from '../../../timelines/store/timeline';
-import { useSourcererScope } from '../../../common/containers/sourcerer';
+import { useSourcererDataView } from '../../../common/containers/sourcerer';
 import { SourcererScopeName } from '../../../common/store/sourcerer/model';
 import { DetailsPanel } from '../../../timelines/components/side_panel';
 import { InvestigateInTimelineAction } from '../../../detections/components/alerts_table/timeline_actions/investigate_in_timeline_action';
@@ -53,14 +53,16 @@ export interface CaseProps extends Props {
 }
 
 const TimelineDetailsPanel = () => {
-  const { browserFields, docValueFields } = useSourcererScope(SourcererScopeName.detections);
-
+  const { browserFields, docValueFields, runtimeMappings } = useSourcererDataView(
+    SourcererScopeName.detections
+  );
   return (
     <DetailsPanel
       browserFields={browserFields}
       docValueFields={docValueFields}
       entityType="events"
       isFlyoutView
+      runtimeMappings={runtimeMappings}
       timelineId={TimelineId.casePage}
     />
   );
@@ -134,6 +136,7 @@ export const CaseView = React.memo(
         timelineActions.createTimeline({
           id: TimelineId.casePage,
           columns: [],
+          dataViewId: '',
           indexNames: [],
           expandedDetail: {},
           show: false,

x-pack/plugins/security_solution/public/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap

@@ -125,6 +125,7 @@ exports[`DragDropContextWrapper rendering it renders against the snapshot 1`] =
               "packetbeat",
             ],
             "name": "@timestamp",
+            "readFromDocValues": true,
             "searchable": true,
             "type": "date",
           },

x-pack/plugins/security_solution/public/common/components/event_details/__snapshots__/alert_summary_view.test.tsx.snap

@@ -676,7 +676,7 @@ exports[`AlertSummaryView Behavior event code renders additional summary rows 1`
                   <div
                     data-test-subj="more-actions-source.ip"
                     field="source.ip"
-                    items="[object Object]"
+                    items="[object Object],[object Object]"
                     value="185.156.74.3"
                   >
                     Overflow button
@@ -1368,7 +1368,7 @@ exports[`AlertSummaryView Memory event code renders additional summary rows 1`]
                   <div
                     data-test-subj="more-actions-source.ip"
                     field="source.ip"
-                    items="[object Object]"
+                    items="[object Object],[object Object]"
                     value="185.156.74.3"
                   >
                     Overflow button

x-pack/plugins/security_solution/public/common/components/event_details/table/field_name_cell.tsx

@@ -10,7 +10,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiBadge, EuiText, EuiToolTip } from '@elast
 import { isEmpty } from 'lodash';
 import * as i18n from '../translations';
 import { FieldIcon } from '../../../../../../../../src/plugins/kibana_react/public';
-import { IndexPatternField } from '../../../../../../../../src/plugins/data/public';
+import { DataViewField } from '../../../../../../../../src/plugins/data_views/common';
 import { getExampleText } from '../helpers';
 import { BrowserField } from '../../../containers/source';
 import { EventFieldsData } from '../types';
@@ -20,7 +20,7 @@ export interface FieldNameCellProps {
   data: EventFieldsData;
   field: string;
   fieldFromBrowserField: BrowserField;
-  fieldMapping?: IndexPatternField;
+  fieldMapping?: DataViewField;
   scripted?: boolean;
 }
 export const FieldNameCell = React.memo(

x-pack/plugins/security_solution/public/common/components/event_details/table/field_value_cell.test.tsx

@@ -30,6 +30,7 @@ const hostIpData: EventFieldsData = {
   isObjectArray: false,
   name: 'host.ip',
   originalValue: ['127.0.0.1', '::1', '10.1.2.3', '2001:0DB8:AC10:FE01::'],
+  readFromDocValues: false,
   searchable: true,
   type: 'ip',
   values: ['127.0.0.1', '::1', '10.1.2.3', '2001:0DB8:AC10:FE01::'],
@@ -101,6 +102,7 @@ describe('FieldValueCell', () => {
       isObjectArray: false,
       name: 'message',
       originalValue: ['Endpoint network event'],
+      readFromDocValues: false,
       searchable: true,
       type: 'string',
       values: ['Endpoint network event'],
@@ -117,6 +119,7 @@ describe('FieldValueCell', () => {
       format: '',
       indexes: ['auditbeat-*', 'filebeat-*', 'logs-*', 'winlogbeat-*'],
       name: 'message',
+      readFromDocValues: false,
       searchable: true,
       type: 'string',
     };
@@ -156,6 +159,7 @@ describe('FieldValueCell', () => {
       format: '',
       indexes: ['auditbeat-*', 'filebeat-*', 'logs-*', 'winlogbeat-*'],
       name: 'host.ip',
+      readFromDocValues: false,
       searchable: true,
       type: 'ip',
     };

x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx

@@ -16,8 +16,12 @@ import { mockEventViewerResponse, mockEventViewerResponseWithEvents } from './mo
 import { StatefulEventsViewer } from '.';
 import { EventsViewer } from './events_viewer';
 import { defaultHeaders } from './default_headers';
-import { useSourcererScope } from '../../containers/sourcerer';
-import { mockBrowserFields, mockDocValueFields } from '../../containers/source/mock';
+import { useSourcererDataView } from '../../containers/sourcerer';
+import {
+  mockBrowserFields,
+  mockDocValueFields,
+  mockRuntimeMappings,
+} from '../../containers/source/mock';
 import { eventsDefaultModel } from './default_model';
 import { useMountAppended } from '../../utils/use_mount_appended';
 import { inputsModel } from '../../store/inputs';
@@ -91,7 +95,7 @@ jest.mock('../../../timelines/containers', () => ({
 
 jest.mock('../../components/url_state/normalize_time_range.ts');
 
-const mockUseSourcererScope: jest.Mock = useSourcererScope as jest.Mock;
+const mockUseSourcererDataView: jest.Mock = useSourcererDataView as jest.Mock;
 jest.mock('../../containers/sourcerer');
 
 const mockUseResizeObserver: jest.Mock = useResizeObserver as jest.Mock;
@@ -107,6 +111,7 @@ const to = '2019-08-27T22:10:56.794Z';
 const defaultMocks = {
   browserFields: mockBrowserFields,
   docValueFields: mockDocValueFields,
+  runtimeMappings: mockRuntimeMappings,
   indexPattern: mockIndexPattern,
   loading: false,
   selectedPatterns: mockIndexNames,
@@ -139,6 +144,7 @@ const eventsViewerDefaultProps = {
   },
   renderCellValue: DefaultCellRenderer,
   rowRenderers: defaultRowRenderers,
+  runtimeMappings: {},
   start: from,
   sort: [
     {
@@ -169,7 +175,7 @@ describe('EventsViewer', () => {
     mockUseTimelineEvents.mockReset();
   });
   beforeAll(() => {
-    mockUseSourcererScope.mockImplementation(() => defaultMocks);
+    mockUseSourcererDataView.mockImplementation(() => defaultMocks);
   });
 
   describe('event details', () => {
@@ -278,7 +284,7 @@ describe('EventsViewer', () => {
 
   describe('loading', () => {
     beforeAll(() => {
-      mockUseSourcererScope.mockImplementation(() => ({ ...defaultMocks, loading: true }));
+      mockUseSourcererDataView.mockImplementation(() => ({ ...defaultMocks, loading: true }));
     });
     beforeEach(() => {
       mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponse]);

x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx

@@ -11,6 +11,8 @@ import styled from 'styled-components';
 import deepEqual from 'fast-deep-equal';
 
 import { useDispatch } from 'react-redux';
+import { DataViewBase, Filter, Query } from '@kbn/es-query';
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { Direction } from '../../../../common/search_strategy';
 import { BrowserFields, DocValueFields } from '../../containers/source';
 import { useTimelineEvents } from '../../../timelines/containers';
@@ -30,12 +32,7 @@ import {
 import { TimelineRefetch } from '../../../timelines/components/timeline/refetch_timeline';
 import { EventDetailsWidthProvider } from './event_details_width_context';
 import * as i18n from './translations';
-import {
-  Filter,
-  esQuery,
-  IIndexPattern,
-  Query,
-} from '../../../../../../../src/plugins/data/public';
+import { esQuery } from '../../../../../../../src/plugins/data/public';
 import { inputsModel } from '../../store';
 import { ExitFullScreen } from '../exit_full_screen';
 import { useGlobalFullScreen } from '../../containers/use_full_screen';
@@ -123,7 +120,7 @@ interface Props {
   headerFilterGroup?: React.ReactNode;
   id: TimelineId;
   indexNames: string[];
-  indexPattern: IIndexPattern;
+  indexPattern: DataViewBase;
   isLive: boolean;
   isLoadingIndexPattern: boolean;
   itemsPerPage: number;
@@ -133,6 +130,7 @@ interface Props {
   onRuleChange?: () => void;
   renderCellValue: (props: CellValueElementProps) => React.ReactNode;
   rowRenderers: RowRenderer[];
+  runtimeMappings: MappingRuntimeFields;
   start: string;
   sort: Sort[];
   showTotalCount?: boolean;
@@ -162,6 +160,7 @@ const EventsViewerComponent: React.FC<Props> = ({
   query,
   renderCellValue,
   rowRenderers,
+  runtimeMappings,
   start,
   sort,
   showTotalCount = true,
@@ -240,6 +239,7 @@ const EventsViewerComponent: React.FC<Props> = ({
       id,
       indexNames,
       limit: itemsPerPage,
+      runtimeMappings,
       sort: sortField,
       startDate: start,
       endDate: end,

x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx

@@ -22,7 +22,7 @@ import { InspectButtonContainer } from '../inspect';
 import { useGlobalFullScreen } from '../../containers/use_full_screen';
 import { useIsExperimentalFeatureEnabled } from '../../hooks/use_experimental_features';
 import { SourcererScopeName } from '../../store/sourcerer/model';
-import { useSourcererScope } from '../../containers/sourcerer';
+import { useSourcererDataView } from '../../containers/sourcerer';
 import type { EntityType } from '../../../../../timelines/common';
 import { TGridCellAction } from '../../../../../timelines/common/types';
 import { DetailsPanel } from '../../../timelines/components/side_panel';
@@ -117,9 +117,12 @@ const StatefulEventsViewerComponent: React.FC<Props> = ({
     browserFields,
     docValueFields,
     indexPattern,
+    runtimeMappings,
     selectedPatterns,
+    dataViewId: selectedDataViewId,
     loading: isLoadingIndexPattern,
-  } = useSourcererScope(scopeId);
+  } = useSourcererDataView(scopeId);
+
   const { globalFullScreen } = useGlobalFullScreen();
   // TODO: Once we are past experimental phase this code should be removed
   const tGridEnabled = useIsExperimentalFeatureEnabled('tGridEnabled');
@@ -129,14 +132,15 @@ const StatefulEventsViewerComponent: React.FC<Props> = ({
   useEffect(() => {
     if (createTimeline != null) {
       createTimeline({
-        id,
         columns,
+        dataViewId: selectedDataViewId,
         defaultColumns,
         excludedRowRendererIds,
+        id,
         indexNames: selectedPatterns,
-        sort,
         itemsPerPage,
         showCheckboxes,
+        sort,
       });
     }
     return () => {
@@ -206,6 +210,7 @@ const StatefulEventsViewerComponent: React.FC<Props> = ({
               query,
               renderCellValue,
               rowRenderers,
+              runtimeMappings,
               setQuery,
               sort,
               start,
@@ -235,6 +240,7 @@ const StatefulEventsViewerComponent: React.FC<Props> = ({
               onRuleChange={onRuleChange}
               renderCellValue={renderCellValue}
               rowRenderers={rowRenderers}
+              runtimeMappings={runtimeMappings}
               start={start}
               sort={sort}
               showTotalCount={isEmpty(graphEventId) ? true : false}
@@ -249,6 +255,7 @@ const StatefulEventsViewerComponent: React.FC<Props> = ({
         entityType={entityType}
         docValueFields={docValueFields}
         isFlyoutView
+        runtimeMappings={runtimeMappings}
         timelineId={id}
       />
     </>

x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx

@@ -42,13 +42,13 @@ import { getCommentsArrayMock } from '../../../../../lists/common/schemas/types/
 import { fields } from '../../../../../../../src/plugins/data/common/mocks';
 import { ENTRIES, OLD_DATE_RELATIVE_TO_DATE_NOW } from '../../../../../lists/common/constants.mock';
 import { CodeSignature } from '../../../../common/ecs/file';
-import { IndexPatternBase } from '@kbn/es-query';
+import { DataViewBase } from '@kbn/es-query';
 
 jest.mock('uuid', () => ({
   v4: jest.fn().mockReturnValue('123'),
 }));
 
-const getMockIndexPattern = (): IndexPatternBase => ({
+const getMockIndexPattern = (): DataViewBase => ({
   fields,
   id: '1234',
   title: 'logstash-*',
@@ -364,7 +364,7 @@ describe('Exception helpers', () => {
           name: 'nested.field',
         },
       ],
-    } as IndexPatternBase;
+    } as DataViewBase;
 
     test('it should return false with an empty array', () => {
       const payload: ExceptionListItemSchema[] = [];

x-pack/plugins/security_solution/public/common/components/exceptions/helpers.tsx

@@ -33,7 +33,7 @@ import {
   addIdToEntries,
   ExceptionsBuilderExceptionItem,
 } from '@kbn/securitysolution-list-utils';
-import { IndexPatternBase } from '@kbn/es-query';
+import { DataViewBase } from '@kbn/es-query';
 import * as i18n from './translations';
 import { AlertData, Flattened } from './types';
 
@@ -46,10 +46,10 @@ import exceptionableEndpointFields from './exceptionable_endpoint_fields.json';
 import exceptionableEndpointEventFields from './exceptionable_endpoint_event_fields.json';
 
 export const filterIndexPatterns = (
-  patterns: IndexPatternBase,
+  patterns: DataViewBase,
   type: ExceptionListType,
   osTypes?: OsTypeArray
-): IndexPatternBase => {
+): DataViewBase => {
   switch (type) {
     case 'endpoint':
       const osFilterForEndpoint: (name: string) => boolean = osTypes?.includes('linux')
@@ -752,7 +752,7 @@ export const getPrepopulatedBehaviorException = ({
  */
 export const entryHasNonEcsType = (
   exceptionItems: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>,
-  indexPatterns: IndexPatternBase
+  indexPatterns: DataViewBase
 ): boolean => {
   const doesFieldNameExist = (exceptionEntry: Entry): boolean => {
     return indexPatterns.fields.some(({ name }) => name === exceptionEntry.field);

x-pack/plugins/security_solution/public/common/components/hover_actions/actions/show_top_n.tsx

@@ -17,7 +17,7 @@ import { i18n } from '@kbn/i18n';
 import { StatefulTopN } from '../../top_n';
 import { TimelineId } from '../../../../../common/types/timeline';
 import { SourcererScopeName } from '../../../store/sourcerer/model';
-import { useSourcererScope } from '../../../containers/sourcerer';
+import { useSourcererDataView } from '../../../containers/sourcerer';
 import { TooltipWithKeyboardShortcut } from '../../accessibility';
 import { getAdditionalScreenReaderOnlyContext } from '../utils';
 import { SHOW_TOP_N_KEYBOARD_SHORTCUT } from '../keyboard_shortcut_constants';
@@ -85,7 +85,8 @@ export const ShowTopNButton: React.FC<Props> = React.memo(
           )
         ? SourcererScopeName.detections
         : SourcererScopeName.default;
-    const { browserFields, indexPattern } = useSourcererScope(activeScope);
+    const { browserFields, indexPattern } = useSourcererDataView(activeScope);
+
     const icon = iconType ?? 'visBarVertical';
     const side = iconSide ?? 'left';
     const buttonTitle = title ?? SHOW_TOP(field);

x-pack/plugins/security_solution/public/common/components/hover_actions/use_hover_action_items.test.tsx

@@ -13,7 +13,7 @@ import { DataProvider } from '../../../../common/types/timeline';
 jest.mock('../../lib/kibana');
 jest.mock('../../hooks/use_selector');
 jest.mock('../../containers/sourcerer', () => ({
-  useSourcererScope: jest.fn().mockReturnValue({ browserFields: {} }),
+  useSourcererDataView: jest.fn().mockReturnValue({ browserFields: {} }),
 }));
 
 describe('useHoverActionItems', () => {

x-pack/plugins/security_solution/public/common/components/hover_actions/use_hover_action_items.tsx

@@ -17,7 +17,7 @@ import { allowTopN } from '../drag_and_drop/helpers';
 import { useDeepEqualSelector } from '../../hooks/use_selector';
 import { ColumnHeaderOptions, DataProvider, TimelineId } from '../../../../common/types/timeline';
 import { SourcererScopeName } from '../../store/sourcerer/model';
-import { useSourcererScope } from '../../containers/sourcerer';
+import { useSourcererDataView } from '../../containers/sourcerer';
 import { timelineSelectors } from '../../../timelines/store/timeline';
 import { ShowTopNButton } from './actions/show_top_n';
 import { FilterManager } from '../../../../../../../src/plugins/data/public';
@@ -116,7 +116,7 @@ export const useHoverActionItems = ({
         )
       ? SourcererScopeName.detections
       : SourcererScopeName.default;
-  const { browserFields } = useSourcererScope(activeScope);
+  const { browserFields } = useSourcererDataView(activeScope);
 
   /*
    * In the case of `DisableOverflowButton`, we show filters only when topN is NOT opened. As after topN button is clicked, the chart panel replace current hover actions in the hover actions' popover, so we have to hide all the actions.

x-pack/plugins/security_solution/public/common/components/navigation/helpers.ts

@@ -20,7 +20,7 @@ import {
 import { Query, Filter } from '../../../../../../../src/plugins/data/public';
 
 import { SearchNavTab } from './types';
-import { SourcererScopePatterns } from '../../store/sourcerer/model';
+import { SourcererUrlState } from '../../store/sourcerer/model';
 
 export const getSearch = (tab: SearchNavTab, urlState: UrlState): string => {
   if (tab && tab.urlKey != null && !isAdministration(tab.urlKey)) {
@@ -29,7 +29,7 @@ export const getSearch = (tab: SearchNavTab, urlState: UrlState): string => {
         let urlStateToReplace:
           | Filter[]
           | Query
-          | SourcererScopePatterns
+          | SourcererUrlState
           | TimelineUrl
           | UrlInputsModel
           | string = '';

x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/types.ts

@@ -7,7 +7,7 @@
 
 import { UrlInputsModel } from '../../../store/inputs/model';
 import { CONSTANTS } from '../../url_state/constants';
-import { SourcererScopePatterns } from '../../../store/sourcerer/model';
+import { SourcererUrlState } from '../../../store/sourcerer/model';
 import { TimelineUrl } from '../../../../timelines/store/timeline/model';
 import { Filter, Query } from '../../../../../../../../src/plugins/data/public';
 
@@ -21,7 +21,7 @@ export interface TabNavigationProps extends SecuritySolutionTabNavigationProps {
   [CONSTANTS.appQuery]?: Query;
   [CONSTANTS.filters]?: Filter[];
   [CONSTANTS.savedQuery]?: string;
-  [CONSTANTS.sourcerer]: SourcererScopePatterns;
+  [CONSTANTS.sourcerer]: SourcererUrlState;
   [CONSTANTS.timerange]: UrlInputsModel;
   [CONSTANTS.timeline]: TimelineUrl;
 }

x-pack/plugins/security_solution/public/common/components/query_bar/index.tsx

@@ -8,11 +8,9 @@
 import React, { memo, useMemo, useCallback } from 'react';
 import deepEqual from 'fast-deep-equal';
 
+import { DataViewBase, Filter, Query } from '@kbn/es-query';
 import {
-  Filter,
-  IIndexPattern,
   FilterManager,
-  Query,
   TimeHistory,
   TimeRange,
   SavedQuery,
@@ -26,7 +24,7 @@ export interface QueryBarComponentProps {
   dateRangeFrom?: string;
   dateRangeTo?: string;
   hideSavedQuery?: boolean;
-  indexPattern: IIndexPattern;
+  indexPattern: DataViewBase;
   isLoading?: boolean;
   isRefreshPaused?: boolean;
   filterQuery: Query;

x-pack/plugins/security_solution/public/common/components/search_bar/index.test.tsx

@@ -6,13 +6,42 @@
  */
 
 import React from 'react';
-import { mount } from 'enzyme';
-
+import { render, fireEvent } from '@testing-library/react';
 import { InputsModelId } from '../../store/inputs/constants';
 import { SearchBarComponent } from '.';
 import { TestProviders } from '../../mock';
+import { FilterManager } from '../../../../../../../src/plugins/data/public';
+import { coreMock } from '../../../../../../../src/core/public/mocks';
 
-jest.mock('../../lib/kibana');
+const mockFilterManager = new FilterManager(coreMock.createStart().uiSettings);
+jest.mock('../../lib/kibana', () => {
+  const original = jest.requireActual('../../lib/kibana');
+  return {
+    useKibana: () => ({
+      services: {
+        ...original.useKibana().services,
+        data: {
+          ...original.useKibana().services.data,
+          query: {
+            ...original.useKibana().services.data.query,
+            filterManager: mockFilterManager,
+          },
+          ui: {
+            SearchBar: jest.fn().mockImplementation((props) => (
+              <button
+                data-test-subj="querySubmitButton"
+                onClick={() => props.onQuerySubmit({ dateRange: { from: 'now', to: 'now' } })}
+                type="button"
+              >
+                {'Hello world'}
+              </button>
+            )),
+          },
+        },
+      },
+    }),
+  };
+});
 
 describe('SearchBarComponent', () => {
   const props = {
@@ -37,9 +66,38 @@ describe('SearchBarComponent', () => {
     savedQuery: undefined,
   };
 
+  const pollForSignalIndex = jest.fn();
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
   it('calls setSearchBarFilter on mount', () => {
-    mount(<SearchBarComponent {...props} />, { wrappingComponent: TestProviders });
+    render(
+      <TestProviders>
+        <SearchBarComponent {...props} />
+      </TestProviders>
+    );
 
     expect(props.setSearchBarFilter).toHaveBeenCalled();
   });
+
+  it('calls pollForSignalIndex on Refresh button click', () => {
+    const { getByTestId } = render(
+      <TestProviders>
+        <SearchBarComponent {...props} pollForSignalIndex={pollForSignalIndex} />
+      </TestProviders>
+    );
+    fireEvent.click(getByTestId('querySubmitButton'));
+    expect(pollForSignalIndex).toHaveBeenCalled();
+  });
+
+  it('does not call pollForSignalIndex on Refresh button click if pollForSignalIndex not passed', () => {
+    const { getByTestId } = render(
+      <TestProviders>
+        <SearchBarComponent {...props} />
+      </TestProviders>
+    );
+    fireEvent.click(getByTestId('querySubmitButton'));
+    expect(pollForSignalIndex).not.toHaveBeenCalled();
+  });
 });

x-pack/plugins/security_solution/public/common/components/search_bar/index.tsx

@@ -13,14 +13,9 @@ import { Dispatch } from 'redux';
 import { Subscription } from 'rxjs';
 import styled from 'styled-components';
 import deepEqual from 'fast-deep-equal';
-import {
-  FilterManager,
-  IIndexPattern,
-  TimeRange,
-  Query,
-  Filter,
-  SavedQuery,
-} from 'src/plugins/data/public';
+
+import { DataViewBase, Filter, Query } from '@kbn/es-query';
+import { FilterManager, TimeRange, SavedQuery } from 'src/plugins/data/public';
 
 import { OnTimeChangeProps } from '@elastic/eui';
 
@@ -48,7 +43,8 @@ const APP_STATE_STORAGE_KEY = 'securitySolution.searchBar.appState';
 
 interface SiemSearchBarProps {
   id: InputsModelId;
-  indexPattern: IIndexPattern;
+  indexPattern: DataViewBase;
+  pollForSignalIndex?: () => void;
   timelineId?: string;
   dataTestSubj?: string;
 }
@@ -67,6 +63,7 @@ export const SearchBarComponent = memo<SiemSearchBarProps & PropsFromRedux>(
     id,
     indexPattern,
     isLoading = false,
+    pollForSignalIndex,
     queries,
     savedQuery,
     setSavedQuery,
@@ -100,6 +97,11 @@ export const SearchBarComponent = memo<SiemSearchBarProps & PropsFromRedux>(
 
     const onQuerySubmit = useCallback(
       (payload: { dateRange: TimeRange; query?: Query }) => {
+        // if the function is there, call it to check if the signals index exists yet
+        // in order to update the index fields
+        if (pollForSignalIndex != null) {
+          pollForSignalIndex();
+        }
         const isQuickSelection =
           payload.dateRange.from.includes('now') || payload.dateRange.to.includes('now');
         let updateSearchBar: UpdateReduxSearchBar = {
@@ -144,7 +146,18 @@ export const SearchBarComponent = memo<SiemSearchBarProps & PropsFromRedux>(
 
         window.setTimeout(() => updateSearch(updateSearchBar), 0);
       },
-      [id, toStr, end, fromStr, start, filterManager, filterQuery, queries, updateSearch]
+      [
+        id,
+        pollForSignalIndex,
+        toStr,
+        end,
+        fromStr,
+        start,
+        filterManager,
+        filterQuery,
+        queries,
+        updateSearch,
+      ]
     );
 
     const onRefresh = useCallback(

x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx

@@ -6,13 +6,9 @@
  */
 
 import React from 'react';
-import { mount } from 'enzyme';
-import { waitFor } from '@testing-library/react';
-
-import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
+import { mount, ReactWrapper } from 'enzyme';
 import { SourcererScopeName } from '../../store/sourcerer/model';
 import { Sourcerer } from './index';
-import { DEFAULT_INDEX_PATTERN } from '../../../../common/constants';
 import { sourcererActions, sourcererModel } from '../../store/sourcerer';
 import {
   createSecuritySolutionStorageMock,
@@ -22,6 +18,7 @@ import {
   TestProviders,
 } from '../../mock';
 import { createStore, State } from '../../store';
+import { EuiSuperSelectOption } from '@elastic/eui/src/components/form/super_select/super_select_control';
 
 const mockDispatch = jest.fn();
 jest.mock('react-redux', () => {
@@ -49,16 +46,28 @@ const defaultProps = {
 };
 
 describe('Sourcerer component', () => {
-  beforeEach(() => {
-    jest.clearAllMocks();
-    jest.restoreAllMocks();
-  });
   const state: State = mockGlobalState;
+  const { id, patternList, title } = state.sourcerer.defaultDataView;
+  const patternListNoSignals = patternList
+    .filter((p) => p !== state.sourcerer.signalIndexName)
+    .sort();
+  const checkOptionsAndSelections = (wrapper: ReactWrapper, patterns: string[]) => ({
+    availableOptionCount: wrapper.find(`[data-test-subj="sourcerer-combo-option"]`).length,
+    optionsSelected: patterns.every((pattern) =>
+      wrapper
+        .find(`[data-test-subj="sourcerer-combo-box"] span[title="${pattern}"]`)
+        .first()
+        .exists()
+    ),
+  });
+
   const { storage } = createSecuritySolutionStorageMock();
   let store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
 
   beforeEach(() => {
     store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    jest.clearAllMocks();
+    jest.restoreAllMocks();
   });
 
   it('renders tooltip', () => {
@@ -99,72 +108,43 @@ describe('Sourcerer component', () => {
     );
     wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
     expect(
-      wrapper.find(`[data-test-subj="indexPattern-switcher"]`).first().prop('selectedOptions')
-    ).toEqual(mockOptions);
-  });
-  it('Mounts with some options selected', () => {
-    const state2 = {
-      ...mockGlobalState,
-      sourcerer: {
-        ...mockGlobalState.sourcerer,
-        sourcererScopes: {
-          ...mockGlobalState.sourcerer.sourcererScopes,
-          [SourcererScopeName.default]: {
-            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
-            loading: false,
-            selectedPatterns: [DEFAULT_INDEX_PATTERN[0]],
-          },
-        },
-      },
-    };
-
-    store = createStore(state2, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
-    const wrapper = mount(
-      <TestProviders store={store}>
-        <Sourcerer {...defaultProps} />
-      </TestProviders>
-    );
-    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
-    expect(
-      wrapper.find(`[data-test-subj="indexPattern-switcher"]`).first().prop('selectedOptions')
-    ).toEqual([mockOptions[0]]);
-  });
-  it('onChange calls updateSourcererScopeIndices', async () => {
-    const wrapper = mount(
-      <TestProviders store={store}>
-        <Sourcerer {...defaultProps} />
-      </TestProviders>
-    );
-    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
-    expect(
-      wrapper.find(`[data-test-subj="sourcerer-popover"]`).first().prop('isOpen')
-    ).toBeTruthy();
-    await waitFor(() => {
-      (
-        wrapper.find(EuiComboBox).props() as unknown as {
-          onChange: (a: EuiComboBoxOptionOption[]) => void;
-        }
-      ).onChange([mockOptions[0], mockOptions[1]]);
-      wrapper.update();
-    });
-    wrapper.find(`[data-test-subj="add-index"]`).first().simulate('click');
-    expect(wrapper.find(`[data-test-subj="sourcerer-popover"]`).first().prop('isOpen')).toBeFalsy();
-
-    expect(mockDispatch).toHaveBeenCalledWith(
-      sourcererActions.setSelectedIndexPatterns({
-        id: SourcererScopeName.default,
-        selectedPatterns: [mockOptions[0].value, mockOptions[1].value],
-      })
+      wrapper.find(`[data-test-subj="sourcerer-combo-box"]`).first().prop('selectedOptions')
+    ).toEqual(
+      patternListNoSignals.map((p) => ({
+        label: p,
+        value: p,
+      }))
     );
   });
-  it('resets to config index patterns', async () => {
+  it('Removes duplicate options from title', () => {
     store = createStore(
       {
         ...state,
         sourcerer: {
           ...state.sourcerer,
-          kibanaIndexPatterns: [{ id: '1234', title: 'auditbeat-*' }],
-          configIndexPatterns: ['packetbeat-*'],
+          defaultDataView: {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'filebeat-*,auditbeat-*,auditbeat-*,auditbeat-*,auditbeat-*',
+            patternList: ['filebeat-*', 'auditbeat-*'],
+          },
+          kibanaDataViews: [
+            {
+              ...state.sourcerer.defaultDataView,
+              id: '1234',
+              title: 'filebeat-*,auditbeat-*,auditbeat-*,auditbeat-*,auditbeat-*',
+              patternList: ['filebeat-*', 'auditbeat-*'],
+            },
+          ],
+          sourcererScopes: {
+            ...state.sourcerer.sourcererScopes,
+            [SourcererScopeName.default]: {
+              ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+              loading: false,
+              selectedDataViewId: '1234',
+              selectedPatterns: ['filebeat-*'],
+            },
+          },
         },
       },
       SUB_PLUGINS_REDUCER,
@@ -177,16 +157,252 @@ describe('Sourcerer component', () => {
       </TestProviders>
     );
     wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
-    expect(wrapper.find(`[data-test-subj="config-option"]`).first().exists()).toBeFalsy();
+    wrapper
+      .find(`[data-test-subj="sourcerer-combo-box"] [data-test-subj="comboBoxToggleListButton"]`)
+      .first()
+      .simulate('click');
+    const options: Array<EuiSuperSelectOption<string>> = wrapper
+      .find(`[data-test-subj="sourcerer-combo-box"]`)
+      .first()
+      .prop('options');
+    expect(options.length).toEqual(2);
+  });
+  it('Disables options with no data', () => {
+    store = createStore(
+      {
+        ...state,
+        sourcerer: {
+          ...state.sourcerer,
+          defaultDataView: {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'filebeat-*,auditbeat-*,fakebeat-*',
+            patternList: ['filebeat-*', 'auditbeat-*'],
+          },
+          kibanaDataViews: [
+            {
+              ...state.sourcerer.defaultDataView,
+              id: '1234',
+              title: 'filebeat-*,auditbeat-*,fakebeat-*',
+              patternList: ['filebeat-*', 'auditbeat-*'],
+            },
+          ],
+          sourcererScopes: {
+            ...state.sourcerer.sourcererScopes,
+            [SourcererScopeName.default]: {
+              ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+              loading: false,
+              selectedDataViewId: '1234',
+              selectedPatterns: ['filebeat-*'],
+            },
+          },
+        },
+      },
+      SUB_PLUGINS_REDUCER,
+      kibanaObservable,
+      storage
+    );
+    const wrapper = mount(
+      <TestProviders store={store}>
+        <Sourcerer {...defaultProps} />
+      </TestProviders>
+    );
+    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
+    wrapper
+      .find(`[data-test-subj="sourcerer-combo-box"] [data-test-subj="comboBoxToggleListButton"]`)
+      .first()
+      .simulate('click');
+    const options: Array<EuiSuperSelectOption<string>> = wrapper
+      .find(`[data-test-subj="sourcerer-combo-box"]`)
+      .first()
+      .prop('options');
+    const disabledOption = options.find((o) => o.disabled);
+    expect(disabledOption?.value).toEqual('fakebeat-*');
+  });
+  it('Mounts with multiple options selected - default', () => {
+    const state2 = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        kibanaDataViews: [
+          state.sourcerer.defaultDataView,
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'auditbeat-*',
+            patternList: ['auditbeat-*'],
+          },
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '12347',
+            title: 'packetbeat-*',
+            patternList: ['packetbeat-*'],
+          },
+        ],
+        sourcererScopes: {
+          ...mockGlobalState.sourcerer.sourcererScopes,
+          [SourcererScopeName.default]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+            loading: false,
+            patternList,
+            selectedDataViewId: id,
+            selectedPatterns: patternList.slice(0, 2),
+          },
+        },
+      },
+    };
+
+    store = createStore(state2, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    const wrapper = mount(
+      <TestProviders store={store}>
+        <Sourcerer {...defaultProps} />
+      </TestProviders>
+    );
+    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
+    wrapper.find(`[data-test-subj="comboBoxInput"]`).first().simulate('click');
+    expect(checkOptionsAndSelections(wrapper, patternList.slice(0, 2))).toEqual({
+      // should hide signal index
+      availableOptionCount: title.split(',').length - 3,
+      optionsSelected: true,
+    });
+  });
+  it('Mounts with multiple options selected - timeline', () => {
+    const state2 = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        kibanaDataViews: [
+          state.sourcerer.defaultDataView,
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'auditbeat-*',
+            patternList: ['auditbeat-*'],
+          },
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '12347',
+            title: 'packetbeat-*',
+            patternList: ['packetbeat-*'],
+          },
+        ],
+        sourcererScopes: {
+          ...mockGlobalState.sourcerer.sourcererScopes,
+          [SourcererScopeName.timeline]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline],
+            loading: false,
+            patternList,
+            selectedDataViewId: id,
+            selectedPatterns: patternList.slice(0, 2),
+          },
+        },
+      },
+    };
+
+    store = createStore(state2, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    const wrapper = mount(
+      <TestProviders store={store}>
+        <Sourcerer scope={sourcererModel.SourcererScopeName.timeline} />
+      </TestProviders>
+    );
+    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
+    wrapper.find(`[data-test-subj="comboBoxInput"]`).first().simulate('click');
+    expect(checkOptionsAndSelections(wrapper, patternList.slice(0, 2))).toEqual({
+      // should show every option except fakebeat-*
+      availableOptionCount: title.split(',').length - 2,
+      optionsSelected: true,
+    });
+  });
+  it('onSave dispatches setSelectedDataView', async () => {
+    store = createStore(
+      {
+        ...state,
+        sourcerer: {
+          ...state.sourcerer,
+          kibanaDataViews: [
+            state.sourcerer.defaultDataView,
+            {
+              ...state.sourcerer.defaultDataView,
+              id: '1234',
+              title: 'filebeat-*',
+              patternList: ['filebeat-*'],
+            },
+          ],
+          sourcererScopes: {
+            ...state.sourcerer.sourcererScopes,
+            [SourcererScopeName.default]: {
+              ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+              loading: false,
+              selectedDataViewId: id,
+              selectedPatterns: patternList.slice(0, 2),
+            },
+          },
+        },
+      },
+      SUB_PLUGINS_REDUCER,
+      kibanaObservable,
+      storage
+    );
+    const wrapper = mount(
+      <TestProviders store={store}>
+        <Sourcerer {...defaultProps} />
+      </TestProviders>
+    );
+    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
+    wrapper.find(`[data-test-subj="comboBoxInput"]`).first().simulate('click');
+    expect(checkOptionsAndSelections(wrapper, patternList.slice(0, 2))).toEqual({
+      availableOptionCount: title.split(',').length - 3,
+      optionsSelected: true,
+    });
+
+    wrapper.find(`[data-test-subj="sourcerer-combo-option"]`).first().simulate('click');
+    expect(checkOptionsAndSelections(wrapper, patternList.slice(0, 3))).toEqual({
+      availableOptionCount: title.split(',').length - 4,
+      optionsSelected: true,
+    });
+    wrapper.find(`[data-test-subj="sourcerer-save"]`).first().simulate('click');
+    expect(wrapper.find(`[data-test-subj="sourcerer-popover"]`).first().prop('isOpen')).toBeFalsy();
+
+    expect(mockDispatch).toHaveBeenCalledWith(
+      sourcererActions.setSelectedDataView({
+        id: SourcererScopeName.default,
+        selectedDataViewId: id,
+        selectedPatterns: patternList.slice(0, 3),
+      })
+    );
+  });
+  it('resets to default index pattern', async () => {
+    const wrapper = mount(
+      <TestProviders store={store}>
+        <Sourcerer {...defaultProps} />
+      </TestProviders>
+    );
+    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
+    wrapper.find(`[data-test-subj="comboBoxInput"]`).first().simulate('click');
+
+    expect(checkOptionsAndSelections(wrapper, patternListNoSignals)).toEqual({
+      availableOptionCount: 1,
+      optionsSelected: true,
+    });
+
     wrapper
       .find(
-        `[data-test-subj="indexPattern-switcher"] [title="packetbeat-*"] button.euiBadge__iconButton`
+        `[data-test-subj="sourcerer-combo-box"] [title="${patternList[0]}"] button.euiBadge__iconButton`
       )
       .first()
       .simulate('click');
-    expect(wrapper.find(`[data-test-subj="config-option"]`).first().exists()).toBeTruthy();
+    expect(
+      checkOptionsAndSelections(wrapper, patternListNoSignals.slice(1, patternListNoSignals.length))
+    ).toEqual({
+      availableOptionCount: 2,
+      optionsSelected: true,
+    });
+
     wrapper.find(`[data-test-subj="sourcerer-reset"]`).first().simulate('click');
-    expect(wrapper.find(`[data-test-subj="config-option"]`).first().exists()).toBeFalsy();
+    expect(checkOptionsAndSelections(wrapper, patternListNoSignals)).toEqual({
+      availableOptionCount: 1,
+      optionsSelected: true,
+    });
   });
   it('disables saving when no index patterns are selected', () => {
     store = createStore(
@@ -194,7 +410,15 @@ describe('Sourcerer component', () => {
         ...state,
         sourcerer: {
           ...state.sourcerer,
-          kibanaIndexPatterns: [{ id: '1234', title: 'auditbeat-*' }],
+          kibanaDataViews: [
+            state.sourcerer.defaultDataView,
+            {
+              ...state.sourcerer.defaultDataView,
+              id: '1234',
+              title: 'auditbeat-*',
+              patternList: ['auditbeat-*'],
+            },
+          ],
         },
       },
       SUB_PLUGINS_REDUCER,
@@ -208,78 +432,152 @@ describe('Sourcerer component', () => {
     );
     wrapper.find('[data-test-subj="sourcerer-trigger"]').first().simulate('click');
     wrapper.find('[data-test-subj="comboBoxClearButton"]').first().simulate('click');
-    expect(wrapper.find('[data-test-subj="add-index"]').first().prop('disabled')).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="sourcerer-save"]').first().prop('disabled')).toBeTruthy();
   });
-  it('returns index pattern options for kibanaIndexPatterns and configIndexPatterns', () => {
-    store = createStore(
-      {
-        ...state,
-        sourcerer: {
-          ...state.sourcerer,
-          kibanaIndexPatterns: [{ id: '1234', title: 'auditbeat-*' }],
-          configIndexPatterns: ['packetbeat-*'],
+  it('Selects a different index pattern', async () => {
+    const state2 = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        kibanaDataViews: [
+          state.sourcerer.defaultDataView,
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'fakebeat-*,neatbeat-*',
+            patternList: ['fakebeat-*'],
+          },
+        ],
+        sourcererScopes: {
+          ...mockGlobalState.sourcerer.sourcererScopes,
+          [SourcererScopeName.default]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+            loading: false,
+            patternList,
+            selectedDataViewId: id,
+            selectedPatterns: patternList.slice(0, 2),
+          },
         },
       },
-      SUB_PLUGINS_REDUCER,
-      kibanaObservable,
-      storage
-    );
+    };
+
+    store = createStore(state2, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
     const wrapper = mount(
       <TestProviders store={store}>
         <Sourcerer {...defaultProps} />
       </TestProviders>
     );
     wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
-    expect(wrapper.find(`[data-test-subj="config-option"]`).first().exists()).toBeFalsy();
-    wrapper
-      .find(
-        `[data-test-subj="indexPattern-switcher"] [title="auditbeat-*"] button.euiBadge__iconButton`
-      )
-      .first()
-      .simulate('click');
-    wrapper.update();
-    expect(wrapper.find(`[data-test-subj="kip-option"]`).first().text()).toEqual(' auditbeat-*');
-    wrapper
-      .find(
-        `[data-test-subj="indexPattern-switcher"] [title="packetbeat-*"] button.euiBadge__iconButton`
-      )
-      .first()
-      .simulate('click');
-    wrapper.update();
-    expect(wrapper.find(`[data-test-subj="config-option"]`).first().text()).toEqual('packetbeat-*');
+    wrapper.find(`button[data-test-subj="sourcerer-select"]`).first().simulate('click');
+
+    wrapper.find(`[data-test-subj="dataView-option-super"]`).first().simulate('click');
+    expect(checkOptionsAndSelections(wrapper, ['fakebeat-*'])).toEqual({
+      availableOptionCount: 0,
+      optionsSelected: true,
+    });
+    wrapper.find(`[data-test-subj="sourcerer-save"]`).first().simulate('click');
+
+    expect(mockDispatch).toHaveBeenCalledWith(
+      sourcererActions.setSelectedDataView({
+        id: SourcererScopeName.default,
+        selectedDataViewId: '1234',
+        selectedPatterns: ['fakebeat-*'],
+      })
+    );
   });
-  it('combines index pattern options for kibanaIndexPatterns and configIndexPatterns', () => {
-    store = createStore(
-      {
-        ...state,
-        sourcerer: {
-          ...state.sourcerer,
-          kibanaIndexPatterns: [
-            { id: '1234', title: 'auditbeat-*' },
-            { id: '5678', title: 'packetbeat-*' },
-          ],
-          configIndexPatterns: ['packetbeat-*'],
+  it('Does display signals index on timeline sourcerer', () => {
+    const state2 = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        kibanaDataViews: [
+          state.sourcerer.defaultDataView,
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'auditbeat-*',
+            patternList: ['auditbeat-*'],
+          },
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '12347',
+            title: 'packetbeat-*',
+            patternList: ['packetbeat-*'],
+          },
+        ],
+        sourcererScopes: {
+          ...mockGlobalState.sourcerer.sourcererScopes,
+          [SourcererScopeName.timeline]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline],
+            loading: false,
+            patternList,
+            selectedDataViewId: id,
+            selectedPatterns: patternList.slice(0, 2),
+          },
         },
       },
-      SUB_PLUGINS_REDUCER,
-      kibanaObservable,
-      storage
+    };
+
+    store = createStore(state2, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    const wrapper = mount(
+      <TestProviders store={store}>
+        <Sourcerer scope={sourcererModel.SourcererScopeName.timeline} />
+      </TestProviders>
     );
+    wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
+    wrapper.find(`[data-test-subj="comboBoxToggleListButton"]`).first().simulate('click');
+    expect(wrapper.find(`[data-test-subj="sourcerer-combo-option"]`).at(6).text()).toEqual(
+      mockGlobalState.sourcerer.signalIndexName
+    );
+  });
+  it('Does not display signals index on default sourcerer', () => {
+    const state2 = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        kibanaDataViews: [
+          state.sourcerer.defaultDataView,
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '1234',
+            title: 'auditbeat-*',
+            patternList: ['auditbeat-*'],
+          },
+          {
+            ...state.sourcerer.defaultDataView,
+            id: '12347',
+            title: 'packetbeat-*',
+            patternList: ['packetbeat-*'],
+          },
+        ],
+        sourcererScopes: {
+          ...mockGlobalState.sourcerer.sourcererScopes,
+          [SourcererScopeName.default]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+            loading: false,
+            patternList,
+            selectedDataViewId: id,
+            selectedPatterns: patternList.slice(0, 2),
+          },
+        },
+      },
+    };
+
+    store = createStore(state2, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
     const wrapper = mount(
       <TestProviders store={store}>
         <Sourcerer {...defaultProps} />
       </TestProviders>
     );
     wrapper.find(`[data-test-subj="sourcerer-trigger"]`).first().simulate('click');
-    wrapper
-      .find(
-        `[data-test-subj="indexPattern-switcher"] [title="packetbeat-*"] button.euiBadge__iconButton`
-      )
-      .first()
-      .simulate('click');
-    wrapper.update();
+    wrapper.find(`[data-test-subj="comboBoxInput"]`).first().simulate('click');
     expect(
-      wrapper.find(`[title="packetbeat-*"] [data-test-subj="kip-option"]`).first().text()
-    ).toEqual(' packetbeat-*');
+      wrapper
+        .find(
+          `[data-test-subj="sourcerer-combo-box"] span[title="${mockGlobalState.sourcerer.signalIndexName}"]`
+        )
+        .first()
+        .exists()
+    ).toBeFalsy();
   });
 });

x-pack/plugins/security_solution/public/common/components/sourcerer/index.tsx

@@ -10,24 +10,26 @@ import {
   EuiButtonEmpty,
   EuiComboBox,
   EuiComboBoxOptionOption,
-  EuiIcon,
   EuiFlexGroup,
   EuiFlexItem,
+  EuiIcon,
   EuiPopover,
   EuiPopoverTitle,
   EuiSpacer,
+  EuiSuperSelect,
   EuiText,
   EuiToolTip,
 } from '@elastic/eui';
 import deepEqual from 'fast-deep-equal';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
-import { useDispatch, useSelector } from 'react-redux';
+import { useDispatch } from 'react-redux';
 import styled from 'styled-components';
 
 import * as i18n from './translations';
-import { sourcererActions, sourcererModel } from '../../store/sourcerer';
-import { State } from '../../store';
-import { getSourcererScopeSelector, SourcererScopeSelector } from './selectors';
+import { sourcererActions, sourcererModel, sourcererSelectors } from '../../store/sourcerer';
+import { getScopePatternListSelection } from '../../store/sourcerer/helpers';
+import { useDeepEqualSelector } from '../../hooks/use_selector';
+import { SourcererScopeName } from '../../store/sourcerer/model';
 
 const PopoverContent = styled.div`
   width: 600px;
@@ -40,30 +42,77 @@ interface SourcererComponentProps {
   scope: sourcererModel.SourcererScopeName;
 }
 
+const getPatternListWithoutSignals = (
+  patternList: string[],
+  signalIndexName: string | null
+): string[] => patternList.filter((p) => p !== signalIndexName);
+
 export const Sourcerer = React.memo<SourcererComponentProps>(({ scope: scopeId }) => {
   const dispatch = useDispatch();
-  const sourcererScopeSelector = useMemo(getSourcererScopeSelector, []);
-  const { configIndexPatterns, kibanaIndexPatterns, sourcererScope } = useSelector<
-    State,
-    SourcererScopeSelector
-  >((state) => sourcererScopeSelector(state, scopeId), deepEqual);
-  const { selectedPatterns, loading } = sourcererScope;
+  const sourcererScopeSelector = useMemo(() => sourcererSelectors.getSourcererScopeSelector(), []);
+  const {
+    defaultDataView,
+    kibanaDataViews,
+    signalIndexName,
+    sourcererScope: { selectedDataViewId, selectedPatterns, loading },
+  } = useDeepEqualSelector((state) => sourcererScopeSelector(state, scopeId));
+
   const [isPopoverOpen, setPopoverIsOpen] = useState(false);
+
+  const [dataViewId, setDataViewId] = useState<string>(selectedDataViewId ?? defaultDataView.id);
+
+  const { patternList, selectablePatterns } = useMemo(() => {
+    const theDataView = kibanaDataViews.find((dataView) => dataView.id === dataViewId);
+    return theDataView != null
+      ? scopeId === SourcererScopeName.default
+        ? {
+            patternList: getPatternListWithoutSignals(
+              theDataView.title
+                .split(',')
+                // remove duplicates patterns from selector
+                .filter((pattern, i, self) => self.indexOf(pattern) === i),
+              signalIndexName
+            ),
+            selectablePatterns: getPatternListWithoutSignals(
+              theDataView.patternList,
+              signalIndexName
+            ),
+          }
+        : {
+            patternList: theDataView.title
+              .split(',')
+              // remove duplicates patterns from selector
+              .filter((pattern, i, self) => self.indexOf(pattern) === i),
+            selectablePatterns: theDataView.patternList,
+          }
+      : { patternList: [], selectablePatterns: [] };
+  }, [kibanaDataViews, scopeId, signalIndexName, dataViewId]);
+
+  const selectableOptions = useMemo(
+    () =>
+      patternList.map((indexName) => ({
+        label: indexName,
+        value: indexName,
+        disabled: !selectablePatterns.includes(indexName),
+      })),
+    [selectablePatterns, patternList]
+  );
+
   const [selectedOptions, setSelectedOptions] = useState<Array<EuiComboBoxOptionOption<string>>>(
-    selectedPatterns.map((indexSelected) => ({
-      label: indexSelected,
-      value: indexSelected,
+    selectedPatterns.map((indexName) => ({
+      label: indexName,
+      value: indexName,
     }))
   );
   const isSavingDisabled = useMemo(() => selectedOptions.length === 0, [selectedOptions]);
 
   const setPopoverIsOpenCb = useCallback(() => setPopoverIsOpen((prevState) => !prevState), []);
-
-  const onChangeIndexPattern = useCallback(
-    (newSelectedPatterns: string[]) => {
+  const onChangeDataView = useCallback(
+    (newSelectedDataView: string, newSelectedPatterns: string[]) => {
       dispatch(
-        sourcererActions.setSelectedIndexPatterns({
+        sourcererActions.setSelectedDataView({
           id: scopeId,
+          selectedDataViewId: newSelectedDataView,
           selectedPatterns: newSelectedPatterns,
         })
       );
@@ -72,52 +121,55 @@ export const Sourcerer = React.memo<SourcererComponentProps>(({ scope: scopeId }
   );
 
   const renderOption = useCallback(
-    ({ value }) =>
-      kibanaIndexPatterns.some((kip) => kip.title === value) ? (
-        <span data-test-subj="kip-option">
-          <EuiIcon type="logoKibana" size="s" /> {value}
-        </span>
-      ) : (
-        <span data-test-subj="config-option">{value}</span>
-      ),
-    [kibanaIndexPatterns]
+    ({ value }) => <span data-test-subj="sourcerer-combo-option">{value}</span>,
+    []
   );
 
   const onChangeCombo = useCallback((newSelectedOptions) => {
     setSelectedOptions(newSelectedOptions);
   }, []);
 
+  const onChangeSuper = useCallback(
+    (newSelectedOption) => {
+      setDataViewId(newSelectedOption);
+      setSelectedOptions(
+        getScopePatternListSelection(
+          kibanaDataViews.find((dataView) => dataView.id === newSelectedOption),
+          scopeId,
+          signalIndexName,
+          newSelectedOption === defaultDataView.id
+        ).map((indexSelected: string) => ({
+          label: indexSelected,
+          value: indexSelected,
+        }))
+      );
+    },
+    [defaultDataView.id, kibanaDataViews, scopeId, signalIndexName]
+  );
+
   const resetDataSources = useCallback(() => {
+    setDataViewId(defaultDataView.id);
     setSelectedOptions(
-      configIndexPatterns.map((indexSelected) => ({
-        label: indexSelected,
-        value: indexSelected,
-      }))
+      getScopePatternListSelection(defaultDataView, scopeId, signalIndexName, true).map(
+        (indexSelected: string) => ({
+          label: indexSelected,
+          value: indexSelected,
+        })
+      )
     );
-  }, [configIndexPatterns]);
+  }, [defaultDataView, scopeId, signalIndexName]);
 
   const handleSaveIndices = useCallback(() => {
-    onChangeIndexPattern(selectedOptions.map((so) => so.label));
+    onChangeDataView(
+      dataViewId,
+      selectedOptions.map((so) => so.label)
+    );
     setPopoverIsOpen(false);
-  }, [onChangeIndexPattern, selectedOptions]);
+  }, [onChangeDataView, dataViewId, selectedOptions]);
 
   const handleClosePopOver = useCallback(() => {
     setPopoverIsOpen(false);
   }, []);
-
-  const indexesPatternOptions = useMemo(
-    () =>
-      [...configIndexPatterns, ...kibanaIndexPatterns.map((kip) => kip.title)].reduce<
-        Array<EuiComboBoxOptionOption<string>>
-      >((acc, index) => {
-        if (index != null && !acc.some((o) => o.label.includes(index))) {
-          return [...acc, { label: index, value: index }];
-        }
-        return acc;
-      }, []),
-    [configIndexPatterns, kibanaIndexPatterns]
-  );
-
   const trigger = useMemo(
     () => (
       <EuiButtonEmpty
@@ -136,37 +188,43 @@ export const Sourcerer = React.memo<SourcererComponentProps>(({ scope: scopeId }
     [setPopoverIsOpenCb, loading]
   );
 
-  const comboBox = useMemo(
-    () => (
-      <EuiComboBox
-        data-test-subj="indexPattern-switcher"
-        placeholder={i18n.PICK_INDEX_PATTERNS}
-        fullWidth
-        options={indexesPatternOptions}
-        selectedOptions={selectedOptions}
-        onChange={onChangeCombo}
-        renderOption={renderOption}
-      />
-    ),
-    [indexesPatternOptions, onChangeCombo, renderOption, selectedOptions]
+  const dataViewSelectOptions = useMemo(
+    () =>
+      kibanaDataViews.map(({ title, id }) => ({
+        inputDisplay:
+          id === defaultDataView.id ? (
+            <span data-test-subj="security-option-super">
+              <EuiIcon type="logoSecurity" size="s" /> {i18n.SIEM_DATA_VIEW_LABEL}
+            </span>
+          ) : (
+            <span data-test-subj="dataView-option-super">
+              <EuiIcon type="logoKibana" size="s" /> {title}
+            </span>
+          ),
+        value: id,
+      })),
+    [defaultDataView.id, kibanaDataViews]
   );
 
   useEffect(() => {
-    const newSelecteOptions = selectedPatterns.map((indexSelected) => ({
-      label: indexSelected,
-      value: indexSelected,
-    }));
-    setSelectedOptions((prevSelectedOptions) => {
-      if (!deepEqual(newSelecteOptions, prevSelectedOptions)) {
-        return newSelecteOptions;
-      }
-      return prevSelectedOptions;
-    });
+    setDataViewId((prevSelectedOption) =>
+      selectedDataViewId != null && !deepEqual(selectedDataViewId, prevSelectedOption)
+        ? selectedDataViewId
+        : prevSelectedOption
+    );
+  }, [selectedDataViewId]);
+  useEffect(() => {
+    setSelectedOptions(
+      selectedPatterns.map((indexName) => ({
+        label: indexName,
+        value: indexName,
+      }))
+    );
   }, [selectedPatterns]);
 
   const tooltipContent = useMemo(
-    () => (isPopoverOpen ? null : sourcererScope.selectedPatterns.sort().join(', ')),
-    [isPopoverOpen, sourcererScope.selectedPatterns]
+    () => (isPopoverOpen ? null : selectedPatterns.join(', ')),
+    [selectedPatterns, isPopoverOpen]
   );
 
   const buttonWithTooptip = useMemo(() => {
@@ -196,7 +254,24 @@ export const Sourcerer = React.memo<SourcererComponentProps>(({ scope: scopeId }
         <EuiSpacer size="s" />
         <EuiText color="default">{i18n.INDEX_PATTERNS_SELECTION_LABEL}</EuiText>
         <EuiSpacer size="xs" />
-        {comboBox}
+        <EuiSuperSelect
+          data-test-subj="sourcerer-select"
+          placeholder={i18n.PICK_INDEX_PATTERNS}
+          fullWidth
+          options={dataViewSelectOptions}
+          valueOfSelected={dataViewId}
+          onChange={onChangeSuper}
+        />
+        <EuiSpacer size="xs" />
+        <EuiComboBox
+          data-test-subj="sourcerer-combo-box"
+          fullWidth
+          onChange={onChangeCombo}
+          options={selectableOptions}
+          placeholder={i18n.PICK_INDEX_PATTERNS}
+          renderOption={renderOption}
+          selectedOptions={selectedOptions}
+        />
         <EuiSpacer size="s" />
         <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
           <EuiFlexItem>
@@ -214,7 +289,7 @@ export const Sourcerer = React.memo<SourcererComponentProps>(({ scope: scopeId }
             <EuiButton
               onClick={handleSaveIndices}
               disabled={isSavingDisabled}
-              data-test-subj="add-index"
+              data-test-subj="sourcerer-save"
               fill
               fullWidth
               size="s"

x-pack/plugins/security_solution/public/common/components/sourcerer/selectors.tsx

@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { State } from '../../store';
-import { sourcererSelectors } from '../../store/sourcerer';
-import { KibanaIndexPatterns, ManageScope, SourcererScopeName } from '../../store/sourcerer/model';
-
-export interface SourcererScopeSelector {
-  configIndexPatterns: string[];
-  kibanaIndexPatterns: KibanaIndexPatterns;
-  sourcererScope: ManageScope;
-}
-
-export const getSourcererScopeSelector = () => {
-  const getKibanaIndexPatternsSelector = sourcererSelectors.kibanaIndexPatternsSelector();
-  const getScopesSelector = sourcererSelectors.scopesSelector();
-  const getConfigIndexPatternsSelector = sourcererSelectors.configIndexPatternsSelector();
-
-  const mapStateToProps = (state: State, scopeId: SourcererScopeName): SourcererScopeSelector => {
-    const kibanaIndexPatterns = getKibanaIndexPatternsSelector(state);
-    const scope = getScopesSelector(state)[scopeId];
-    const configIndexPatterns = getConfigIndexPatternsSelector(state);
-
-    return {
-      kibanaIndexPatterns,
-      configIndexPatterns,
-      sourcererScope: scope,
-    };
-  };
-
-  return mapStateToProps;
-};

x-pack/plugins/security_solution/public/common/components/sourcerer/translations.ts

@@ -11,9 +11,12 @@ export const SOURCERER = i18n.translate('xpack.securitySolution.indexPatterns.da
   defaultMessage: 'Data sources',
 });
 
-export const ALL_DEFAULT = i18n.translate('xpack.securitySolution.indexPatterns.allDefault', {
-  defaultMessage: 'All default',
-});
+export const SIEM_DATA_VIEW_LABEL = i18n.translate(
+  'xpack.securitySolution.indexPatterns.kipLabel',
+  {
+    defaultMessage: 'Default Security Data View',
+  }
+);
 
 export const SELECT_INDEX_PATTERNS = i18n.translate('xpack.securitySolution.indexPatterns.help', {
   defaultMessage: 'Data sources selection',

x-pack/plugins/security_solution/public/common/components/threat_match/entry_item.test.tsx

@@ -11,7 +11,7 @@ import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
 
 import { EntryItem } from './entry_item';
 import { fields, getField } from '../../../../../../../src/plugins/data/common/mocks';
-import { IndexPattern } from 'src/plugins/data/public';
+import { DataViewBase } from '@kbn/es-query';
 
 jest.mock('../../../common/lib/kibana');
 
@@ -31,7 +31,7 @@ describe('EntryItem', () => {
             id: '1234',
             title: 'logstash-*',
             fields,
-          } as IndexPattern
+          } as DataViewBase
         }
         showLabel={true}
         onChange={jest.fn()}
@@ -40,7 +40,7 @@ describe('EntryItem', () => {
             id: '1234',
             title: 'logstash-*',
             fields,
-          } as IndexPattern
+          } as DataViewBase
         }
       />
     );
@@ -64,14 +64,14 @@ describe('EntryItem', () => {
             id: '1234',
             title: 'logstash-*',
             fields,
-          } as IndexPattern
+          } as DataViewBase
         }
         threatIndexPatterns={
           {
             id: '1234',
             title: 'logstash-*',
             fields,
-          } as IndexPattern
+          } as DataViewBase
         }
         showLabel={false}
         onChange={mockOnChange}
@@ -111,14 +111,14 @@ describe('EntryItem', () => {
             id: '1234',
             title: 'logstash-*',
             fields,
-          } as IndexPattern
+          } as DataViewBase
         }
         threatIndexPatterns={
           {
             id: '1234',
             title: 'logstash-*',
             fields,
-          } as IndexPattern
+          } as DataViewBase
         }
         showLabel={false}
         onChange={mockOnChange}

x-pack/plugins/security_solution/public/common/components/threat_match/entry_item.tsx

@@ -10,16 +10,15 @@ import { EuiFormRow, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import styled from 'styled-components';
 
 import { FieldComponent } from '@kbn/securitysolution-autocomplete';
-import { IndexPatternFieldBase } from '@kbn/es-query';
-import { IndexPattern } from '../../../../../../../src/plugins/data/common';
+import { DataViewBase, DataViewFieldBase } from '@kbn/es-query';
 import { FormattedEntry, Entry } from './types';
 import * as i18n from './translations';
 import { getEntryOnFieldChange, getEntryOnThreatFieldChange } from './helpers';
 
 interface EntryItemProps {
   entry: FormattedEntry;
-  indexPattern: IndexPattern;
-  threatIndexPatterns: IndexPattern;
+  indexPattern: DataViewBase;
+  threatIndexPatterns: DataViewBase;
   showLabel: boolean;
   onChange: (arg: Entry, i: number) => void;
 }
@@ -41,7 +40,7 @@ export const EntryItem: React.FC<EntryItemProps> = ({
   onChange,
 }): JSX.Element => {
   const handleFieldChange = useCallback(
-    ([newField]: IndexPatternFieldBase[]): void => {
+    ([newField]: DataViewFieldBase[]): void => {
       const { updatedEntry, index } = getEntryOnFieldChange(entry, newField);
       onChange(updatedEntry, index);
     },
@@ -49,7 +48,7 @@ export const EntryItem: React.FC<EntryItemProps> = ({
   );
 
   const handleThreatFieldChange = useCallback(
-    ([newField]: IndexPatternFieldBase[]): void => {
+    ([newField]: DataViewFieldBase[]): void => {
       const { updatedEntry, index } = getEntryOnThreatFieldChange(entry, newField);
       onChange(updatedEntry, index);
     },

x-pack/plugins/security_solution/public/common/components/threat_match/helpers.test.tsx

@@ -7,7 +7,8 @@
 
 import { fields, getField } from '../../../../../../../src/plugins/data/common/mocks';
 import { Entry, EmptyEntry, ThreatMapEntries, FormattedEntry } from './types';
-import { FieldSpec, IndexPattern } from '../../../../../../../src/plugins/data/common';
+import { FieldSpec } from '../../../../../../../src/plugins/data/common';
+import { DataViewBase } from '@kbn/es-query';
 import moment from 'moment-timezone';
 
 import {
@@ -24,12 +25,12 @@ jest.mock('uuid', () => ({
   v4: jest.fn().mockReturnValue('123'),
 }));
 
-const getMockIndexPattern = (): IndexPattern =>
+const getMockIndexPattern = (): DataViewBase =>
   ({
     id: '1234',
     title: 'logstash-*',
     fields,
-  } as IndexPattern);
+  } as DataViewBase);
 
 const getMockEntry = (): FormattedEntry => ({
   id: '123',
@@ -51,7 +52,7 @@ describe('Helpers', () => {
 
   describe('#getFormattedEntry', () => {
     test('it returns entry with a value when "item.field" is of type "text" and matching keyword field exists', () => {
-      const payloadIndexPattern: IndexPattern = {
+      const payloadIndexPattern: DataViewBase = {
         ...getMockIndexPattern(),
         fields: [
           ...fields,
@@ -66,7 +67,7 @@ describe('Helpers', () => {
             readFromDocValues: true,
           },
         ],
-      } as IndexPattern;
+      } as DataViewBase;
       const payloadItem: Entry = {
         field: 'machine.os.raw.text',
         type: 'mapping',
@@ -171,7 +172,7 @@ describe('Helpers', () => {
     });
 
     test('it returns formatted entries', () => {
-      const payloadIndexPattern: IndexPattern = getMockIndexPattern();
+      const payloadIndexPattern: DataViewBase = getMockIndexPattern();
       const payloadItems: Entry[] = [
         { field: 'machine.os', type: 'mapping', value: 'machine.os' },
         { field: 'ip', type: 'mapping', value: 'ip' },

x-pack/plugins/security_solution/public/common/components/threat_match/helpers.tsx

@@ -10,8 +10,7 @@ import { i18n } from '@kbn/i18n';
 import { addIdToItem } from '@kbn/securitysolution-utils';
 import { ThreatMap, threatMap, ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types';
 
-import { IndexPatternFieldBase } from '@kbn/es-query';
-import { IndexPattern } from '../../../../../../../src/plugins/data/common';
+import { DataViewBase, DataViewFieldBase } from '@kbn/es-query';
 import { Entry, FormattedEntry, ThreatMapEntries, EmptyEntry } from './types';
 import { ValidationFunc } from '../../../../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib';
 import { ERROR_CODE } from '../../../../../../../src/plugins/es_ui_shared/static/forms/helpers/field_validators/types';
@@ -19,13 +18,13 @@ import { ERROR_CODE } from '../../../../../../../src/plugins/es_ui_shared/static
 /**
  * Formats the entry into one that is easily usable for the UI.
  *
- * @param patterns IndexPattern containing available fields on rule index
+ * @param patterns DataViewBase containing available fields on rule index
  * @param item item entry
  * @param itemIndex entry index
  */
 export const getFormattedEntry = (
-  indexPattern: IndexPattern,
-  threatIndexPatterns: IndexPattern,
+  indexPattern: DataViewBase,
+  threatIndexPatterns: DataViewBase,
   item: Entry,
   itemIndex: number,
   uuidGen: () => string = uuid.v4
@@ -51,12 +50,12 @@ export const getFormattedEntry = (
 /**
  * Formats the entries to be easily usable for the UI
  *
- * @param patterns IndexPattern containing available fields on rule index
+ * @param patterns DataViewBase containing available fields on rule index
  * @param entries item entries
  */
 export const getFormattedEntries = (
-  indexPattern: IndexPattern,
-  threatIndexPatterns: IndexPattern,
+  indexPattern: DataViewBase,
+  threatIndexPatterns: DataViewBase,
   entries: Entry[]
 ): FormattedEntry[] => {
   return entries.reduce<FormattedEntry[]>((acc, item, index) => {
@@ -91,7 +90,7 @@ export const getUpdatedEntriesOnDelete = (
  */
 export const getEntryOnFieldChange = (
   item: FormattedEntry,
-  newField: IndexPatternFieldBase
+  newField: DataViewFieldBase
 ): { updatedEntry: Entry; index: number } => {
   const { entryIndex } = item;
   return {
@@ -114,7 +113,7 @@ export const getEntryOnFieldChange = (
  */
 export const getEntryOnThreatFieldChange = (
   item: FormattedEntry,
-  newField: IndexPatternFieldBase
+  newField: DataViewFieldBase
 ): { updatedEntry: Entry; index: number } => {
   const { entryIndex } = item;
   return {

x-pack/plugins/security_solution/public/common/components/threat_match/index.test.tsx

@@ -16,7 +16,7 @@ import { useKibana } from '../../../common/lib/kibana';
 
 import { ThreatMatchComponent } from './';
 import { ThreatMapEntries } from './types';
-import { IndexPattern } from 'src/plugins/data/public';
+import { DataViewBase } from '@kbn/es-query';
 import { getMockTheme } from '../../lib/kibana/kibana_react.mock';
 
 const mockTheme = getMockTheme({
@@ -65,14 +65,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -94,14 +94,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -123,14 +123,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -151,14 +151,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -188,14 +188,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -225,14 +225,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -255,14 +255,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />
@@ -286,14 +286,14 @@ describe('ThreatMatchComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           onChange={jest.fn()}
         />

x-pack/plugins/security_solution/public/common/components/threat_match/index.tsx

@@ -8,10 +8,9 @@
 import React, { useCallback, useEffect, useReducer } from 'react';
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import styled from 'styled-components';
-
+import { DataViewBase } from '@kbn/es-query';
 import { ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types';
 import { ListItemComponent } from './list_item';
-import { IndexPattern } from '../../../../../../../src/plugins/data/common';
 import { AndOrBadge } from '../and_or_badge';
 import { LogicButtons } from './logic_buttons';
 import { ThreatMapEntries } from './types';
@@ -45,8 +44,8 @@ interface OnChangeProps {
 
 interface ThreatMatchComponentProps {
   listItems: ThreatMapEntries[];
-  indexPatterns: IndexPattern;
-  threatIndexPatterns: IndexPattern;
+  indexPatterns: DataViewBase;
+  threatIndexPatterns: DataViewBase;
   onChange: (arg: OnChangeProps) => void;
 }
 

x-pack/plugins/security_solution/public/common/components/threat_match/list_item.test.tsx

@@ -14,7 +14,7 @@ import { fields } from '../../../../../../../src/plugins/data/common/mocks';
 
 import { ListItemComponent } from './list_item';
 import { ThreatMapEntries } from './types';
-import { IndexPattern } from 'src/plugins/data/public';
+import { DataViewBase } from '@kbn/es-query';
 import { getMockTheme } from '../../lib/kibana/kibana_react.mock';
 
 const mockTheme = getMockTheme({
@@ -81,14 +81,14 @@ describe('ListItemComponent', () => {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             threatIndexPatterns={
               {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             andLogicIncluded={true}
             isOnlyItem={false}
@@ -114,7 +114,7 @@ describe('ListItemComponent', () => {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             andLogicIncluded={true}
             isOnlyItem={false}
@@ -125,7 +125,7 @@ describe('ListItemComponent', () => {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
           />
         </ThemeProvider>
@@ -145,14 +145,14 @@ describe('ListItemComponent', () => {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             threatIndexPatterns={
               {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             andLogicIncluded={true}
             isOnlyItem={false}
@@ -178,14 +178,14 @@ describe('ListItemComponent', () => {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             threatIndexPatterns={
               {
                 id: '1234',
                 title: 'logstash-*',
                 fields,
-              } as IndexPattern
+              } as DataViewBase
             }
             andLogicIncluded={false}
             isOnlyItem={false}
@@ -219,14 +219,14 @@ describe('ListItemComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           andLogicIncluded={false}
           isOnlyItem={true}
@@ -250,14 +250,14 @@ describe('ListItemComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           andLogicIncluded={false}
           isOnlyItem={false}
@@ -281,14 +281,14 @@ describe('ListItemComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           andLogicIncluded={false}
           // if entryItemIndex is not 0, wouldn't make sense for
@@ -314,14 +314,14 @@ describe('ListItemComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           andLogicIncluded={false}
           isOnlyItem={true}
@@ -346,14 +346,14 @@ describe('ListItemComponent', () => {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           threatIndexPatterns={
             {
               id: '1234',
               title: 'logstash-*',
               fields,
-            } as IndexPattern
+            } as DataViewBase
           }
           andLogicIncluded={false}
           isOnlyItem={true}

x-pack/plugins/security_solution/public/common/components/threat_match/list_item.tsx

@@ -9,7 +9,7 @@ import React, { useMemo, useCallback } from 'react';
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import styled from 'styled-components';
 
-import { IndexPattern } from '../../../../../../../src/plugins/data/common';
+import { DataViewBase } from '@kbn/es-query';
 import { getFormattedEntries, getUpdatedEntriesOnDelete } from './helpers';
 import { FormattedEntry, ThreatMapEntries, Entry } from './types';
 import { EntryItem } from './entry_item';
@@ -24,8 +24,8 @@ const MyOverflowContainer = styled(EuiFlexItem)`
 interface ListItemProps {
   listItem: ThreatMapEntries;
   listItemIndex: number;
-  indexPattern: IndexPattern;
-  threatIndexPatterns: IndexPattern;
+  indexPattern: DataViewBase;
+  threatIndexPatterns: DataViewBase;
   andLogicIncluded: boolean;
   isOnlyItem: boolean;
   onDeleteEntryItem: (item: ThreatMapEntries, index: number) => void;

x-pack/plugins/security_solution/public/common/components/threat_match/types.ts

@@ -5,14 +5,14 @@
  * 2.0.
  */
 
-import { IndexPatternFieldBase } from '@kbn/es-query';
+import { DataViewFieldBase } from '@kbn/es-query';
 import { ThreatMap, ThreatMapEntry } from '@kbn/securitysolution-io-ts-alerting-types';
 
 export interface FormattedEntry {
   id: string;
-  field: IndexPatternFieldBase | undefined;
+  field: DataViewFieldBase | undefined;
   type: 'mapping';
-  value: IndexPatternFieldBase | undefined;
+  value: DataViewFieldBase | undefined;
   entryIndex: number;
 }
 

x-pack/plugins/security_solution/public/common/components/top_n/index.tsx

@@ -8,15 +8,11 @@
 import React, { useMemo } from 'react';
 import { connect, ConnectedProps } from 'react-redux';
 
+import { DataViewBase, Filter, Query } from '@kbn/es-query';
 import { useGlobalTime } from '../../containers/use_global_time';
 import { BrowserFields } from '../../containers/source';
 import { useKibana } from '../../lib/kibana';
-import {
-  esQuery,
-  Filter,
-  Query,
-  IIndexPattern,
-} from '../../../../../../../src/plugins/data/public';
+import { esQuery } from '../../../../../../../src/plugins/data/public';
 import { inputsModel, inputsSelectors, State } from '../../store';
 import { timelineDefaults } from '../../../timelines/store/timeline/defaults';
 import { timelineSelectors } from '../../../timelines/store/timeline';
@@ -77,7 +73,7 @@ const connector = connect(makeMapStateToProps);
 export interface OwnProps {
   browserFields: BrowserFields;
   field: string;
-  indexPattern: IIndexPattern;
+  indexPattern: DataViewBase;
   timelineId?: string;
   toggleTopN: () => void;
   onFilterAdded?: () => void;

x-pack/plugins/security_solution/public/common/components/top_n/selectors.tsx

@@ -7,6 +7,7 @@
 
 import { State } from '../../store';
 import { sourcererSelectors } from '../../store/selectors';
+import { SourcererScopeName } from '../../store/sourcerer/model';
 
 export interface IndicesSelector {
   all: string[];
@@ -14,25 +15,17 @@ export interface IndicesSelector {
 }
 
 export const getIndicesSelector = () => {
-  const getkibanaIndexPatternsSelector = sourcererSelectors.kibanaIndexPatternsSelector();
-  const getConfigIndexPatternsSelector = sourcererSelectors.configIndexPatternsSelector();
   const getSignalIndexNameSelector = sourcererSelectors.signalIndexNameSelector();
+  const getScopeSelector = sourcererSelectors.scopeIdSelector();
 
-  const mapStateToProps = (state: State): IndicesSelector => {
-    const rawIndices = new Set(getConfigIndexPatternsSelector(state));
-    const kibanaIndexPatterns = getkibanaIndexPatternsSelector(state);
-    const alertIndexName = getSignalIndexNameSelector(state);
-    kibanaIndexPatterns.forEach(({ title }) => {
-      if (title !== alertIndexName) {
-        rawIndices.add(title);
-      }
-    });
+  return (state: State, scopeId: SourcererScopeName): IndicesSelector => {
+    const signalIndexName = getSignalIndexNameSelector(state);
+    const { selectedPatterns } = getScopeSelector(state, scopeId);
+    const raw: string[] = selectedPatterns.filter((index) => index !== signalIndexName);
 
     return {
-      all: alertIndexName != null ? [...rawIndices, alertIndexName] : [...rawIndices],
-      raw: [...rawIndices],
+      all: signalIndexName != null ? [...raw, signalIndexName] : [...raw],
+      raw,
     };
   };
-
-  return mapStateToProps;
 };

x-pack/plugins/security_solution/public/common/components/top_n/top_n.tsx

@@ -11,10 +11,10 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import { useSelector } from 'react-redux';
 import styled from 'styled-components';
 
+import { DataViewBase, Filter, Query } from '@kbn/es-query';
 import { GlobalTimeArgs } from '../../containers/use_global_time';
 import { EventsByDataset } from '../../../overview/components/events_by_dataset';
 import { SignalsByCategory } from '../../../overview/components/signals_by_category';
-import { Filter, IIndexPattern, Query } from '../../../../../../../src/plugins/data/public';
 import { InputsModelId } from '../../store/inputs/constants';
 import { TimelineEventsType } from '../../../../common/types/timeline';
 
@@ -23,6 +23,7 @@ import * as i18n from './translations';
 import { getIndicesSelector, IndicesSelector } from './selectors';
 import { State } from '../../store';
 import { AlertsStackByField } from '../../../detections/components/alerts_kpis/common/types';
+import { SourcererScopeName } from '../../store/sourcerer/model';
 
 const TopNContainer = styled.div`
   min-width: 600px;
@@ -53,7 +54,7 @@ export interface Props extends Pick<GlobalTimeArgs, 'from' | 'to' | 'deleteQuery
   defaultView: TimelineEventsType;
   field: AlertsStackByField;
   filters: Filter[];
-  indexPattern: IIndexPattern;
+  indexPattern: DataViewBase;
   options: TopNOption[];
   paddingSize?: 's' | 'm' | 'l' | 'none';
   query: Query;
@@ -90,7 +91,15 @@ const TopNComponent: React.FC<Props> = ({
   );
   const indicesSelector = useMemo(getIndicesSelector, []);
   const { all: allIndices, raw: rawIndices } = useSelector<State, IndicesSelector>(
-    (state) => indicesSelector(state),
+    (state) =>
+      indicesSelector(
+        state,
+        timelineId != null
+          ? defaultView === 'alert'
+            ? SourcererScopeName.detections
+            : SourcererScopeName.timeline
+          : SourcererScopeName.default
+      ),
     deepEqual
   );
 

x-pack/plugins/security_solution/public/common/components/url_state/helpers.ts

@@ -24,7 +24,7 @@ import { NavTab } from '../navigation/types';
 import { CONSTANTS, UrlStateType } from './constants';
 import { ReplaceStateInLocation, KeyUrlState, ValueUrlState } from './types';
 import { sourcererSelectors } from '../../store/sourcerer';
-import { SourcererScopeName, SourcererScopePatterns } from '../../store/sourcerer/model';
+import { SourcererScopeName, SourcererUrlState } from '../../store/sourcerer/model';
 
 export const isDetectionsPages = (pageName: string) =>
   pageName === SecurityPageName.alerts ||
@@ -156,9 +156,18 @@ export const makeMapStateToProps = () => {
     }
     const sourcerer = getSourcererScopes(state);
     const activeScopes: SourcererScopeName[] = Object.keys(sourcerer) as SourcererScopeName[];
-    const selectedPatterns: SourcererScopePatterns = activeScopes
+    const selectedPatterns: SourcererUrlState = activeScopes
       .filter((scope) => scope === SourcererScopeName.default)
-      .reduce((acc, scope) => ({ ...acc, [scope]: sourcerer[scope]?.selectedPatterns }), {});
+      .reduce(
+        (acc, scope) => ({
+          ...acc,
+          [scope]: {
+            id: sourcerer[scope]?.selectedDataViewId,
+            selectedPatterns: sourcerer[scope]?.selectedPatterns,
+          },
+        }),
+        {}
+      );
 
     return {
       urlState: {

x-pack/plugins/security_solution/public/common/components/url_state/initialize_redux_by_url.tsx

@@ -28,7 +28,7 @@ import {
   queryTimelineById,
   dispatchUpdateTimeline,
 } from '../../../timelines/components/open_timeline/helpers';
-import { SourcererScopeName, SourcererScopePatterns } from '../../store/sourcerer/model';
+import { SourcererScopeName, SourcererUrlState } from '../../store/sourcerer/model';
 import { timelineActions } from '../../../timelines/store/timeline';
 
 export const useSetInitialStateFromUrl = () => {
@@ -55,16 +55,17 @@ export const useSetInitialStateFromUrl = () => {
           updateTimerange(newUrlStateString, dispatch);
         }
         if (urlKey === CONSTANTS.sourcerer) {
-          const sourcererState = decodeRisonUrlState<SourcererScopePatterns>(newUrlStateString);
+          const sourcererState = decodeRisonUrlState<SourcererUrlState>(newUrlStateString);
           if (sourcererState != null) {
             const activeScopes: SourcererScopeName[] = Object.keys(sourcererState).filter(
               (key) => !(key === SourcererScopeName.default && isDetectionsPages(pageName))
             ) as SourcererScopeName[];
             activeScopes.forEach((scope) =>
               dispatch(
-                sourcererActions.setSelectedIndexPatterns({
+                sourcererActions.setSelectedDataView({
                   id: scope,
-                  selectedPatterns: sourcererState[scope] ?? [],
+                  selectedDataViewId: sourcererState[scope]?.id ?? '',
+                  selectedPatterns: sourcererState[scope]?.selectedPatterns ?? [],
                 })
               )
             );

x-pack/plugins/security_solution/public/common/components/url_state/normalize_time_range.test.ts

@@ -13,23 +13,12 @@ import {
   RelativeTimeRange,
   isRelativeTimeRange,
 } from '../../store/inputs/model';
-
+import DateMath from '@elastic/datemath';
 import { getTimeRangeSettings } from '../../utils/default_date_settings';
 
 const getTimeRangeSettingsMock = getTimeRangeSettings as jest.Mock;
 
 jest.mock('../../utils/default_date_settings');
-jest.mock('@elastic/datemath', () => ({
-  parse: (date: string) => {
-    if (date === 'now') {
-      return { toISOString: () => '2020-07-08T08:20:18.966Z' };
-    }
-
-    if (date === 'now-24h') {
-      return { toISOString: () => '2020-07-07T08:20:18.966Z' };
-    }
-  },
-}));
 
 getTimeRangeSettingsMock.mockImplementation(() => ({
   from: '2020-07-04T08:20:18.966Z',
@@ -39,6 +28,19 @@ getTimeRangeSettingsMock.mockImplementation(() => ({
 }));
 
 describe('#normalizeTimeRange', () => {
+  let dateMathSpy: jest.SpyInstance;
+  beforeAll(() => {
+    dateMathSpy = jest.spyOn(DateMath, 'parse');
+    dateMathSpy.mockImplementation((date: string) =>
+      date === 'now'
+        ? { toISOString: () => new Date('2020-07-08T08:20:18.966Z') }
+        : { toISOString: () => new Date('2020-07-07T08:20:18.966Z') }
+    );
+  });
+
+  afterAll(() => {
+    jest.clearAllMocks();
+  });
   test('Absolute time range returns defaults for empty strings', () => {
     const dateTimeRange: URLTimeRange = {
       kind: 'absolute',

x-pack/plugins/security_solution/public/common/components/url_state/test_dependencies.ts

@@ -84,9 +84,7 @@ export const defaultProps: UrlStateContainerPropTypes = {
   indexPattern: {
     fields: [
       {
-        aggregatable: true,
         name: '@timestamp',
-        searchable: true,
         type: 'date',
       },
     ],

x-pack/plugins/security_solution/public/common/components/url_state/types.ts

@@ -5,21 +5,15 @@
  * 2.0.
  */
 
-import {
-  Filter,
-  FilterManager,
-  IIndexPattern,
-  Query,
-  SavedQueryService,
-} from 'src/plugins/data/public';
-
+import { Filter, FilterManager, Query, SavedQueryService } from 'src/plugins/data/public';
+import { DataViewBase } from '@kbn/es-query';
 import { UrlInputsModel } from '../../store/inputs/model';
 import { TimelineUrl } from '../../../timelines/store/timeline/model';
 import { RouteSpyState } from '../../utils/route/types';
 import { SecurityNav } from '../navigation/types';
 
 import { CONSTANTS, UrlStateType } from './constants';
-import { SourcererScopePatterns } from '../../store/sourcerer/model';
+import { SourcererUrlState } from '../../store/sourcerer/model';
 
 export const ALL_URL_STATE_KEYS: KeyUrlState[] = [
   CONSTANTS.appQuery,
@@ -48,7 +42,7 @@ export interface UrlState {
   [CONSTANTS.appQuery]?: Query;
   [CONSTANTS.filters]?: Filter[];
   [CONSTANTS.savedQuery]?: string;
-  [CONSTANTS.sourcerer]: SourcererScopePatterns;
+  [CONSTANTS.sourcerer]: SourcererUrlState;
   [CONSTANTS.timerange]: UrlInputsModel;
   [CONSTANTS.timeline]: TimelineUrl;
 }
@@ -58,7 +52,7 @@ export type ValueUrlState = UrlState[keyof UrlState];
 
 export interface UrlStateProps {
   navTabs: SecurityNav;
-  indexPattern?: IIndexPattern;
+  indexPattern?: DataViewBase;
   mapToUrlState?: (value: string) => UrlState;
   onChange?: (urlState: UrlState, previousUrlState: UrlState) => void;
   onInitialize?: (urlState: UrlState) => void;
@@ -89,7 +83,7 @@ export interface UrlStateToRedux {
 
 export interface SetInitialStateFromUrl {
   filterManager: FilterManager;
-  indexPattern: IIndexPattern | undefined;
+  indexPattern: DataViewBase | undefined;
   pageName: string;
   savedQueries: SavedQueryService;
   urlStateToUpdate: UrlStateToRedux[];

x-pack/plugins/security_solution/public/common/containers/kuery_autocompletion/index.tsx

@@ -1,85 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import React, { useState } from 'react';
-import { QuerySuggestion, IIndexPattern } from '../../../../../../../src/plugins/data/public';
-import { useKibana } from '../../lib/kibana';
-
-type RendererResult = React.ReactElement<JSX.Element> | null;
-type RendererFunction<RenderArgs, Result = RendererResult> = (args: RenderArgs) => Result;
-
-interface KueryAutocompletionLifecycleProps {
-  children: RendererFunction<{
-    isLoadingSuggestions: boolean;
-    loadSuggestions: (expression: string, cursorPosition: number, maxSuggestions?: number) => void;
-    suggestions: QuerySuggestion[];
-  }>;
-  indexPattern: IIndexPattern;
-}
-
-interface KueryAutocompletionCurrentRequest {
-  expression: string;
-  cursorPosition: number;
-}
-
-export const KueryAutocompletion = React.memo<KueryAutocompletionLifecycleProps>(
-  ({ children, indexPattern }) => {
-    const [currentRequest, setCurrentRequest] = useState<KueryAutocompletionCurrentRequest | null>(
-      null
-    );
-    const [suggestions, setSuggestions] = useState<QuerySuggestion[]>([]);
-    const kibana = useKibana();
-    const loadSuggestions = async (
-      expression: string,
-      cursorPosition: number,
-      maxSuggestions?: number
-    ) => {
-      const language = 'kuery';
-
-      if (!kibana.services.data.autocomplete.hasQuerySuggestions(language)) {
-        return;
-      }
-
-      const futureRequest = {
-        expression,
-        cursorPosition,
-      };
-      setCurrentRequest({
-        expression,
-        cursorPosition,
-      });
-      setSuggestions([]);
-
-      if (
-        futureRequest &&
-        futureRequest.expression !== (currentRequest && currentRequest.expression) &&
-        futureRequest.cursorPosition !== (currentRequest && currentRequest.cursorPosition)
-      ) {
-        const newSuggestions =
-          (await kibana.services.data.autocomplete.getQuerySuggestions({
-            language: 'kuery',
-            indexPatterns: [indexPattern],
-            boolFilter: [],
-            query: expression,
-            selectionStart: cursorPosition,
-            selectionEnd: cursorPosition,
-          })) || [];
-
-        setCurrentRequest(null);
-        setSuggestions(maxSuggestions ? newSuggestions.slice(0, maxSuggestions) : newSuggestions);
-      }
-    };
-
-    return children({
-      isLoadingSuggestions: currentRequest !== null,
-      loadSuggestions,
-      suggestions,
-    });
-  }
-);
-
-KueryAutocompletion.displayName = 'KueryAutocompletion';

x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx

@@ -7,7 +7,31 @@
 
 import { IndexField } from '../../../../common/search_strategy/index_fields';
 import { getBrowserFields } from '.';
+import { useDataView } from './use_data_view';
 import { mockBrowserFields, mocksSource } from './mock';
+import { SourcererScopeName } from '../../store/sourcerer/model';
+import { createStore, State } from '../../store';
+import {
+  createSecuritySolutionStorageMock,
+  kibanaObservable,
+  mockGlobalState,
+  SUB_PLUGINS_REDUCER,
+} from '../../mock';
+import { act, renderHook } from '@testing-library/react-hooks';
+import { Provider } from 'react-redux';
+import React from 'react';
+import { useKibana } from '../../lib/kibana';
+
+const mockDispatch = jest.fn();
+jest.mock('react-redux', () => {
+  const original = jest.requireActual('react-redux');
+
+  return {
+    ...original,
+    useDispatch: () => mockDispatch,
+  };
+});
+jest.mock('../../lib/kibana'); // , () => ({
 
 describe('source/index.tsx', () => {
   describe('getBrowserFields', () => {
@@ -28,4 +52,89 @@ describe('source/index.tsx', () => {
       expect(fields).toEqual(mockBrowserFields);
     });
   });
+  describe('useDataView hook', () => {
+    const sourcererState = mockGlobalState.sourcerer;
+    const state: State = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...sourcererState,
+        kibanaDataViews: [
+          ...sourcererState.kibanaDataViews,
+          {
+            ...sourcererState.defaultDataView,
+            id: 'something-random',
+            title: 'something,random',
+            patternList: ['something', 'random'],
+          },
+        ],
+        sourcererScopes: {
+          ...sourcererState.sourcererScopes,
+          [SourcererScopeName.default]: {
+            ...sourcererState.sourcererScopes[SourcererScopeName.default],
+          },
+          [SourcererScopeName.detections]: {
+            ...sourcererState.sourcererScopes[SourcererScopeName.detections],
+          },
+          [SourcererScopeName.timeline]: {
+            ...sourcererState.sourcererScopes[SourcererScopeName.timeline],
+          },
+        },
+      },
+    };
+    const mockSearchResponse = {
+      ...mocksSource,
+      indicesExist: ['auditbeat-*', sourcererState.signalIndexName],
+      isRestore: false,
+      rawResponse: {},
+      runtimeMappings: {},
+    };
+    const { storage } = createSecuritySolutionStorageMock();
+    const store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+      const mock = {
+        subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
+        unsubscribe: jest.fn(),
+      };
+
+      (useKibana as jest.Mock).mockReturnValue({
+        services: {
+          data: {
+            search: {
+              search: jest.fn().mockReturnValue({
+                subscribe: ({ next }: { next: Function }) => {
+                  next(mockSearchResponse);
+                  return mock;
+                },
+                unsubscribe: jest.fn(),
+              }),
+            },
+          },
+        },
+      });
+    });
+    it('sets field data for data view', async () => {
+      await act(async () => {
+        const { rerender, waitForNextUpdate, result } = renderHook<
+          string,
+          { indexFieldsSearch: (id: string) => void }
+        >(() => useDataView(), {
+          wrapper: ({ children }) => <Provider store={store}>{children}</Provider>,
+        });
+        await waitForNextUpdate();
+        rerender();
+        act(() => result.current.indexFieldsSearch('neato'));
+        expect(mockDispatch.mock.calls[0][0]).toEqual({
+          type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
+          payload: { id: 'neato', loading: true },
+        });
+        const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
+        expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
+        expect(payload.id).toEqual('neato');
+        expect(Object.keys(payload.browserFields)).toHaveLength(10);
+        expect(payload.docValueFields).toEqual([{ field: '@timestamp' }]);
+      });
+    });
+  });
 });

x-pack/plugins/security_solution/public/common/containers/source/index.tsx

@@ -5,27 +5,23 @@
  * 2.0.
  */
 
-import { keyBy, pick, isEmpty, isEqual, isUndefined } from 'lodash/fp';
+import { isEmpty, isEqual, isUndefined, keyBy, pick } from 'lodash/fp';
 import memoizeOne from 'memoize-one';
-import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-import { useDispatch } from 'react-redux';
-import { IIndexPattern } from 'src/plugins/data/public';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { DataViewBase } from '@kbn/es-query';
 import { Subscription } from 'rxjs';
 
 import { useKibana } from '../../lib/kibana';
 import {
-  IndexField,
-  IndexFieldsStrategyResponse,
-  IndexFieldsStrategyRequest,
   BrowserField,
   BrowserFields,
-} from '../../../../common/search_strategy/index_fields';
-import { isErrorResponse, isCompleteResponse } from '../../../../../../../src/plugins/data/common';
-import { useDeepEqualSelector } from '../../../common/hooks/use_selector';
+  DocValueFields,
+  IndexField,
+  IndexFieldsStrategyRequest,
+  IndexFieldsStrategyResponse,
+} from '../../../../../timelines/common';
+import { isCompleteResponse, isErrorResponse } from '../../../../../../../src/plugins/data/common';
 import * as i18n from './translations';
-import { SourcererScopeName } from '../../store/sourcerer/model';
-import { sourcererActions, sourcererSelectors } from '../../store/sourcerer';
-import { DocValueFields } from '../../../../common/search_strategy/common';
 import { useAppToasts } from '../../hooks/use_app_toasts';
 
 export { BrowserField, BrowserFields, DocValueFields };
@@ -45,7 +41,7 @@ export const getAllFieldsByName = (
   keyBy('name', getAllBrowserFields(browserFields));
 
 export const getIndexFields = memoizeOne(
-  (title: string, fields: IndexField[]): IIndexPattern =>
+  (title: string, fields: IndexField[]): DataViewBase =>
     fields && fields.length > 0
       ? {
           fields: fields.map((field) =>
@@ -95,7 +91,6 @@ export const getDocValueFields = memoizeOne(
               ...accumulator,
               {
                 field: field.name,
-                format: field.format ? field.format : undefined,
               },
             ];
           }
@@ -119,9 +114,13 @@ interface FetchIndexReturn {
   docValueFields: DocValueFields[];
   indexes: string[];
   indexExists: boolean;
-  indexPatterns: IIndexPattern;
+  indexPatterns: DataViewBase;
 }
 
+/**
+ * Independent index fields hook/request
+ * returns state directly, no redux
+ */
 export const useFetchIndex = (
   indexNames: string[],
   onlyCheckIfIndicesExist: boolean = false
@@ -147,7 +146,7 @@ export const useFetchIndex = (
         abortCtrl.current = new AbortController();
         setLoading(true);
         searchSubscription$.current = data.search
-          .search<IndexFieldsStrategyRequest, IndexFieldsStrategyResponse>(
+          .search<IndexFieldsStrategyRequest<'indices'>, IndexFieldsStrategyResponse>(
             { indices: iNames, onlyCheckIfIndicesExist },
             {
               abortSignal: abortCtrl.current.signal,
@@ -161,7 +160,6 @@ export const useFetchIndex = (
 
                 previousIndexesName.current = response.indicesExist;
                 setLoading(false);
-
                 setState({
                   browserFields: getBrowserFields(stringifyIndices, response.indexFields),
                   docValueFields: getDocValueFields(stringifyIndices, response.indexFields),
@@ -205,95 +203,3 @@ export const useFetchIndex = (
 
   return [isLoading, state];
 };
-
-export const useIndexFields = (sourcererScopeName: SourcererScopeName) => {
-  const { data } = useKibana().services;
-  const abortCtrl = useRef(new AbortController());
-  const searchSubscription$ = useRef(new Subscription());
-  const dispatch = useDispatch();
-  const indexNamesSelectedSelector = useMemo(
-    () => sourcererSelectors.getIndexNamesSelectedSelector(),
-    []
-  );
-  const { indexNames, previousIndexNames } = useDeepEqualSelector<{
-    indexNames: string[];
-    previousIndexNames: string;
-  }>((state) => indexNamesSelectedSelector(state, sourcererScopeName));
-  const { addError, addWarning } = useAppToasts();
-
-  const setLoading = useCallback(
-    (loading: boolean) => {
-      dispatch(sourcererActions.setSourcererScopeLoading({ id: sourcererScopeName, loading }));
-    },
-    [dispatch, sourcererScopeName]
-  );
-
-  const indexFieldsSearch = useCallback(
-    (indicesName) => {
-      const asyncSearch = async () => {
-        abortCtrl.current = new AbortController();
-        setLoading(true);
-        searchSubscription$.current = data.search
-          .search<IndexFieldsStrategyRequest, IndexFieldsStrategyResponse>(
-            { indices: indicesName, onlyCheckIfIndicesExist: false },
-            {
-              abortSignal: abortCtrl.current.signal,
-              strategy: 'indexFields',
-            }
-          )
-          .subscribe({
-            next: (response) => {
-              if (isCompleteResponse(response)) {
-                const stringifyIndices = response.indicesExist.sort().join();
-                dispatch(
-                  sourcererActions.setSource({
-                    id: sourcererScopeName,
-                    payload: {
-                      browserFields: getBrowserFields(stringifyIndices, response.indexFields),
-                      docValueFields: getDocValueFields(stringifyIndices, response.indexFields),
-                      errorMessage: null,
-                      id: sourcererScopeName,
-                      indexPattern: getIndexFields(stringifyIndices, response.indexFields),
-                      // If checking for DE signals index, lie and say the index is created (it's
-                      // no longer created on startup, but is created lazily before writing).
-                      indicesExist:
-                        sourcererScopeName === SourcererScopeName.detections
-                          ? true
-                          : response.indicesExist.length > 0,
-                      loading: false,
-                    },
-                  })
-                );
-                searchSubscription$.current.unsubscribe();
-              } else if (isErrorResponse(response)) {
-                setLoading(false);
-                addWarning(i18n.ERROR_BEAT_FIELDS);
-                searchSubscription$.current.unsubscribe();
-              }
-            },
-            error: (msg) => {
-              setLoading(false);
-              addError(msg, {
-                title: i18n.FAIL_BEAT_FIELDS,
-              });
-              searchSubscription$.current.unsubscribe();
-            },
-          });
-      };
-      searchSubscription$.current.unsubscribe();
-      abortCtrl.current.abort();
-      asyncSearch();
-    },
-    [data.search, dispatch, addError, addWarning, setLoading, sourcererScopeName]
-  );
-
-  useEffect(() => {
-    if (!isEmpty(indexNames) && previousIndexNames !== indexNames.sort().join()) {
-      indexFieldsSearch(indexNames);
-    }
-    return () => {
-      searchSubscription$.current.unsubscribe();
-      abortCtrl.current.abort();
-    };
-  }, [indexNames, indexFieldsSearch, previousIndexNames]);
-};

x-pack/plugins/security_solution/public/common/containers/source/mock.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { DEFAULT_INDEX_PATTERN } from '../../../../common/constants';
 import { DocValueFields } from '../../../../common/search_strategy';
 import { BrowserFields } from '../../../../common/search_strategy/index_fields';
@@ -22,6 +23,7 @@ export const mocksSource = {
       searchable: true,
       type: 'date',
       aggregatable: true,
+      readFromDocValues: true,
     },
     {
       category: 'agent',
@@ -330,7 +332,13 @@ export const mocksSource = {
 };
 
 export const mockIndexFields = [
-  { aggregatable: true, name: '@timestamp', searchable: true, type: 'date' },
+  {
+    aggregatable: true,
+    name: '@timestamp',
+    searchable: true,
+    type: 'date',
+    readFromDocValues: true,
+  },
   { aggregatable: true, name: 'agent.ephemeral_id', searchable: true, type: 'string' },
   { aggregatable: true, name: 'agent.hostname', searchable: true, type: 'string' },
   { aggregatable: true, name: 'agent.id', searchable: true, type: 'string' },
@@ -459,6 +467,7 @@ export const mockBrowserFields: BrowserFields = {
         name: '@timestamp',
         searchable: true,
         type: 'date',
+        readFromDocValues: true,
       },
     },
   },
@@ -726,3 +735,12 @@ export const mockDocValueFields: DocValueFields[] = [
     format: 'date_time',
   },
 ];
+
+export const mockRuntimeMappings: MappingRuntimeFields = {
+  '@a.runtime.field': {
+    script: {
+      source: 'emit("Radical dude: " + doc[\'host.name\'].value)',
+    },
+    type: 'keyword',
+  },
+};

x-pack/plugins/security_solution/public/common/containers/source/use_data_view.tsx

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useCallback, useEffect, useRef } from 'react';
+import { Subscription } from 'rxjs';
+import { useDispatch } from 'react-redux';
+import memoizeOne from 'memoize-one';
+import { pick } from 'lodash/fp';
+import { useKibana } from '../../lib/kibana';
+import { useAppToasts } from '../../hooks/use_app_toasts';
+import { sourcererActions } from '../../store/sourcerer';
+import {
+  DELETED_SECURITY_SOLUTION_DATA_VIEW,
+  IndexField,
+  IndexFieldsStrategyRequest,
+  IndexFieldsStrategyResponse,
+} from '../../../../../timelines/common';
+import {
+  FieldSpec,
+  isCompleteResponse,
+  isErrorResponse,
+} from '../../../../../../../src/plugins/data/common';
+import * as i18n from './translations';
+import { getBrowserFields, getDocValueFields } from './';
+
+const getEsFields = memoizeOne(
+  (fields: IndexField[]): FieldSpec[] =>
+    fields && fields.length > 0
+      ? fields.map((field) =>
+          pick(['name', 'searchable', 'type', 'aggregatable', 'esTypes', 'subType'], field)
+        )
+      : [],
+  (newArgs, lastArgs) => newArgs[0].length === lastArgs[0].length
+);
+
+export const useDataView = (): { indexFieldsSearch: (selectedDataViewId: string) => void } => {
+  const { data } = useKibana().services;
+  const abortCtrl = useRef(new AbortController());
+  const searchSubscription$ = useRef(new Subscription());
+  const dispatch = useDispatch();
+  const { addError, addWarning } = useAppToasts();
+
+  const setLoading = useCallback(
+    ({ id, loading }: { id: string; loading: boolean }) => {
+      dispatch(sourcererActions.setDataViewLoading({ id, loading }));
+    },
+    [dispatch]
+  );
+
+  const indexFieldsSearch = useCallback(
+    (selectedDataViewId: string) => {
+      const asyncSearch = async () => {
+        abortCtrl.current = new AbortController();
+        setLoading({ id: selectedDataViewId, loading: true });
+        searchSubscription$.current = data.search
+          .search<IndexFieldsStrategyRequest<'dataView'>, IndexFieldsStrategyResponse>(
+            {
+              dataViewId: selectedDataViewId,
+              onlyCheckIfIndicesExist: false,
+            },
+            {
+              abortSignal: abortCtrl.current.signal,
+              strategy: 'indexFields',
+            }
+          )
+          .subscribe({
+            next: (response) => {
+              if (isCompleteResponse(response)) {
+                const patternString = response.indicesExist.sort().join();
+
+                dispatch(
+                  sourcererActions.setDataView({
+                    browserFields: getBrowserFields(patternString, response.indexFields),
+                    docValueFields: getDocValueFields(patternString, response.indexFields),
+                    id: selectedDataViewId,
+                    indexFields: getEsFields(response.indexFields),
+                    loading: false,
+                    runtimeMappings: response.runtimeMappings,
+                  })
+                );
+                searchSubscription$.current.unsubscribe();
+              } else if (isErrorResponse(response)) {
+                setLoading({ id: selectedDataViewId, loading: false });
+                addWarning(i18n.ERROR_BEAT_FIELDS);
+                searchSubscription$.current.unsubscribe();
+              }
+            },
+            error: (msg) => {
+              if (msg.message === DELETED_SECURITY_SOLUTION_DATA_VIEW) {
+                // reload app if security solution data view is deleted
+                return location.reload();
+              }
+              setLoading({ id: selectedDataViewId, loading: false });
+              addError(msg, {
+                title: i18n.FAIL_BEAT_FIELDS,
+              });
+              searchSubscription$.current.unsubscribe();
+            },
+          });
+      };
+      searchSubscription$.current.unsubscribe();
+      abortCtrl.current.abort();
+      asyncSearch();
+    },
+    [addError, addWarning, data.search, dispatch, setLoading]
+  );
+
+  useEffect(() => {
+    return () => {
+      searchSubscription$.current.unsubscribe();
+      abortCtrl.current.abort();
+    };
+  }, []);
+
+  return { indexFieldsSearch };
+};

x-pack/plugins/security_solution/public/common/containers/sourcerer/api.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaServices } from '../../lib/kibana';
+import { SOURCERER_API_URL } from '../../../../common/constants';
+import { KibanaDataView } from '../../store/sourcerer/model';
+
+export interface GetSourcererDataView {
+  signal: AbortSignal;
+  body: {
+    patternList: string[];
+  };
+}
+
+export interface SecurityDataView {
+  defaultDataView: KibanaDataView;
+  kibanaDataViews: KibanaDataView[];
+}
+
+export const postSourcererDataView = async ({
+  body,
+  signal,
+}: GetSourcererDataView): Promise<SecurityDataView> =>
+  KibanaServices.get().http.fetch(SOURCERER_API_URL, {
+    method: 'POST',
+    body: JSON.stringify(body),
+    signal,
+  });

x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx

@@ -7,14 +7,14 @@
 
 import React from 'react';
 import { act, renderHook } from '@testing-library/react-hooks';
+import { waitFor } from '@testing-library/react';
 import { Provider } from 'react-redux';
 
-import { getScopeFromPath, useInitSourcerer } from '.';
+import { getScopeFromPath, useInitSourcerer, useSourcererDataView } from '.';
 import { mockPatterns } from './mocks';
-// import { SourcererScopeName } from '../../store/sourcerer/model';
 import { RouteSpyState } from '../../utils/route/types';
-import { SecurityPageName } from '../../../../common/constants';
-import { createStore, State } from '../../store';
+import { DEFAULT_INDEX_PATTERN, SecurityPageName } from '../../../../common/constants';
+import { createStore } from '../../store';
 import {
   useUserInfo,
   initialState as userInfoState,
@@ -24,8 +24,10 @@ import {
   kibanaObservable,
   mockGlobalState,
   SUB_PLUGINS_REDUCER,
+  mockSourcererState,
 } from '../../mock';
-import { SourcererScopeName } from '../../store/sourcerer/model';
+import { SelectedDataView, SourcererScopeName } from '../../store/sourcerer/model';
+import { postSourcererDataView } from './api';
 
 const mockRouteSpy: RouteSpyState = {
   pageName: SecurityPageName.overview,
@@ -37,6 +39,7 @@ const mockRouteSpy: RouteSpyState = {
 const mockDispatch = jest.fn();
 const mockUseUserInfo = useUserInfo as jest.Mock;
 jest.mock('../../../detections/components/user_info');
+jest.mock('./api');
 jest.mock('react-redux', () => {
   const original = jest.requireActual('react-redux');
 
@@ -48,6 +51,7 @@ jest.mock('react-redux', () => {
 jest.mock('../../utils/route/use_route_spy', () => ({
   useRouteSpy: () => [mockRouteSpy],
 }));
+
 jest.mock('../../lib/kibana', () => ({
   useToasts: jest.fn().mockReturnValue({
     addError: jest.fn(),
@@ -84,36 +88,13 @@ jest.mock('../../lib/kibana', () => ({
 }));
 
 describe('Sourcerer Hooks', () => {
-  const state: State = {
-    ...mockGlobalState,
-    sourcerer: {
-      ...mockGlobalState.sourcerer,
-      sourcererScopes: {
-        ...mockGlobalState.sourcerer.sourcererScopes,
-        [SourcererScopeName.default]: {
-          ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
-          indexPattern: {
-            fields: [],
-            title: '',
-          },
-        },
-        [SourcererScopeName.timeline]: {
-          ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline],
-          indexPattern: {
-            fields: [],
-            title: '',
-          },
-        },
-      },
-    },
-  };
   const { storage } = createSecuritySolutionStorageMock();
-  let store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+  let store: ReturnType<typeof createStore>;
 
   beforeEach(() => {
     jest.clearAllMocks();
     jest.restoreAllMocks();
-    store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    store = createStore(mockGlobalState, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
     mockUseUserInfo.mockImplementation(() => userInfoState);
   });
   it('initializes loading default and timeline index patterns', async () => {
@@ -125,34 +106,88 @@ describe('Sourcerer Hooks', () => {
       rerender();
       expect(mockDispatch).toBeCalledTimes(2);
       expect(mockDispatch.mock.calls[0][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_SOURCERER_SCOPE_LOADING',
-        payload: { id: 'default', loading: true },
+        type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
+        payload: { id: 'security-solution', loading: true },
       });
       expect(mockDispatch.mock.calls[1][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_SOURCERER_SCOPE_LOADING',
-        payload: { id: 'timeline', loading: true },
+        type: 'x-pack/security_solution/local/sourcerer/SET_SELECTED_DATA_VIEW',
+        payload: {
+          id: 'timeline',
+          selectedDataViewId: 'security-solution',
+          selectedPatterns: [
+            '.siem-signals-spacename',
+            'apm-*-transaction*',
+            'auditbeat-*',
+            'endgame-*',
+            'filebeat-*',
+            'logs-*',
+            'packetbeat-*',
+            'traces-apm*',
+            'winlogbeat-*',
+          ],
+        },
       });
     });
   });
   it('sets signal index name', async () => {
+    const mockNewDataViews = {
+      defaultDataView: mockSourcererState.defaultDataView,
+      kibanaDataViews: [mockSourcererState.defaultDataView],
+    };
+    (postSourcererDataView as jest.Mock).mockResolvedValue(mockNewDataViews);
+
+    store = createStore(
+      {
+        ...mockGlobalState,
+        sourcerer: {
+          ...mockGlobalState.sourcerer,
+          signalIndexName: null,
+          defaultDataView: {
+            ...mockGlobalState.sourcerer.defaultDataView,
+            title: DEFAULT_INDEX_PATTERN.join(','),
+            patternList: DEFAULT_INDEX_PATTERN,
+          },
+        },
+      },
+      SUB_PLUGINS_REDUCER,
+      kibanaObservable,
+      storage
+    );
     await act(async () => {
       mockUseUserInfo.mockImplementation(() => ({
         ...userInfoState,
         loading: false,
-        signalIndexName: 'signals-*',
+        signalIndexName: mockSourcererState.signalIndexName,
       }));
       const { rerender, waitForNextUpdate } = renderHook<string, void>(() => useInitSourcerer(), {
         wrapper: ({ children }) => <Provider store={store}>{children}</Provider>,
       });
       await waitForNextUpdate();
       rerender();
-      expect(mockDispatch.mock.calls[2][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_SIGNAL_INDEX_NAME',
-        payload: { signalIndexName: 'signals-*' },
-      });
-      expect(mockDispatch.mock.calls[3][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_SELECTED_INDEX_PATTERNS',
-        payload: { id: 'timeline', selectedPatterns: ['signals-*'] },
+      await waitFor(() => {
+        expect(mockDispatch.mock.calls[2][0]).toEqual({
+          type: 'x-pack/security_solution/local/sourcerer/SET_SOURCERER_SCOPE_LOADING',
+          payload: { loading: true },
+        });
+        expect(mockDispatch.mock.calls[3][0]).toEqual({
+          type: 'x-pack/security_solution/local/sourcerer/SET_SIGNAL_INDEX_NAME',
+          payload: { signalIndexName: mockSourcererState.signalIndexName },
+        });
+        expect(mockDispatch.mock.calls[4][0]).toEqual({
+          type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
+          payload: {
+            id: mockSourcererState.defaultDataView.id,
+            loading: true,
+          },
+        });
+        expect(mockDispatch.mock.calls[5][0]).toEqual({
+          type: 'x-pack/security_solution/local/sourcerer/SET_SOURCERER_DATA_VIEWS',
+          payload: mockNewDataViews,
+        });
+        expect(mockDispatch.mock.calls[6][0]).toEqual({
+          type: 'x-pack/security_solution/local/sourcerer/SET_SOURCERER_SCOPE_LOADING',
+          payload: { loading: false },
+        });
       });
     });
   });
@@ -160,7 +195,7 @@ describe('Sourcerer Hooks', () => {
     await act(async () => {
       mockUseUserInfo.mockImplementation(() => ({
         ...userInfoState,
-        signalIndexName: 'signals-*',
+        signalIndexName: mockSourcererState.signalIndexName,
         isSignalIndexExists: true,
       }));
       const { rerender, waitForNextUpdate } = renderHook<string, void>(
@@ -171,9 +206,130 @@ describe('Sourcerer Hooks', () => {
       );
       await waitForNextUpdate();
       rerender();
-      expect(mockDispatch.mock.calls[1][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_SELECTED_INDEX_PATTERNS',
-        payload: { id: 'detections', selectedPatterns: ['signals-*'] },
+      expect(mockDispatch.mock.calls[2][0]).toEqual({
+        type: 'x-pack/security_solution/local/sourcerer/SET_SELECTED_DATA_VIEW',
+        payload: {
+          id: 'detections',
+          selectedDataViewId: mockSourcererState.defaultDataView.id,
+          selectedPatterns: [mockSourcererState.signalIndexName],
+        },
+      });
+    });
+  });
+
+  describe('useSourcererDataView', () => {
+    it('Should exclude elastic cloud alias when selected patterns include "logs-*" as an alias', async () => {
+      await act(async () => {
+        const { result, rerender, waitForNextUpdate } = renderHook<
+          SourcererScopeName,
+          SelectedDataView
+        >(() => useSourcererDataView(), {
+          wrapper: ({ children }) => <Provider store={store}>{children}</Provider>,
+        });
+        await waitForNextUpdate();
+        rerender();
+        expect(result.current.selectedPatterns).toEqual([
+          '-*elastic-cloud-logs-*',
+          ...mockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns,
+        ]);
+      });
+    });
+
+    it('Should NOT exclude elastic cloud alias when selected patterns does NOT include "logs-*" as an alias', async () => {
+      await act(async () => {
+        store = createStore(
+          {
+            ...mockGlobalState,
+            sourcerer: {
+              ...mockGlobalState.sourcerer,
+              sourcererScopes: {
+                ...mockGlobalState.sourcerer.sourcererScopes,
+                [SourcererScopeName.default]: {
+                  ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+                  selectedPatterns: [
+                    'apm-*-transaction*',
+                    'auditbeat-*',
+                    'endgame-*',
+                    'filebeat-*',
+                    'packetbeat-*',
+                    'traces-apm*',
+                    'winlogbeat-*',
+                  ],
+                },
+              },
+            },
+          },
+          SUB_PLUGINS_REDUCER,
+          kibanaObservable,
+          storage
+        );
+        const { result, rerender, waitForNextUpdate } = renderHook<
+          SourcererScopeName,
+          SelectedDataView
+        >(() => useSourcererDataView(), {
+          wrapper: ({ children }) => <Provider store={store}>{children}</Provider>,
+        });
+        await waitForNextUpdate();
+        rerender();
+        expect(result.current.selectedPatterns).toEqual([
+          'apm-*-transaction*',
+          'auditbeat-*',
+          'endgame-*',
+          'filebeat-*',
+          'packetbeat-*',
+          'traces-apm*',
+          'winlogbeat-*',
+        ]);
+      });
+    });
+
+    it('Should NOT exclude elastic cloud alias when selected patterns include "logs-endpoint.event-*" as an alias', async () => {
+      await act(async () => {
+        store = createStore(
+          {
+            ...mockGlobalState,
+            sourcerer: {
+              ...mockGlobalState.sourcerer,
+              sourcererScopes: {
+                ...mockGlobalState.sourcerer.sourcererScopes,
+                [SourcererScopeName.default]: {
+                  ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+                  selectedPatterns: [
+                    'apm-*-transaction*',
+                    'auditbeat-*',
+                    'endgame-*',
+                    'filebeat-*',
+                    'packetbeat-*',
+                    'traces-apm*',
+                    'winlogbeat-*',
+                    'logs-endpoint.event-*',
+                  ],
+                },
+              },
+            },
+          },
+          SUB_PLUGINS_REDUCER,
+          kibanaObservable,
+          storage
+        );
+        const { result, rerender, waitForNextUpdate } = renderHook<
+          SourcererScopeName,
+          SelectedDataView
+        >(() => useSourcererDataView(), {
+          wrapper: ({ children }) => <Provider store={store}>{children}</Provider>,
+        });
+        await waitForNextUpdate();
+        rerender();
+        expect(result.current.selectedPatterns).toEqual([
+          'apm-*-transaction*',
+          'auditbeat-*',
+          'endgame-*',
+          'filebeat-*',
+          'logs-endpoint.event-*',
+          'packetbeat-*',
+          'traces-apm*',
+          'winlogbeat-*',
+        ]);
       });
     });
   });

x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx

@@ -5,133 +5,298 @@
  * 2.0.
  */
 
-import { useEffect, useMemo, useRef } from 'react';
+import { useCallback, useEffect, useMemo, useRef } from 'react';
 import { useDispatch } from 'react-redux';
-
+import { i18n } from '@kbn/i18n';
 import { matchPath } from 'react-router-dom';
 import { sourcererActions, sourcererSelectors } from '../../store/sourcerer';
-import { SourcererScopeName } from '../../store/sourcerer/model';
-import { useIndexFields } from '../source';
+import { SelectedDataView, SourcererScopeName } from '../../store/sourcerer/model';
 import { useUserInfo } from '../../../detections/components/user_info';
 import { timelineSelectors } from '../../../timelines/store/timeline';
 import { ALERTS_PATH, CASES_PATH, RULES_PATH, UEBA_PATH } from '../../../../common/constants';
 import { TimelineId } from '../../../../common';
 import { useDeepEqualSelector } from '../../hooks/use_selector';
+import { getScopePatternListSelection } from '../../store/sourcerer/helpers';
+import { useAppToasts } from '../../hooks/use_app_toasts';
+import { postSourcererDataView } from './api';
+import { useDataView } from '../source/use_data_view';
 
 export const useInitSourcerer = (
   scopeId: SourcererScopeName.default | SourcererScopeName.detections = SourcererScopeName.default
 ) => {
   const dispatch = useDispatch();
+  const abortCtrl = useRef(new AbortController());
   const initialTimelineSourcerer = useRef(true);
   const initialDetectionSourcerer = useRef(true);
   const { loading: loadingSignalIndex, isSignalIndexExists, signalIndexName } = useUserInfo();
-  const getConfigIndexPatternsSelector = useMemo(
-    () => sourcererSelectors.configIndexPatternsSelector(),
+  const getDefaultDataViewSelector = useMemo(
+    () => sourcererSelectors.defaultDataViewSelector(),
     []
   );
-  const ConfigIndexPatterns = useDeepEqualSelector(getConfigIndexPatternsSelector);
+  const defaultDataView = useDeepEqualSelector(getDefaultDataViewSelector);
+
+  const { addError } = useAppToasts();
+
+  useEffect(() => {
+    if (defaultDataView.error != null) {
+      addError(defaultDataView.error, {
+        title: i18n.translate('xpack.securitySolution.sourcerer.permissions.title', {
+          defaultMessage: 'Write role required to generate data',
+        }),
+        toastMessage: i18n.translate('xpack.securitySolution.sourcerer.permissions.toastMessage', {
+          defaultMessage:
+            'Users with write permission need to access the Elastic Security app to initialize the app source data.',
+        }),
+      });
+    }
+  }, [addError, defaultDataView.error]);
 
   const getSignalIndexNameSelector = useMemo(
     () => sourcererSelectors.signalIndexNameSelector(),
     []
   );
-  const signalIndexNameSelector = useDeepEqualSelector(getSignalIndexNameSelector);
+  const signalIndexNameSourcerer = useDeepEqualSelector(getSignalIndexNameSelector);
 
   const getTimelineSelector = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []);
   const activeTimeline = useDeepEqualSelector((state) =>
     getTimelineSelector(state, TimelineId.active)
   );
+  const scopeIdSelector = useMemo(() => sourcererSelectors.scopeIdSelector(), []);
+  const { selectedDataViewId: scopeDataViewId } = useDeepEqualSelector((state) =>
+    scopeIdSelector(state, scopeId)
+  );
+  const { selectedDataViewId: timelineDataViewId } = useDeepEqualSelector((state) =>
+    scopeIdSelector(state, SourcererScopeName.timeline)
+  );
+  const activeDataViewIds = useMemo(
+    () => [...new Set([scopeDataViewId, timelineDataViewId])],
+    [scopeDataViewId, timelineDataViewId]
+  );
+  const { indexFieldsSearch } = useDataView();
 
-  useIndexFields(scopeId);
-  useIndexFields(SourcererScopeName.timeline);
-
-  useEffect(() => {
-    if (!loadingSignalIndex && signalIndexName != null && signalIndexNameSelector == null) {
-      dispatch(sourcererActions.setSignalIndexName({ signalIndexName }));
-    }
-  }, [dispatch, loadingSignalIndex, signalIndexName, signalIndexNameSelector]);
+  useEffect(
+    () => activeDataViewIds.forEach((id) => id != null && id.length > 0 && indexFieldsSearch(id)),
+    [activeDataViewIds, indexFieldsSearch]
+  );
 
   // Related to timeline
   useEffect(() => {
     if (
       !loadingSignalIndex &&
       signalIndexName != null &&
-      signalIndexNameSelector == null &&
+      signalIndexNameSourcerer == null &&
       (activeTimeline == null || activeTimeline.savedObjectId == null) &&
-      initialTimelineSourcerer.current
+      initialTimelineSourcerer.current &&
+      defaultDataView.id.length > 0
     ) {
       initialTimelineSourcerer.current = false;
       dispatch(
-        sourcererActions.setSelectedIndexPatterns({
+        sourcererActions.setSelectedDataView({
           id: SourcererScopeName.timeline,
-          selectedPatterns: [...ConfigIndexPatterns, signalIndexName],
+          selectedDataViewId: defaultDataView.id,
+          selectedPatterns: getScopePatternListSelection(
+            defaultDataView,
+            SourcererScopeName.timeline,
+            signalIndexName,
+            true
+          ),
         })
       );
     } else if (
-      signalIndexNameSelector != null &&
+      signalIndexNameSourcerer != null &&
       (activeTimeline == null || activeTimeline.savedObjectId == null) &&
-      initialTimelineSourcerer.current
+      initialTimelineSourcerer.current &&
+      defaultDataView.id.length > 0
     ) {
       initialTimelineSourcerer.current = false;
       dispatch(
-        sourcererActions.setSelectedIndexPatterns({
+        sourcererActions.setSelectedDataView({
           id: SourcererScopeName.timeline,
-          selectedPatterns: [...ConfigIndexPatterns, signalIndexNameSelector],
+          selectedDataViewId: defaultDataView.id,
+          selectedPatterns: getScopePatternListSelection(
+            defaultDataView,
+            SourcererScopeName.timeline,
+            signalIndexNameSourcerer,
+            true
+          ),
         })
       );
     }
   }, [
     activeTimeline,
-    ConfigIndexPatterns,
+    defaultDataView,
     dispatch,
     loadingSignalIndex,
     signalIndexName,
-    signalIndexNameSelector,
+    signalIndexNameSourcerer,
   ]);
 
+  const updateSourcererDataView = useCallback(
+    (newSignalsIndex: string) => {
+      const asyncSearch = async (newPatternList: string[]) => {
+        abortCtrl.current = new AbortController();
+
+        dispatch(sourcererActions.setSourcererScopeLoading({ loading: true }));
+        try {
+          const response = await postSourcererDataView({
+            body: { patternList: newPatternList },
+            signal: abortCtrl.current.signal,
+          });
+
+          if (response.defaultDataView.patternList.includes(newSignalsIndex)) {
+            // first time signals is defined and validated in the sourcerer
+            // redo indexFieldsSearch
+            indexFieldsSearch(response.defaultDataView.id);
+          }
+          dispatch(sourcererActions.setSourcererDataViews(response));
+          dispatch(sourcererActions.setSourcererScopeLoading({ loading: false }));
+        } catch (err) {
+          addError(err, {
+            title: i18n.translate('xpack.securitySolution.sourcerer.error.title', {
+              defaultMessage: 'Error updating Security Data View',
+            }),
+            toastMessage: i18n.translate('xpack.securitySolution.sourcerer.error.toastMessage', {
+              defaultMessage: 'Refresh the page',
+            }),
+          });
+          dispatch(sourcererActions.setSourcererScopeLoading({ loading: false }));
+        }
+      };
+
+      if (defaultDataView.title.indexOf(newSignalsIndex) === -1) {
+        abortCtrl.current.abort();
+        asyncSearch([...defaultDataView.title.split(','), newSignalsIndex]);
+      }
+    },
+    [defaultDataView.title, dispatch, indexFieldsSearch, addError]
+  );
+  useEffect(() => {
+    if (
+      !loadingSignalIndex &&
+      signalIndexName != null &&
+      signalIndexNameSourcerer == null &&
+      defaultDataView.id.length > 0
+    ) {
+      // update signal name also updates sourcerer
+      // we hit this the first time signal index is created
+      updateSourcererDataView(signalIndexName);
+      dispatch(sourcererActions.setSignalIndexName({ signalIndexName }));
+    }
+  }, [
+    defaultDataView.id,
+    dispatch,
+    indexFieldsSearch,
+    isSignalIndexExists,
+    loadingSignalIndex,
+    signalIndexName,
+    signalIndexNameSourcerer,
+    updateSourcererDataView,
+  ]);
   // Related to the detection page
   useEffect(() => {
     if (
       scopeId === SourcererScopeName.detections &&
       isSignalIndexExists &&
       signalIndexName != null &&
-      initialDetectionSourcerer.current
+      initialDetectionSourcerer.current &&
+      defaultDataView.id.length > 0
     ) {
       initialDetectionSourcerer.current = false;
       dispatch(
-        sourcererActions.setSelectedIndexPatterns({
-          id: scopeId,
-          selectedPatterns: [signalIndexName],
+        sourcererActions.setSelectedDataView({
+          id: SourcererScopeName.detections,
+          selectedDataViewId: defaultDataView.id,
+          selectedPatterns: getScopePatternListSelection(
+            defaultDataView,
+            SourcererScopeName.detections,
+            signalIndexName,
+            true
+          ),
         })
       );
     } else if (
       scopeId === SourcererScopeName.detections &&
-      signalIndexNameSelector != null &&
-      initialTimelineSourcerer.current
+      signalIndexNameSourcerer != null &&
+      initialTimelineSourcerer.current &&
+      defaultDataView.id.length > 0
     ) {
       initialDetectionSourcerer.current = false;
-      dispatch(
-        sourcererActions.setSelectedIndexPatterns({
-          id: scopeId,
-          selectedPatterns: [signalIndexNameSelector],
-        })
-      );
+      sourcererActions.setSelectedDataView({
+        id: SourcererScopeName.detections,
+        selectedDataViewId: defaultDataView.id,
+        selectedPatterns: getScopePatternListSelection(
+          defaultDataView,
+          SourcererScopeName.detections,
+          signalIndexNameSourcerer,
+          true
+        ),
+      });
     }
-  }, [dispatch, isSignalIndexExists, scopeId, signalIndexName, signalIndexNameSelector]);
+  }, [
+    defaultDataView,
+    dispatch,
+    isSignalIndexExists,
+    scopeId,
+    signalIndexName,
+    signalIndexNameSourcerer,
+  ]);
 };
 
-export const useSourcererScope = (scope: SourcererScopeName = SourcererScopeName.default) => {
+const LOGS_WILDCARD_INDEX = 'logs-*';
+export const EXCLUDE_ELASTIC_CLOUD_INDEX = '-*elastic-cloud-logs-*';
+
+export const useSourcererDataView = (
+  scopeId: SourcererScopeName = SourcererScopeName.default
+): SelectedDataView => {
   const sourcererScopeSelector = useMemo(() => sourcererSelectors.getSourcererScopeSelector(), []);
-  return useDeepEqualSelector((state) => sourcererScopeSelector(state, scope));
+  const {
+    signalIndexName,
+    sourcererDataView: selectedDataView,
+    sourcererScope: { selectedPatterns: scopeSelectedPatterns, loading },
+  }: sourcererSelectors.SourcererScopeSelector = useDeepEqualSelector((state) =>
+    sourcererScopeSelector(state, scopeId)
+  );
+
+  const selectedPatterns = useMemo(
+    () =>
+      scopeSelectedPatterns.some((index) => index === LOGS_WILDCARD_INDEX)
+        ? [...scopeSelectedPatterns, EXCLUDE_ELASTIC_CLOUD_INDEX]
+        : scopeSelectedPatterns,
+    [scopeSelectedPatterns]
+  );
+
+  return useMemo(
+    () => ({
+      browserFields: selectedDataView.browserFields,
+      dataViewId: selectedDataView.id,
+      docValueFields: selectedDataView.docValueFields,
+      indexPattern: {
+        fields: selectedDataView.indexFields,
+        title: selectedPatterns.join(','),
+      },
+      indicesExist:
+        scopeId === SourcererScopeName.detections
+          ? selectedDataView.patternList.includes(`${signalIndexName}`)
+          : scopeId === SourcererScopeName.default
+          ? selectedDataView.patternList.filter((i) => i !== signalIndexName).length > 0
+          : selectedDataView.patternList.length > 0,
+      loading: loading || selectedDataView.loading,
+      runtimeMappings: selectedDataView.runtimeMappings,
+      // all active & inactive patterns in DATA_VIEW
+      patternList: selectedDataView.title.split(','),
+      // selected patterns in DATA_VIEW
+      selectedPatterns: selectedPatterns.sort(),
+    }),
+    [loading, selectedPatterns, signalIndexName, scopeId, selectedDataView]
+  );
 };
 
 export const getScopeFromPath = (
   pathname: string
-): SourcererScopeName.default | SourcererScopeName.detections => {
-  return matchPath(pathname, {
+): SourcererScopeName.default | SourcererScopeName.detections =>
+  matchPath(pathname, {
     path: [ALERTS_PATH, `${RULES_PATH}/id/:id`, `${UEBA_PATH}/:id`, `${CASES_PATH}/:detailName`],
     strict: false,
   }) == null
     ? SourcererScopeName.default
     : SourcererScopeName.detections;
-};

x-pack/plugins/security_solution/public/common/containers/sourcerer/use_signal_helpers.test.tsx

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import {
+  createSecuritySolutionStorageMock,
+  kibanaObservable,
+  mockGlobalState,
+  SUB_PLUGINS_REDUCER,
+  TestProviders,
+} from '../../mock';
+import { act, renderHook } from '@testing-library/react-hooks';
+import { useSignalHelpers } from './use_signal_helpers';
+import { createStore, State } from '../../store';
+
+describe('useSignalHelpers', () => {
+  const wrapperContainer: React.FC<{ children?: React.ReactNode }> = ({ children }) => (
+    <TestProviders>{children}</TestProviders>
+  );
+
+  test('Default state, does not need init and does not need poll', async () => {
+    await act(async () => {
+      const { result, waitForNextUpdate } = renderHook(() => useSignalHelpers(), {
+        wrapper: wrapperContainer,
+      });
+      await waitForNextUpdate();
+      expect(result.current.signalIndexNeedsInit).toEqual(false);
+      expect(result.current.pollForSignalIndex).toEqual(undefined);
+    });
+  });
+  test('Needs init and does not need poll when signal index is not yet in default data view', async () => {
+    const state: State = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        defaultDataView: {
+          ...mockGlobalState.sourcerer.defaultDataView,
+          title: `auditbeat-*,packetbeat-*`,
+          patternList: ['packetbeat-*', 'auditbeat-*'],
+        },
+        kibanaDataViews: [
+          {
+            ...mockGlobalState.sourcerer.defaultDataView,
+            title: `auditbeat-*,packetbeat-*`,
+            patternList: ['packetbeat-*', 'auditbeat-*'],
+          },
+        ],
+      },
+    };
+    const { storage } = createSecuritySolutionStorageMock();
+    const store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    await act(async () => {
+      const { result, waitForNextUpdate } = renderHook(() => useSignalHelpers(), {
+        wrapper: ({ children }) => <TestProviders store={store}>{children}</TestProviders>,
+      });
+      await waitForNextUpdate();
+      expect(result.current.signalIndexNeedsInit).toEqual(true);
+      expect(result.current.pollForSignalIndex).toEqual(undefined);
+    });
+  });
+  test('Init happened and signal index does not have data yet, poll function becomes available', async () => {
+    const state: State = {
+      ...mockGlobalState,
+      sourcerer: {
+        ...mockGlobalState.sourcerer,
+        defaultDataView: {
+          ...mockGlobalState.sourcerer.defaultDataView,
+          title: `auditbeat-*,packetbeat-*,${mockGlobalState.sourcerer.signalIndexName}`,
+          patternList: ['packetbeat-*', 'auditbeat-*'],
+        },
+        kibanaDataViews: [
+          {
+            ...mockGlobalState.sourcerer.defaultDataView,
+            title: `auditbeat-*,packetbeat-*,${mockGlobalState.sourcerer.signalIndexName}`,
+            patternList: ['packetbeat-*', 'auditbeat-*'],
+          },
+        ],
+      },
+    };
+    const { storage } = createSecuritySolutionStorageMock();
+    const store = createStore(state, SUB_PLUGINS_REDUCER, kibanaObservable, storage);
+    await act(async () => {
+      const { result, waitForNextUpdate } = renderHook(() => useSignalHelpers(), {
+        wrapper: ({ children }) => <TestProviders store={store}>{children}</TestProviders>,
+      });
+      await waitForNextUpdate();
+      expect(result.current.signalIndexNeedsInit).toEqual(false);
+      expect(result.current.pollForSignalIndex).not.toEqual(undefined);
+    });
+  });
+});

x-pack/plugins/security_solution/public/common/containers/sourcerer/use_signal_helpers.tsx

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useCallback, useMemo, useRef } from 'react';
+import { i18n } from '@kbn/i18n';
+import { useDispatch } from 'react-redux';
+import { sourcererSelectors } from '../../store';
+import { useDeepEqualSelector } from '../../hooks/use_selector';
+import { useSourcererDataView } from '.';
+import { SourcererScopeName } from '../../store/sourcerer/model';
+import { postSourcererDataView } from './api';
+import { sourcererActions } from '../../store/sourcerer';
+import { useDataView } from '../source/use_data_view';
+import { useAppToasts } from '../../hooks/use_app_toasts';
+
+export const useSignalHelpers = (): {
+  /* when defined, signal index has been initiated but does not exist */
+  pollForSignalIndex?: () => void;
+  /* when false, signal index has been initiated */
+  signalIndexNeedsInit: boolean;
+} => {
+  const { indicesExist } = useSourcererDataView(SourcererScopeName.detections);
+  const { indexFieldsSearch } = useDataView();
+  const dispatch = useDispatch();
+  const { addError } = useAppToasts();
+  const abortCtrl = useRef(new AbortController());
+
+  const getDefaultDataViewSelector = useMemo(
+    () => sourcererSelectors.defaultDataViewSelector(),
+    []
+  );
+  const getSignalIndexNameSelector = useMemo(
+    () => sourcererSelectors.signalIndexNameSelector(),
+    []
+  );
+  const signalIndexNameSourcerer = useDeepEqualSelector(getSignalIndexNameSelector);
+  const defaultDataView = useDeepEqualSelector(getDefaultDataViewSelector);
+
+  const signalIndexNeedsInit = useMemo(
+    () => !defaultDataView.title.includes(`${signalIndexNameSourcerer}`),
+    [defaultDataView.title, signalIndexNameSourcerer]
+  );
+  const shouldWePollForIndex = useMemo(
+    () => !indicesExist && !signalIndexNeedsInit,
+    [indicesExist, signalIndexNeedsInit]
+  );
+
+  const pollForSignalIndex = useCallback(() => {
+    const asyncSearch = async () => {
+      abortCtrl.current = new AbortController();
+      try {
+        const response = await postSourcererDataView({
+          body: { patternList: defaultDataView.title.split(',') },
+          signal: abortCtrl.current.signal,
+        });
+
+        if (
+          signalIndexNameSourcerer !== null &&
+          response.defaultDataView.patternList.includes(signalIndexNameSourcerer)
+        ) {
+          // first time signals is defined and validated in the sourcerer
+          // redo indexFieldsSearch
+          indexFieldsSearch(response.defaultDataView.id);
+          dispatch(sourcererActions.setSourcererDataViews(response));
+        }
+      } catch (err) {
+        addError(err, {
+          title: i18n.translate('xpack.securitySolution.sourcerer.error.title', {
+            defaultMessage: 'Error updating Security Data View',
+          }),
+          toastMessage: i18n.translate('xpack.securitySolution.sourcerer.error.toastMessage', {
+            defaultMessage: 'Refresh the page',
+          }),
+        });
+      }
+    };
+
+    if (signalIndexNameSourcerer !== null) {
+      abortCtrl.current.abort();
+      asyncSearch();
+    }
+  }, [addError, defaultDataView.title, dispatch, indexFieldsSearch, signalIndexNameSourcerer]);
+
+  return {
+    ...(shouldWePollForIndex ? { pollForSignalIndex } : {}),
+    signalIndexNeedsInit,
+  };
+};

x-pack/plugins/security_solution/public/common/lib/keury/index.ts

@@ -14,12 +14,12 @@ import {
   buildEsQuery,
   toElasticsearchQuery,
   fromKueryExpression,
-  IndexPatternBase,
+  DataViewBase,
 } from '@kbn/es-query';
 
 export const convertKueryToElasticSearchQuery = (
   kueryExpression: string,
-  indexPattern?: IndexPatternBase
+  indexPattern?: DataViewBase
 ) => {
   try {
     return kueryExpression
@@ -30,10 +30,7 @@ export const convertKueryToElasticSearchQuery = (
   }
 };
 
-export const convertKueryToDslFilter = (
-  kueryExpression: string,
-  indexPattern: IndexPatternBase
-) => {
+export const convertKueryToDslFilter = (kueryExpression: string, indexPattern: DataViewBase) => {
   try {
     return kueryExpression
       ? toElasticsearchQuery(fromKueryExpression(kueryExpression), indexPattern)
@@ -75,7 +72,7 @@ export const convertToBuildEsQuery = ({
   filters,
 }: {
   config: EsQueryConfig;
-  indexPattern: IndexPatternBase;
+  indexPattern: DataViewBase;
   queries: Query[];
   filters: Filter[];
 }): [string, undefined] | [undefined, Error] => {

x-pack/plugins/security_solution/public/common/mock/global_state.ts

@@ -26,6 +26,8 @@ import {
   DEFAULT_INTERVAL_TYPE,
   DEFAULT_INTERVAL_VALUE,
   DEFAULT_INDEX_PATTERN,
+  DEFAULT_DATA_VIEW_ID,
+  DEFAULT_SIGNALS_INDEX,
 } from '../../../common/constants';
 import { networkModel } from '../../network/store';
 import { uebaModel } from '../../ueba/store';
@@ -33,9 +35,30 @@ import { TimelineType, TimelineStatus, TimelineTabs } from '../../../common/type
 import { mockManagementState } from '../../management/store/reducer';
 import { ManagementState } from '../../management/types';
 import { initialSourcererState, SourcererScopeName } from '../store/sourcerer/model';
-import { mockBrowserFields, mockDocValueFields } from '../containers/source/mock';
-import { mockIndexPattern } from './index_pattern';
 import { allowedExperimentalValues } from '../../../common/experimental_features';
+import { getScopePatternListSelection } from '../store/sourcerer/helpers';
+import {
+  mockBrowserFields,
+  mockDocValueFields,
+  mockIndexFields,
+  mockRuntimeMappings,
+} from '../containers/source/mock';
+
+export const mockSourcererState = {
+  ...initialSourcererState,
+  signalIndexName: `${DEFAULT_SIGNALS_INDEX}-spacename`,
+  defaultDataView: {
+    ...initialSourcererState.defaultDataView,
+    browserFields: mockBrowserFields,
+    docValueFields: mockDocValueFields,
+    id: DEFAULT_DATA_VIEW_ID,
+    indexFields: mockIndexFields,
+    loading: false,
+    patternList: [...DEFAULT_INDEX_PATTERN, `${DEFAULT_SIGNALS_INDEX}-spacename`],
+    runtimeMappings: mockRuntimeMappings,
+    title: [...DEFAULT_INDEX_PATTERN, `${DEFAULT_SIGNALS_INDEX}-spacename`].join(','),
+  },
+};
 
 export const mockGlobalState: State = {
   app: {
@@ -241,6 +264,7 @@ export const mockGlobalState: State = {
       test: {
         activeTab: TimelineTabs.query,
         prevActiveTab: TimelineTabs.notes,
+        dataViewId: DEFAULT_DATA_VIEW_ID,
         deletedEventIds: [],
         documentType: '',
         queryFields: [],
@@ -294,24 +318,48 @@ export const mockGlobalState: State = {
     insertTimeline: null,
   },
   sourcerer: {
-    ...initialSourcererState,
+    ...mockSourcererState,
+    defaultDataView: {
+      ...mockSourcererState.defaultDataView,
+      title: `${mockSourcererState.defaultDataView.title},fakebeat-*`,
+    },
+    kibanaDataViews: [
+      {
+        ...mockSourcererState.defaultDataView,
+        title: `${mockSourcererState.defaultDataView.title},fakebeat-*`,
+      },
+    ],
     sourcererScopes: {
-      ...initialSourcererState.sourcererScopes,
+      ...mockSourcererState.sourcererScopes,
       [SourcererScopeName.default]: {
-        ...initialSourcererState.sourcererScopes[SourcererScopeName.default],
-        selectedPatterns: DEFAULT_INDEX_PATTERN,
-        browserFields: mockBrowserFields,
-        indexPattern: mockIndexPattern,
-        docValueFields: mockDocValueFields,
-        loading: false,
+        ...mockSourcererState.sourcererScopes[SourcererScopeName.default],
+        selectedDataViewId: mockSourcererState.defaultDataView.id,
+        selectedPatterns: getScopePatternListSelection(
+          mockSourcererState.defaultDataView,
+          SourcererScopeName.default,
+          mockSourcererState.signalIndexName,
+          true
+        ),
+      },
+      [SourcererScopeName.detections]: {
+        ...mockSourcererState.sourcererScopes[SourcererScopeName.detections],
+        selectedDataViewId: mockSourcererState.defaultDataView.id,
+        selectedPatterns: getScopePatternListSelection(
+          mockSourcererState.defaultDataView,
+          SourcererScopeName.detections,
+          mockSourcererState.signalIndexName,
+          true
+        ),
       },
       [SourcererScopeName.timeline]: {
-        ...initialSourcererState.sourcererScopes[SourcererScopeName.timeline],
-        selectedPatterns: DEFAULT_INDEX_PATTERN,
-        browserFields: mockBrowserFields,
-        indexPattern: mockIndexPattern,
-        docValueFields: mockDocValueFields,
-        loading: false,
+        ...mockSourcererState.sourcererScopes[SourcererScopeName.timeline],
+        selectedDataViewId: mockSourcererState.defaultDataView.id,
+        selectedPatterns: getScopePatternListSelection(
+          mockSourcererState.defaultDataView,
+          SourcererScopeName.timeline,
+          mockSourcererState.signalIndexName,
+          true
+        ),
       },
     },
   },

x-pack/plugins/security_solution/public/common/mock/index_pattern.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { IIndexPattern } from '../../../../../../src/plugins/data/common';
+import { SecuritySolutionDataViewBase } from '../types';
 
-export const mockIndexPattern: IIndexPattern = {
+export const mockIndexPattern: SecuritySolutionDataViewBase = {
   fields: [
     {
       name: '@timestamp',

x-pack/plugins/security_solution/public/common/mock/timeline_results.ts

@@ -1955,6 +1955,7 @@ export const mockTimelineModel: TimelineModel = {
   columns: mockTimelineModelColumns,
   defaultColumns: mockTimelineModelColumns,
   dataProviders: [],
+  dataViewId: '',
   dateRange: {
     end: '2020-03-18T13:52:38.929Z',
     start: '2020-03-18T13:46:38.929Z',
@@ -2091,6 +2092,7 @@ export const defaultTimelineProps: CreateTimelineProps = {
         queryMatch: { field: '_id', operator: ':', value: '1' },
       },
     ],
+    dataViewId: '',
     dateRange: { end: '2018-11-05T19:03:25.937Z', start: '2018-11-05T18:58:25.937Z' },
     deletedEventIds: [],
     description: '',

x-pack/plugins/security_solution/public/common/store/reducer.test.ts

@@ -5,10 +5,15 @@
  * 2.0.
  */
 
-import { parseExperimentalConfigValue } from '../../..//common/experimental_features';
+import { parseExperimentalConfigValue } from '../../../common/experimental_features';
 import { SecuritySubPlugins } from '../../app/types';
 import { createInitialState } from './reducer';
+import { mockSourcererState } from '../mock';
+import { useSourcererDataView } from '../containers/sourcerer';
+import { useDeepEqualSelector } from '../hooks/use_selector';
+import { renderHook } from '@testing-library/react-hooks';
 
+jest.mock('../hooks/use_selector');
 jest.mock('../lib/kibana', () => ({
   KibanaServices: {
     get: jest.fn(() => ({ uiSettings: { get: () => ({ from: 'now-24h', to: 'now' }) } })),
@@ -21,26 +26,38 @@ describe('createInitialState', () => {
       SecuritySubPlugins['store']['initialState'],
       'app' | 'dragAndDrop' | 'inputs' | 'sourcerer'
     >;
-    test('indicesExist should be TRUE if configIndexPatterns is NOT empty', () => {
-      const initState = createInitialState(mockPluginState, {
-        kibanaIndexPatterns: [{ id: '1234567890987654321', title: 'mock-kibana' }],
-        configIndexPatterns: ['auditbeat-*', 'filebeat'],
-        signalIndexName: 'siem-signals-default',
-        enableExperimental: parseExperimentalConfigValue([]),
-      });
+    const defaultState = {
+      defaultDataView: mockSourcererState.defaultDataView,
+      enableExperimental: parseExperimentalConfigValue([]),
+      kibanaDataViews: [mockSourcererState.defaultDataView],
+      signalIndexName: 'siem-signals-default',
+    };
+    const initState = createInitialState(mockPluginState, defaultState);
+    beforeEach(() => {
+      (useDeepEqualSelector as jest.Mock).mockImplementation((cb) => cb(initState));
+    });
+    afterEach(() => {
+      (useDeepEqualSelector as jest.Mock).mockClear();
+    });
 
-      expect(initState.sourcerer?.sourcererScopes.default.indicesExist).toEqual(true);
+    test('indicesExist should be TRUE if configIndexPatterns is NOT empty', async () => {
+      const { result } = renderHook(() => useSourcererDataView());
+      expect(result.current.indicesExist).toEqual(true);
     });
 
     test('indicesExist should be FALSE if configIndexPatterns is empty', () => {
-      const initState = createInitialState(mockPluginState, {
-        kibanaIndexPatterns: [{ id: '1234567890987654321', title: 'mock-kibana' }],
-        configIndexPatterns: [],
-        signalIndexName: 'siem-signals-default',
-        enableExperimental: parseExperimentalConfigValue([]),
+      const state = createInitialState(mockPluginState, {
+        ...defaultState,
+        defaultDataView: {
+          ...defaultState.defaultDataView,
+          id: '',
+          title: '',
+          patternList: [],
+        },
       });
-
-      expect(initState.sourcerer?.sourcererScopes.default.indicesExist).toEqual(false);
+      (useDeepEqualSelector as jest.Mock).mockImplementation((cb) => cb(state));
+      const { result } = renderHook(() => useSourcererDataView());
+      expect(result.current.indicesExist).toEqual(false);
     });
   });
 });

x-pack/plugins/security_solution/public/common/store/reducer.ts

@@ -21,15 +21,15 @@ import { SecuritySubPlugins } from '../../app/types';
 import { ManagementPluginReducer } from '../../management';
 import { State } from './types';
 import { AppAction } from './actions';
-import { KibanaIndexPatterns } from './sourcerer/model';
+import { initDataView, SourcererModel, SourcererScopeName } from './sourcerer/model';
 import { ExperimentalFeatures } from '../../../common/experimental_features';
+import { getScopePatternListSelection } from './sourcerer/helpers';
 
 export type SubPluginsInitReducer = HostsPluginReducer &
   UebaPluginReducer &
   NetworkPluginReducer &
   TimelinePluginReducer &
   ManagementPluginReducer;
-
 /**
  * Factory for the 'initialState' that is used to preload state into the Security App's redux store.
  */
@@ -39,17 +39,38 @@ export const createInitialState = (
     'app' | 'dragAndDrop' | 'inputs' | 'sourcerer'
   >,
   {
-    kibanaIndexPatterns,
-    configIndexPatterns,
+    defaultDataView,
+    kibanaDataViews,
     signalIndexName,
     enableExperimental,
   }: {
-    kibanaIndexPatterns: KibanaIndexPatterns;
-    configIndexPatterns: string[];
-    signalIndexName: string | null;
+    defaultDataView: SourcererModel['defaultDataView'];
+    kibanaDataViews: SourcererModel['kibanaDataViews'];
+    signalIndexName: SourcererModel['signalIndexName'];
     enableExperimental: ExperimentalFeatures;
   }
 ): State => {
+  const initialPatterns = {
+    [SourcererScopeName.default]: getScopePatternListSelection(
+      defaultDataView,
+      SourcererScopeName.default,
+      signalIndexName,
+      true
+    ),
+    [SourcererScopeName.detections]: getScopePatternListSelection(
+      defaultDataView,
+      SourcererScopeName.detections,
+      signalIndexName,
+      true
+    ),
+    [SourcererScopeName.timeline]: getScopePatternListSelection(
+      defaultDataView,
+      SourcererScopeName.timeline,
+      signalIndexName,
+      true
+    ),
+  };
+
   const preloadedState: State = {
     ...pluginsInitState,
     app: { ...initialAppState, enableExperimental },
@@ -59,13 +80,24 @@ export const createInitialState = (
       ...sourcererModel.initialSourcererState,
       sourcererScopes: {
         ...sourcererModel.initialSourcererState.sourcererScopes,
-        default: {
+        [SourcererScopeName.default]: {
           ...sourcererModel.initialSourcererState.sourcererScopes.default,
-          indicesExist: configIndexPatterns.length > 0,
+          selectedDataViewId: defaultDataView.id,
+          selectedPatterns: initialPatterns[SourcererScopeName.default],
+        },
+        [SourcererScopeName.detections]: {
+          ...sourcererModel.initialSourcererState.sourcererScopes.detections,
+          selectedDataViewId: defaultDataView.id,
+          selectedPatterns: initialPatterns[SourcererScopeName.detections],
+        },
+        [SourcererScopeName.timeline]: {
+          ...sourcererModel.initialSourcererState.sourcererScopes.timeline,
+          selectedDataViewId: defaultDataView.id,
+          selectedPatterns: initialPatterns[SourcererScopeName.timeline],
         },
       },
-      kibanaIndexPatterns,
-      configIndexPatterns,
+      defaultDataView,
+      kibanaDataViews: kibanaDataViews.map((dataView) => ({ ...initDataView, ...dataView })),
       signalIndexName,
     },
   };

x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts

@@ -8,35 +8,39 @@
 import actionCreatorFactory from 'typescript-fsa';
 import { TimelineEventsType } from '../../../../common/types/timeline';
 
-import { KibanaIndexPatterns, ManageScopeInit, SourcererScopeName } from './model';
+import { SourcererDataView, SourcererScopeName } from './model';
+import { SecurityDataView } from '../../containers/sourcerer/api';
 
 const actionCreator = actionCreatorFactory('x-pack/security_solution/local/sourcerer');
 
-export const setSource = actionCreator<{
-  id: SourcererScopeName;
-  payload: ManageScopeInit;
-}>('SET_SOURCE');
+export const setDataView = actionCreator<{
+  browserFields: SourcererDataView['browserFields'];
+  docValueFields: SourcererDataView['docValueFields'];
+  id: SourcererDataView['id'];
+  indexFields: SourcererDataView['indexFields'];
+  loading: SourcererDataView['loading'];
+  runtimeMappings: SourcererDataView['runtimeMappings'];
+}>('SET_DATA_VIEW');
 
-export const setIndexPatternsList = actionCreator<{
-  kibanaIndexPatterns: KibanaIndexPatterns;
-  configIndexPatterns: string[];
-}>('SET_INDEX_PATTERNS_LIST');
+export const setDataViewLoading = actionCreator<{
+  id: string;
+  loading: boolean;
+}>('SET_DATA_VIEW_LOADING');
 
 export const setSignalIndexName =
   actionCreator<{ signalIndexName: string }>('SET_SIGNAL_INDEX_NAME');
 
-export const setSourcererScopeLoading = actionCreator<{ id: SourcererScopeName; loading: boolean }>(
-  'SET_SOURCERER_SCOPE_LOADING'
-);
+export const setSourcererDataViews = actionCreator<SecurityDataView>('SET_SOURCERER_DATA_VIEWS');
 
-export const setSelectedIndexPatterns = actionCreator<{
+export const setSourcererScopeLoading = actionCreator<{
+  id?: SourcererScopeName;
+  loading: boolean;
+}>('SET_SOURCERER_SCOPE_LOADING');
+
+export interface SelectedDataViewPayload {
   id: SourcererScopeName;
+  selectedDataViewId: string;
   selectedPatterns: string[];
   eventType?: TimelineEventsType;
-}>('SET_SELECTED_INDEX_PATTERNS');
-
-export const initTimelineIndexPatterns = actionCreator<{
-  id: SourcererScopeName;
-  selectedPatterns: string[];
-  eventType?: TimelineEventsType;
-}>('INIT_TIMELINE_INDEX_PATTERNS');
+}
+export const setSelectedDataView = actionCreator<SelectedDataViewPayload>('SET_SELECTED_DATA_VIEW');

x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.test.ts

@@ -5,59 +5,243 @@
  * 2.0.
  */
 
-import { createDefaultIndexPatterns, Args } from './helpers';
-import { initialSourcererState, SourcererScopeName } from './model';
+import { mockGlobalState } from '../../mock';
+import { SourcererScopeName } from './model';
+import {
+  defaultDataViewByEventType,
+  getScopePatternListSelection,
+  validateSelectedPatterns,
+} from './helpers';
 
-let defaultArgs: Args = {
-  eventType: 'all',
-  id: SourcererScopeName.default,
-  selectedPatterns: ['auditbeat-*', 'packetbeat-*'],
-  state: {
-    ...initialSourcererState,
-    configIndexPatterns: ['filebeat-*', 'auditbeat-*', 'packetbeat-*'],
-    kibanaIndexPatterns: [{ id: '123', title: 'journalbeat-*' }],
-    signalIndexName: 'signals-*',
-  },
+const signalIndexName = mockGlobalState.sourcerer.signalIndexName;
+
+const dataView = {
+  ...mockGlobalState.sourcerer.defaultDataView,
+  title: `auditbeat-*,packetbeat-*,${signalIndexName}`,
+  patternList: ['packetbeat-*', 'auditbeat-*', `${signalIndexName}`],
 };
-const eventTypes: Array<Args['eventType']> = ['all', 'raw', 'alert', 'signal', 'custom'];
-const ids: Array<Args['id']> = [
-  SourcererScopeName.default,
-  SourcererScopeName.detections,
-  SourcererScopeName.timeline,
-];
-describe('createDefaultIndexPatterns', () => {
-  ids.forEach((id) => {
-    eventTypes.forEach((et) => {
-      describe(`id: ${id}, eventType: ${et}`, () => {
-        beforeEach(() => {
-          defaultArgs = {
-            ...defaultArgs,
-            id,
-            eventType: et,
-          };
+const patternListNoSignals = mockGlobalState.sourcerer.defaultDataView.patternList
+  .filter((p) => p !== signalIndexName)
+  .sort();
+const patternListSignals = [
+  signalIndexName,
+  ...mockGlobalState.sourcerer.defaultDataView.patternList.filter((p) => p !== signalIndexName),
+].sort();
+
+describe('sourcerer store helpers', () => {
+  describe('getScopePatternListSelection', () => {
+    it('is not a default data view, returns patternList sorted', () => {
+      const result = getScopePatternListSelection(
+        {
+          ...dataView,
+          id: '1234',
+        },
+        SourcererScopeName.default,
+        signalIndexName,
+        false
+      );
+      expect(result).toEqual([`${signalIndexName}`, 'auditbeat-*', 'packetbeat-*']);
+    });
+    it('default data view, SourcererScopeName.timeline, returns patternList sorted', () => {
+      const result = getScopePatternListSelection(
+        dataView,
+        SourcererScopeName.timeline,
+        signalIndexName,
+        true
+      );
+      expect(result).toEqual([signalIndexName, 'auditbeat-*', 'packetbeat-*']);
+    });
+    it('default data view, SourcererScopeName.default, returns patternList sorted without signals index', () => {
+      const result = getScopePatternListSelection(
+        dataView,
+        SourcererScopeName.default,
+        signalIndexName,
+        true
+      );
+      expect(result).toEqual(['auditbeat-*', 'packetbeat-*']);
+    });
+    it('default data view, SourcererScopeName.detections, returns patternList with only signals index', () => {
+      const result = getScopePatternListSelection(
+        dataView,
+        SourcererScopeName.detections,
+        signalIndexName,
+        true
+      );
+      expect(result).toEqual([signalIndexName]);
+    });
+  });
+  describe('validateSelectedPatterns', () => {
+    const payload = {
+      id: SourcererScopeName.default,
+      selectedDataViewId: dataView.id,
+      selectedPatterns: ['auditbeat-*'],
+    };
+    it('sets selectedPattern', () => {
+      const result = validateSelectedPatterns(mockGlobalState.sourcerer, payload);
+      expect(result).toEqual({
+        [SourcererScopeName.default]: {
+          ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+          selectedPatterns: ['auditbeat-*'],
+        },
+      });
+    });
+    it('sets to default when empty array is passed and scope is default', () => {
+      const result = validateSelectedPatterns(mockGlobalState.sourcerer, {
+        ...payload,
+        selectedPatterns: [],
+      });
+      expect(result).toEqual({
+        [SourcererScopeName.default]: {
+          ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default],
+          selectedPatterns: patternListNoSignals,
+        },
+      });
+    });
+    it('sets to default when empty array is passed and scope is detections', () => {
+      const result = validateSelectedPatterns(mockGlobalState.sourcerer, {
+        ...payload,
+        id: SourcererScopeName.detections,
+        selectedPatterns: [],
+      });
+      expect(result).toEqual({
+        [SourcererScopeName.detections]: {
+          ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.detections],
+          selectedDataViewId: dataView.id,
+          selectedPatterns: [signalIndexName],
+        },
+      });
+    });
+    it('sets to default when empty array is passed and scope is timeline', () => {
+      const result = validateSelectedPatterns(mockGlobalState.sourcerer, {
+        ...payload,
+        id: SourcererScopeName.timeline,
+        selectedPatterns: [],
+      });
+      expect(result).toEqual({
+        [SourcererScopeName.timeline]: {
+          ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline],
+          selectedDataViewId: dataView.id,
+          selectedPatterns: [
+            signalIndexName,
+            ...mockGlobalState.sourcerer.defaultDataView.patternList.filter(
+              (p) => p !== signalIndexName
+            ),
+          ].sort(),
+        },
+      });
+    });
+    describe('handles missing dataViewId, 7.16 -> 8.0', () => {
+      it('selectedPatterns.length > 0 & all selectedPatterns exist in defaultDataView, set dataViewId to defaultDataView.id', () => {
+        const result = validateSelectedPatterns(mockGlobalState.sourcerer, {
+          ...payload,
+          id: SourcererScopeName.timeline,
+          selectedDataViewId: '',
+          selectedPatterns: [
+            mockGlobalState.sourcerer.defaultDataView.patternList[3],
+            mockGlobalState.sourcerer.defaultDataView.patternList[4],
+          ],
         });
-        it('Selected patterns', () => {
-          const result = createDefaultIndexPatterns(defaultArgs);
-          expect(result).toEqual(['auditbeat-*', 'packetbeat-*']);
+        expect(result).toEqual({
+          [SourcererScopeName.timeline]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline],
+            selectedDataViewId: dataView.id,
+            selectedPatterns: [
+              mockGlobalState.sourcerer.defaultDataView.patternList[3],
+              mockGlobalState.sourcerer.defaultDataView.patternList[4],
+            ],
+          },
         });
-        it('No selected patterns', () => {
-          const newArgs = {
-            ...defaultArgs,
-            selectedPatterns: [],
-          };
-          const result = createDefaultIndexPatterns(newArgs);
-          if (
-            id === SourcererScopeName.detections ||
-            (id === SourcererScopeName.timeline && (et === 'alert' || et === 'signal'))
-          ) {
-            expect(result).toEqual(['signals-*']);
-          } else if (id === SourcererScopeName.timeline && et === 'all') {
-            expect(result).toEqual(['filebeat-*', 'auditbeat-*', 'packetbeat-*', 'signals-*']);
-          } else {
-            expect(result).toEqual(['filebeat-*', 'auditbeat-*', 'packetbeat-*']);
-          }
+      });
+      it('selectedPatterns.length > 0 & a pattern in selectedPatterns does not exist in defaultDataView, set dataViewId to null', () => {
+        const result = validateSelectedPatterns(mockGlobalState.sourcerer, {
+          ...payload,
+          id: SourcererScopeName.timeline,
+          selectedDataViewId: '',
+          selectedPatterns: [
+            mockGlobalState.sourcerer.defaultDataView.patternList[3],
+            'journalbeat-*',
+          ],
+        });
+        expect(result).toEqual({
+          [SourcererScopeName.timeline]: {
+            ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline],
+            selectedDataViewId: null,
+            selectedPatterns: [
+              mockGlobalState.sourcerer.defaultDataView.patternList[3],
+              'journalbeat-*',
+            ],
+          },
         });
       });
     });
   });
+  describe('defaultDataViewByEventType', () => {
+    it('defaults with no eventType', () => {
+      const result = defaultDataViewByEventType({ state: mockGlobalState.sourcerer });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: patternListSignals,
+      });
+    });
+    it('defaults with eventType: all', () => {
+      const result = defaultDataViewByEventType({
+        state: mockGlobalState.sourcerer,
+        eventType: 'all',
+      });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: patternListSignals,
+      });
+    });
+    it('defaults with eventType: raw', () => {
+      const result = defaultDataViewByEventType({
+        state: mockGlobalState.sourcerer,
+        eventType: 'raw',
+      });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: patternListNoSignals,
+      });
+    });
+    it('defaults with eventType: alert', () => {
+      const result = defaultDataViewByEventType({
+        state: mockGlobalState.sourcerer,
+        eventType: 'alert',
+      });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: [signalIndexName],
+      });
+    });
+    it('defaults with eventType: signal', () => {
+      const result = defaultDataViewByEventType({
+        state: mockGlobalState.sourcerer,
+        eventType: 'signal',
+      });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: [signalIndexName],
+      });
+    });
+    it('defaults with eventType: custom', () => {
+      const result = defaultDataViewByEventType({
+        state: mockGlobalState.sourcerer,
+        eventType: 'custom',
+      });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: patternListSignals,
+      });
+    });
+    it('defaults with eventType: eql', () => {
+      const result = defaultDataViewByEventType({
+        state: mockGlobalState.sourcerer,
+        eventType: 'eql',
+      });
+      expect(result).toEqual({
+        selectedDataViewId: dataView.id,
+        selectedPatterns: patternListSignals,
+      });
+    });
+  });
 });

x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts

@@ -6,8 +6,9 @@
  */
 
 import { isEmpty } from 'lodash';
-import { SourcererModel, SourcererScopeName } from './model';
-import { TimelineEventsType } from '../../../../common/types/timeline';
+import { SourcererDataView, SourcererModel, SourcererScopeById, SourcererScopeName } from './model';
+import { TimelineEventsType } from '../../../../common';
+import { SelectedDataViewPayload } from './actions';
 
 export interface Args {
   eventType?: TimelineEventsType;
@@ -15,40 +16,131 @@ export interface Args {
   selectedPatterns: string[];
   state: SourcererModel;
 }
-export const createDefaultIndexPatterns = ({ eventType, id, selectedPatterns, state }: Args) => {
-  const kibanaIndexPatterns = state.kibanaIndexPatterns.map((kip) => kip.title);
-  const newSelectedPatterns = selectedPatterns.filter(
-    (sp) =>
-      state.configIndexPatterns.includes(sp) ||
-      kibanaIndexPatterns.includes(sp) ||
-      (!isEmpty(state.signalIndexName) && state.signalIndexName === sp)
-  );
-  if (isEmpty(newSelectedPatterns)) {
-    let defaultIndexPatterns = state.configIndexPatterns;
-    if (id === SourcererScopeName.timeline && isEmpty(newSelectedPatterns)) {
-      defaultIndexPatterns = defaultIndexPatternByEventType({ state, eventType });
-    } else if (id === SourcererScopeName.detections && isEmpty(newSelectedPatterns)) {
-      defaultIndexPatterns = [state.signalIndexName ?? ''];
-    }
-    return defaultIndexPatterns;
+
+export const getScopePatternListSelection = (
+  theDataView: SourcererDataView | undefined,
+  sourcererScope: SourcererScopeName,
+  signalIndexName: SourcererModel['signalIndexName'],
+  isDefaultDataView: boolean
+): string[] => {
+  const patternList: string[] =
+    theDataView != null && theDataView.id !== null ? theDataView.patternList : [];
+
+  if (!isDefaultDataView) {
+    return patternList.sort();
+  }
+  // when our SIEM data view is set, here are the defaults
+  switch (sourcererScope) {
+    case SourcererScopeName.default:
+      return patternList.filter((index) => index !== signalIndexName).sort();
+    case SourcererScopeName.detections:
+      // set to signalIndexName whether or not it exists yet in the patternList
+      return (signalIndexName != null ? [signalIndexName] : []).sort();
+    case SourcererScopeName.timeline:
+      return (
+        signalIndexName != null
+          ? [
+              // remove signalIndexName in case its already in there and add it whether or not it exists yet in the patternList
+              ...patternList.filter((index) => index !== signalIndexName),
+              signalIndexName,
+            ]
+          : patternList
+      ).sort();
   }
-  return newSelectedPatterns;
 };
 
-export const defaultIndexPatternByEventType = ({
+export const validateSelectedPatterns = (
+  state: SourcererModel,
+  payload: SelectedDataViewPayload
+): Partial<SourcererScopeById> => {
+  const { id, eventType, ...rest } = payload;
+  let dataView = state.kibanaDataViews.find((p) => p.id === rest.selectedDataViewId);
+  // dedupe because these could come from a silly url or pre 8.0 timeline
+  const dedupePatterns = [...new Set(rest.selectedPatterns)];
+  let selectedPatterns =
+    dataView != null
+      ? dedupePatterns.filter(
+          (pattern) =>
+            // Typescript is being mean and telling me dataView could be undefined here
+            // so redoing the dataView != null check
+            (dataView != null && dataView.patternList.includes(pattern)) ||
+            // this is a hack, but sometimes signal index is deleted and is getting regenerated. it gets set before it is put in the dataView
+            state.signalIndexName == null
+        )
+      : // 7.16 -> 8.0 this will get hit because dataView == null
+        dedupePatterns;
+  if (selectedPatterns.length > 0 && dataView == null) {
+    // we have index patterns, but not a data view id
+    // find out if we have these index patterns in the defaultDataView
+    const areAllPatternsInDefault = selectedPatterns.every(
+      (pattern) => state.defaultDataView.title.indexOf(pattern) > -1
+    );
+    if (areAllPatternsInDefault) {
+      dataView = state.defaultDataView;
+      selectedPatterns = selectedPatterns.filter(
+        (pattern) => dataView != null && dataView.patternList.includes(pattern)
+      );
+    }
+  }
+
+  // TO DO: Steph/sourcerer If dataView is still undefined here, create temporary dataView
+  // and prompt user to go create this dataView
+  // currently UI will take the undefined dataView and default to defaultDataView anyways
+  // this is a "strategically merged" bug ;)
+  // https://github.com/elastic/security-team/issues/1921
+
+  return {
+    [id]: {
+      ...state.sourcererScopes[id],
+      ...rest,
+      selectedDataViewId: dataView?.id ?? null,
+      selectedPatterns,
+      ...(isEmpty(selectedPatterns)
+        ? id === SourcererScopeName.timeline
+          ? defaultDataViewByEventType({ state, eventType })
+          : {
+              selectedPatterns: getScopePatternListSelection(
+                dataView ?? state.defaultDataView,
+                id,
+                state.signalIndexName,
+                (dataView ?? state.defaultDataView).id === state.defaultDataView.id
+              ),
+            }
+        : {}),
+      loading: false,
+    },
+  };
+};
+
+// TODO: Steph/sourcerer eventType will be alerts only, when ui updates delete raw
+export const defaultDataViewByEventType = ({
   state,
   eventType,
 }: {
   state: SourcererModel;
   eventType?: TimelineEventsType;
 }) => {
-  let defaultIndexPatterns = state.configIndexPatterns;
-  if (eventType === 'all' && !isEmpty(state.signalIndexName)) {
-    defaultIndexPatterns = [...state.configIndexPatterns, state.signalIndexName ?? ''];
+  const {
+    signalIndexName,
+    defaultDataView: { id, patternList },
+  } = state;
+  if (signalIndexName != null && (eventType === 'signal' || eventType === 'alert')) {
+    return {
+      selectedPatterns: [signalIndexName],
+      selectedDataViewId: id,
+    };
   } else if (eventType === 'raw') {
-    defaultIndexPatterns = state.configIndexPatterns;
-  } else if (!isEmpty(state.signalIndexName) && (eventType === 'signal' || eventType === 'alert')) {
-    defaultIndexPatterns = [state.signalIndexName ?? ''];
+    return {
+      selectedPatterns: patternList.filter((index) => index !== signalIndexName).sort(),
+      selectedDataViewId: id,
+    };
   }
-  return defaultIndexPatterns;
+  return {
+    selectedPatterns: [
+      // remove signalIndexName in case its already in there and add it whether or not it exists yet in the patternList
+      ...patternList.filter((index) => index !== signalIndexName),
+      signalIndexName,
+    ].sort(),
+    selectedDataViewId: id,
+  };
 };

x-pack/plugins/security_solution/public/common/store/sourcerer/model.ts

@@ -5,72 +5,133 @@
  * 2.0.
  */
 
-import { IIndexPattern } from '../../../../../../../src/plugins/data/common';
-import { DocValueFields } from '../../../../common/search_strategy/common';
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import {
   BrowserFields,
+  DocValueFields,
   EMPTY_BROWSER_FIELDS,
   EMPTY_DOCVALUE_FIELD,
-  EMPTY_INDEX_PATTERN,
-} from '../../../../common/search_strategy/index_fields';
-
-export type ErrorModel = Error[];
-
+  EMPTY_INDEX_FIELDS,
+} from '../../../../../timelines/common';
+import { SecuritySolutionDataViewBase } from '../../types';
+/** Uniquely identifies a Sourcerer Scope */
 export enum SourcererScopeName {
   default = 'default',
   detections = 'detections',
   timeline = 'timeline',
 }
 
-export interface ManageScope {
-  browserFields: BrowserFields;
-  docValueFields: DocValueFields[];
-  errorMessage: string | null;
+/**
+ * Data related to each sourcerer scope
+ */
+export interface SourcererScope {
+  /** Uniquely identifies a Sourcerer Scope */
   id: SourcererScopeName;
-  indexPattern: Omit<IIndexPattern, 'fieldFormatMap'>;
-  indicesExist: boolean | undefined | null;
+  /** is an update being made to the sourcerer data view */
   loading: boolean;
+  /** selected data view id */
+  selectedDataViewId: string;
+  /** selected patterns within the data view */
   selectedPatterns: string[];
 }
 
-export interface ManageScopeInit extends Partial<ManageScope> {
-  id: SourcererScopeName;
+export type SourcererScopeById = Record<SourcererScopeName, SourcererScope>;
+
+export interface KibanaDataView {
+  /** Uniquely identifies a Kibana Data View */
+  id: string;
+  /**  list of active patterns that return data  */
+  patternList: string[];
+  /**
+   * title of Kibana Data View
+   * title also serves as "all pattern list", including inactive
+   * comma separated string
+   */
+  title: string;
 }
 
-export type SourcererScopeById = Record<SourcererScopeName | string, ManageScope>;
+/**
+ * DataView from Kibana + timelines/index_fields enhanced field data
+ */
+export interface SourcererDataView extends KibanaDataView {
+  /** we need this for @timestamp data */
+  browserFields: BrowserFields;
+  /** we need this for @timestamp data */
+  docValueFields: DocValueFields[];
+  /** comes from dataView.fields.toSpec() */
+  indexFields: SecuritySolutionDataViewBase['fields'];
+  /** set when data view fields are fetched */
+  loading: boolean;
+  /**
+   * Needed to pass to search strategy
+   * Remove once issue resolved: https://github.com/elastic/kibana/issues/111762
+   */
+  runtimeMappings: MappingRuntimeFields;
+}
 
-export type KibanaIndexPatterns = Array<{ id: string; title: string }>;
+/**
+ * Combined data from SourcererDataView and SourcererScope to create
+ * selected data view state
+ */
+export interface SelectedDataView {
+  browserFields: SourcererDataView['browserFields'];
+  dataViewId: SourcererDataView['id'];
+  docValueFields: SourcererDataView['docValueFields'];
+  /**
+   * DataViewBase with enhanced index fields used in timelines
+   */
+  indexPattern: SecuritySolutionDataViewBase;
+  /** do the selected indices exist  */
+  indicesExist: boolean;
+  /** is an update being made to the data view */
+  loading: boolean;
+  /** all active & inactive patterns from SourcererDataView['title']  */
+  patternList: string[];
+  runtimeMappings: SourcererDataView['runtimeMappings'];
+  /** all selected patterns from SourcererScope['selectedPatterns']  */
+  selectedPatterns: string[];
+}
 
-// ManageSourcerer
+/**
+ * sourcerer model for redux
+ */
 export interface SourcererModel {
-  kibanaIndexPatterns: KibanaIndexPatterns;
-  configIndexPatterns: string[];
+  /** default security-solution data view */
+  defaultDataView: SourcererDataView & { error?: unknown };
+  /** all Kibana data views, including security-solution */
+  kibanaDataViews: SourcererDataView[];
+  /** security solution signals index name */
   signalIndexName: string | null;
+  /** sourcerer scope data by id */
   sourcererScopes: SourcererScopeById;
 }
 
-export const initSourcererScope: Pick<
-  ManageScope,
-  | 'browserFields'
-  | 'docValueFields'
-  | 'errorMessage'
-  | 'indexPattern'
-  | 'indicesExist'
-  | 'loading'
-  | 'selectedPatterns'
-> = {
+export type SourcererUrlState = Partial<{
+  [id in SourcererScopeName]: {
+    id: string;
+    selectedPatterns: string[];
+  };
+}>;
+
+export const initSourcererScope: Omit<SourcererScope, 'id'> = {
+  loading: false,
+  selectedDataViewId: '',
+  selectedPatterns: [],
+};
+export const initDataView = {
   browserFields: EMPTY_BROWSER_FIELDS,
   docValueFields: EMPTY_DOCVALUE_FIELD,
-  errorMessage: null,
-  indexPattern: EMPTY_INDEX_PATTERN,
-  indicesExist: true,
+  id: '',
+  indexFields: EMPTY_INDEX_FIELDS,
   loading: false,
-  selectedPatterns: [],
+  patternList: [],
+  runtimeMappings: {},
+  title: '',
 };
 
 export const initialSourcererState: SourcererModel = {
-  kibanaIndexPatterns: [],
-  configIndexPatterns: [],
+  defaultDataView: initDataView,
+  kibanaDataViews: [],
   signalIndexName: null,
   sourcererScopes: {
     [SourcererScopeName.default]: {
@@ -87,8 +148,3 @@ export const initialSourcererState: SourcererModel = {
     },
   },
 };
-
-export type FSourcererScopePatterns = {
-  [id in SourcererScopeName]: string[];
-};
-export type SourcererScopePatterns = Partial<FSourcererScopePatterns>;

x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts

@@ -5,83 +5,89 @@
  * 2.0.
  */
 
-import { isEmpty } from 'lodash/fp';
 import { reducerWithInitialState } from 'typescript-fsa-reducers';
 
 import {
-  setIndexPatternsList,
+  setSourcererDataViews,
   setSourcererScopeLoading,
-  setSelectedIndexPatterns,
+  setSelectedDataView,
   setSignalIndexName,
-  setSource,
-  initTimelineIndexPatterns,
+  setDataView,
+  setDataViewLoading,
 } from './actions';
-import { initialSourcererState, SourcererModel } from './model';
-import { createDefaultIndexPatterns, defaultIndexPatternByEventType } from './helpers';
+import { initDataView, initialSourcererState, SourcererModel, SourcererScopeName } from './model';
+import { validateSelectedPatterns } from './helpers';
 
 export type SourcererState = SourcererModel;
 
 export const sourcererReducer = reducerWithInitialState(initialSourcererState)
-  .case(setIndexPatternsList, (state, { kibanaIndexPatterns, configIndexPatterns }) => ({
-    ...state,
-    kibanaIndexPatterns,
-    configIndexPatterns,
-  }))
   .case(setSignalIndexName, (state, { signalIndexName }) => ({
     ...state,
     signalIndexName,
   }))
+  .case(setDataViewLoading, (state, { id, loading }) => ({
+    ...state,
+    ...(id === state.defaultDataView.id
+      ? {
+          defaultDataView: { ...state.defaultDataView, loading },
+        }
+      : {}),
+    kibanaDataViews: state.kibanaDataViews.map((dv) => (dv.id === id ? { ...dv, loading } : dv)),
+  }))
+  .case(setSourcererDataViews, (state, { defaultDataView, kibanaDataViews }) => ({
+    ...state,
+    defaultDataView: {
+      ...state.defaultDataView,
+      ...defaultDataView,
+    },
+    kibanaDataViews: kibanaDataViews.map((dataView) => ({
+      ...(state.kibanaDataViews.find(({ id }) => id === dataView.id) ?? initDataView),
+      ...dataView,
+    })),
+  }))
   .case(setSourcererScopeLoading, (state, { id, loading }) => ({
     ...state,
     sourcererScopes: {
       ...state.sourcererScopes,
-      [id]: {
-        ...state.sourcererScopes[id],
-        loading,
-      },
+      ...(id != null
+        ? {
+            [id]: {
+              ...state.sourcererScopes[id],
+              loading,
+            },
+          }
+        : {
+            [SourcererScopeName.default]: {
+              ...state.sourcererScopes[SourcererScopeName.default],
+              loading,
+            },
+            [SourcererScopeName.detections]: {
+              ...state.sourcererScopes[SourcererScopeName.detections],
+              loading,
+            },
+            [SourcererScopeName.timeline]: {
+              ...state.sourcererScopes[SourcererScopeName.timeline],
+              loading,
+            },
+          }),
     },
   }))
-  .case(setSelectedIndexPatterns, (state, { id, selectedPatterns, eventType }) => {
-    return {
-      ...state,
-      sourcererScopes: {
-        ...state.sourcererScopes,
-        [id]: {
-          ...state.sourcererScopes[id],
-          selectedPatterns: createDefaultIndexPatterns({ eventType, id, selectedPatterns, state }),
-        },
-      },
-    };
-  })
-  .case(initTimelineIndexPatterns, (state, { id, selectedPatterns, eventType }) => {
-    return {
-      ...state,
-      sourcererScopes: {
-        ...state.sourcererScopes,
-        [id]: {
-          ...state.sourcererScopes[id],
-          selectedPatterns: isEmpty(selectedPatterns)
-            ? defaultIndexPatternByEventType({ state, eventType })
-            : selectedPatterns,
-        },
-      },
-    };
-  })
-
-  .case(setSource, (state, { id, payload }) => {
-    const { ...sourcererScopes } = payload;
-    return {
-      ...state,
-      sourcererScopes: {
-        ...state.sourcererScopes,
-        [id]: {
-          ...state.sourcererScopes[id],
-          ...sourcererScopes,
-          ...(state.sourcererScopes[id].selectedPatterns.length === 0
-            ? { selectedPatterns: state.configIndexPatterns }
-            : {}),
-        },
-      },
-    };
-  })
+  .case(setSelectedDataView, (state, payload) => ({
+    ...state,
+    sourcererScopes: {
+      ...state.sourcererScopes,
+      ...validateSelectedPatterns(state, payload),
+    },
+  }))
+  .case(setDataView, (state, dataView) => ({
+    ...state,
+    ...(dataView.id === state.defaultDataView.id
+      ? {
+          defaultDataView: { ...state.defaultDataView, ...dataView },
+        }
+      : {}),
+    kibanaDataViews: state.kibanaDataViews.map((dv) =>
+      dv.id === dataView.id ? { ...dv, ...dataView } : dv
+    ),
+  }))
   .build();

x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts

@@ -1,75 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { cloneDeep } from 'lodash/fp';
-import { mockGlobalState } from '../../mock';
-import { SourcererScopeName } from './model';
-import { getSourcererScopeSelector } from './selectors';
-
-describe('Sourcerer selectors', () => {
-  describe('getSourcererScopeSelector', () => {
-    it('Should exclude elastic cloud alias when selected patterns include "logs-*" as an alias', () => {
-      const mapStateToProps = getSourcererScopeSelector();
-      expect(mapStateToProps(mockGlobalState, SourcererScopeName.default).selectedPatterns).toEqual(
-        [
-          'apm-*-transaction*',
-          'auditbeat-*',
-          'endgame-*',
-          'filebeat-*',
-          'logs-*',
-          'packetbeat-*',
-          'traces-apm*',
-          'winlogbeat-*',
-          '-*elastic-cloud-logs-*',
-        ]
-      );
-    });
-
-    it('Should NOT exclude elastic cloud alias when selected patterns does NOT include "logs-*" as an alias', () => {
-      const mapStateToProps = getSourcererScopeSelector();
-      const myMockGlobalState = cloneDeep(mockGlobalState);
-      myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns =
-        myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns.filter(
-          (index) => !index.includes('logs-*')
-        );
-      expect(
-        mapStateToProps(myMockGlobalState, SourcererScopeName.default).selectedPatterns
-      ).toEqual([
-        'apm-*-transaction*',
-        'auditbeat-*',
-        'endgame-*',
-        'filebeat-*',
-        'packetbeat-*',
-        'traces-apm*',
-        'winlogbeat-*',
-      ]);
-    });
-
-    it('Should NOT exclude elastic cloud alias when selected patterns include "logs-endpoint.event-*" as an alias', () => {
-      const mapStateToProps = getSourcererScopeSelector();
-      const myMockGlobalState = cloneDeep(mockGlobalState);
-      myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns = [
-        ...myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns.filter(
-          (index) => !index.includes('logs-*')
-        ),
-        'logs-endpoint.event-*',
-      ];
-      expect(
-        mapStateToProps(myMockGlobalState, SourcererScopeName.default).selectedPatterns
-      ).toEqual([
-        'apm-*-transaction*',
-        'auditbeat-*',
-        'endgame-*',
-        'filebeat-*',
-        'logs-endpoint.event-*',
-        'packetbeat-*',
-        'traces-apm*',
-        'winlogbeat-*',
-      ]);
-    });
-  });
-});

x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts

@@ -5,24 +5,34 @@
  * 2.0.
  */
 
-import memoizeOne from 'memoize-one';
 import { createSelector } from 'reselect';
 import { State } from '../types';
-import { SourcererScopeById, ManageScope, KibanaIndexPatterns, SourcererScopeName } from './model';
+import {
+  SourcererDataView,
+  SourcererModel,
+  SourcererScope,
+  SourcererScopeById,
+  SourcererScopeName,
+} from './model';
 
-export const sourcererKibanaIndexPatternsSelector = ({ sourcerer }: State): KibanaIndexPatterns =>
-  sourcerer.kibanaIndexPatterns;
+export const sourcererKibanaDataViewsSelector = ({
+  sourcerer,
+}: State): SourcererModel['kibanaDataViews'] => sourcerer.kibanaDataViews;
 
 export const sourcererSignalIndexNameSelector = ({ sourcerer }: State): string | null =>
   sourcerer.signalIndexName;
 
-export const sourcererConfigIndexPatternsSelector = ({ sourcerer }: State): string[] =>
-  sourcerer.configIndexPatterns;
+export const sourcererDefaultDataViewSelector = ({
+  sourcerer,
+}: State): SourcererModel['defaultDataView'] => sourcerer.defaultDataView;
+
+export const dataViewSelector = ({ sourcerer }: State, id: string): SourcererDataView =>
+  sourcerer.kibanaDataViews.find((dataView) => dataView.id === id) ?? sourcerer.defaultDataView;
 
 export const sourcererScopeIdSelector = (
   { sourcerer }: State,
   scopeId: SourcererScopeName
-): ManageScope => sourcerer.sourcererScopes[scopeId];
+): SourcererScope => sourcerer.sourcererScopes[scopeId];
 
 export const scopeIdSelector = () => createSelector(sourcererScopeIdSelector, (scope) => scope);
 
@@ -31,85 +41,43 @@ export const sourcererScopesSelector = ({ sourcerer }: State): SourcererScopeByI
 
 export const scopesSelector = () => createSelector(sourcererScopesSelector, (scopes) => scopes);
 
-export const kibanaIndexPatternsSelector = () =>
-  createSelector(
-    sourcererKibanaIndexPatternsSelector,
-    (kibanaIndexPatterns) => kibanaIndexPatterns
-  );
+export const kibanaDataViewsSelector = () =>
+  createSelector(sourcererKibanaDataViewsSelector, (dataViews) => dataViews);
 
 export const signalIndexNameSelector = () =>
   createSelector(sourcererSignalIndexNameSelector, (signalIndexName) => signalIndexName);
 
-export const configIndexPatternsSelector = () =>
-  createSelector(
-    sourcererConfigIndexPatternsSelector,
-    (configIndexPatterns) => configIndexPatterns
-  );
+export const defaultDataViewSelector = () =>
+  createSelector(sourcererDefaultDataViewSelector, (dataViews) => dataViews);
 
-export const getIndexNamesSelectedSelector = () => {
-  const getScopeSelector = scopeIdSelector();
-  const getConfigIndexPatternsSelector = configIndexPatternsSelector();
+export const sourcererDataViewSelector = () =>
+  createSelector(dataViewSelector, (dataView) => dataView);
 
-  const mapStateToProps = (
-    state: State,
-    scopeId: SourcererScopeName
-  ): { indexNames: string[]; previousIndexNames: string } => {
-    const scope = getScopeSelector(state, scopeId);
-    const configIndexPatterns = getConfigIndexPatternsSelector(state);
-    return {
-      indexNames:
-        scope.selectedPatterns.length === 0 ? configIndexPatterns : scope.selectedPatterns,
-      previousIndexNames: scope.indexPattern.title,
-    };
-  };
-  return mapStateToProps;
-};
-
-export const getAllExistingIndexNamesSelector = () => {
-  const getSignalIndexNameSelector = signalIndexNameSelector();
-  const getConfigIndexPatternsSelector = configIndexPatternsSelector();
-
-  const mapStateToProps = (state: State): string[] => {
-    const signalIndexName = getSignalIndexNameSelector(state);
-    const configIndexPatterns = getConfigIndexPatternsSelector(state);
-
-    return signalIndexName != null
-      ? [...configIndexPatterns, signalIndexName]
-      : configIndexPatterns;
-  };
-
-  return mapStateToProps;
-};
-
-const EXCLUDE_ELASTIC_CLOUD_INDEX = '-*elastic-cloud-logs-*';
+export interface SourcererScopeSelector extends Omit<SourcererModel, 'sourcererScopes'> {
+  sourcererDataView: SourcererDataView;
+  sourcererScope: SourcererScope;
+}
 
 export const getSourcererScopeSelector = () => {
-  const getScopeIdSelector = scopeIdSelector();
-  const getSelectedPatterns = memoizeOne((selectedPatternsStr: string): string[] => {
-    const selectedPatterns = selectedPatternsStr.length > 0 ? selectedPatternsStr.split(',') : [];
-    return selectedPatterns.some((index) => index === 'logs-*')
-      ? [...selectedPatterns, EXCLUDE_ELASTIC_CLOUD_INDEX]
-      : selectedPatterns;
-  });
+  const getKibanaDataViewsSelector = kibanaDataViewsSelector();
+  const getDefaultDataViewSelector = defaultDataViewSelector();
+  const getSignalIndexNameSelector = signalIndexNameSelector();
+  const getSourcererDataViewSelector = sourcererDataViewSelector();
+  const getScopeSelector = scopeIdSelector();
 
-  const getIndexPattern = memoizeOne(
-    (indexPattern, title) => ({
-      ...indexPattern,
-      title,
-    }),
-    (newArgs, lastArgs) => newArgs[0] === lastArgs[0] && newArgs[1].length === lastArgs[1].length
-  );
-
-  const mapStateToProps = (state: State, scopeId: SourcererScopeName): ManageScope => {
-    const scope = getScopeIdSelector(state, scopeId);
-    const selectedPatterns = getSelectedPatterns(scope.selectedPatterns.sort().join());
-    const indexPattern = getIndexPattern(scope.indexPattern, selectedPatterns.join());
+  return (state: State, scopeId: SourcererScopeName): SourcererScopeSelector => {
+    const kibanaDataViews = getKibanaDataViewsSelector(state);
+    const defaultDataView = getDefaultDataViewSelector(state);
+    const signalIndexName = getSignalIndexNameSelector(state);
+    const scope = getScopeSelector(state, scopeId);
+    const sourcererDataView = getSourcererDataViewSelector(state, scope.selectedDataViewId);
 
     return {
-      ...scope,
-      selectedPatterns,
-      indexPattern,
+      defaultDataView,
+      kibanaDataViews,
+      signalIndexName,
+      sourcererDataView,
+      sourcererScope: scope,
     };
   };
-  return mapStateToProps;
 };

x-pack/plugins/security_solution/public/common/types.ts

@@ -6,6 +6,9 @@
  */
 
 import { ResponseErrorAttributes } from 'kibana/server';
+import { DataViewBase } from '@kbn/es-query';
+import { FieldSpec } from '../../../../../src/plugins/data_views/common';
+
 export interface ServerApiError {
   statusCode: number;
   error: string;
@@ -16,3 +19,10 @@ export interface ServerApiError {
 export interface SecuritySolutionUiConfigType {
   enableExperimental: string[];
 }
+
+/**
+ * DataViewBase with enhanced index fields used in timelines
+ */
+export interface SecuritySolutionDataViewBase extends DataViewBase {
+  fields: FieldSpec[];
+}

x-pack/plugins/security_solution/public/common/utils/empty_view/use_show_pages_with_empty_view.test.tsx

@@ -16,7 +16,7 @@ jest.mock('../route/use_route_spy', () => ({
     .mockImplementationOnce(() => [{ pageName: 'network' }]),
 }));
 jest.mock('../../../common/containers/sourcerer', () => ({
-  useSourcererScope: jest
+  useSourcererDataView: jest
     .fn()
     .mockImplementationOnce(() => [{ indicesExist: false }])
     .mockImplementationOnce(() => [{ indicesExist: false }])

x-pack/plugins/security_solution/public/common/utils/empty_view/use_show_pages_with_empty_view.tsx

@@ -8,7 +8,7 @@
 import { useState, useEffect } from 'react';
 import { useRouteSpy } from '../route/use_route_spy';
 import { SecurityPageName } from '../../../app/types';
-import { useSourcererScope } from '../../../common/containers/sourcerer';
+import { useSourcererDataView } from '../../containers/sourcerer';
 
 // Used to detect if we're on a top level page that is empty and set page background color to match the subdued Empty State
 const isPageNameWithEmptyView = (currentName: string) => {
@@ -23,7 +23,7 @@ const isPageNameWithEmptyView = (currentName: string) => {
 
 export const useShowPagesWithEmptyView = () => {
   const [{ pageName }] = useRouteSpy();
-  const { indicesExist } = useSourcererScope();
+  const { indicesExist } = useSourcererDataView();
 
   const shouldShowEmptyState = isPageNameWithEmptyView(pageName) && !indicesExist;