diff --git a/x-pack/plugins/lists/server/index.ts b/x-pack/plugins/lists/server/index.ts index c1e577aa6019..33f58ba65d3c 100644 --- a/x-pack/plugins/lists/server/index.ts +++ b/x-pack/plugins/lists/server/index.ts @@ -9,6 +9,10 @@ import { PluginInitializerContext } from '../../../../src/core/server'; import { ConfigSchema } from './config'; import { ListPlugin } from './plugin'; +// exporting these since its required at top level in siem plugin +export { ListClient } from './services/lists/list_client'; +export { ListPluginSetup } from './types'; + export const config = { schema: ConfigSchema }; export const plugin = (initializerContext: PluginInitializerContext): ListPlugin => new ListPlugin(initializerContext); diff --git a/x-pack/plugins/siem/kibana.json b/x-pack/plugins/siem/kibana.json index 1106781fd45e..6b43b41df8ee 100644 --- a/x-pack/plugins/siem/kibana.json +++ b/x-pack/plugins/siem/kibana.json @@ -24,7 +24,8 @@ "newsfeed", "security", "spaces", - "usageCollection" + "usageCollection", + "lists" ], "server": true, "ui": true diff --git a/x-pack/plugins/siem/server/lib/detection_engine/README.md b/x-pack/plugins/siem/server/lib/detection_engine/README.md index 610e82fd5f6e..695165e1990a 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/plugins/siem/server/lib/detection_engine/README.md @@ -165,3 +165,12 @@ go about doing so. `./signals/set_status_with_id.sh open` will update the status of the sample signal to open `./signals/set_status_with_query.sh closed` will update the status of the signals in the result of the query to closed. `./signals/set_status_with_query.sh open` will update the status of the signals in the result of the query to open. + +### Large List Exceptions + +To test out the functionality of large lists with rules, the user will need to import a list and post a rule with a reference to that exception list. The following outlines an example using the sample json rule provided in the repo. + +* First, set the appropriate env var in order to enable exceptions features`export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true` and `export ELASTIC_XPACK_SIEM_EXCEPTIONS_LISTS=true` and start kibana +* Second, import a list of ips from a file called `ci-badguys.txt`. The command should look like this: +`cd $HOME/kibana/x-pack/plugins/lists/server/scripts && ./import_list_items_by_filename.sh ip ~/ci-badguys.txt` +* Then, from the detection engine scripts folder (`cd kibana/x-pack/plugins/siem/server/lib/detection_engine/scripts`) run `./post_rule.sh rules/queries/lists/query_with_list_plugin.json` diff --git a/x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json b/x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json new file mode 100644 index 000000000000..fa6fe6ac7111 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json @@ -0,0 +1,24 @@ +{ + "name": "Query with a list", + "description": "Query with a list only generate signals if source.ip is not in list", + "rule_id": "query-with-list", + "risk_score": 2, + "severity": "high", + "type": "query", + "query": "host.name: *", + "interval": "30s", + "language": "kuery", + "exceptions_list": [ + { + "field": "source.ip", + "values_operator": "excluded", + "values_type": "list", + "values": [ + { + "id": "ci-badguys.txt", + "name": "ip" + } + ] + } + ] +} diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts index 251a1e6d118f..2d75ba4f42d1 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -101,7 +101,10 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig }, }); -export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ +export const sampleDocWithSortId = ( + someUuid: string = sampleIdGuid, + ip?: string +): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -110,6 +113,9 @@ export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSour _source: { someKey: 'someValue', '@timestamp': '2020-04-20T21:27:45+0000', + source: { + ip: ip ?? '127.0.0.1', + }, }, sort: ['1234567891111'], }); @@ -313,7 +319,8 @@ export const sampleDocSearchResultsNoSortIdNoHits = ( export const repeatedSearchResultsWithSortId = ( total: number, pageSize: number, - guids: string[] + guids: string[], + ips?: string[] ) => ({ took: 10, timed_out: false, @@ -327,7 +334,7 @@ export const repeatedSearchResultsWithSortId = ( total, max_score: 100, hits: Array.from({ length: pageSize }).map((x, index) => ({ - ...sampleDocWithSortId(guids[index]), + ...sampleDocWithSortId(guids[index], ips ? ips[index] : '127.0.0.1'), })), }, }); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 9ac4d4087016..5862e6c48143 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -86,5 +86,5 @@ export const bulkCreateMlSignals = async ( const anomalyResults = params.someResult; const ecsResults = transformAnomalyResultsToEcs(anomalyResults); - return singleBulkCreate({ ...params, someResult: ecsResults }); + return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts new file mode 100644 index 000000000000..86efdb660349 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts @@ -0,0 +1,242 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { filterEventsAgainstList } from './filter_events_with_list'; +import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results'; + +import { ListClient } from '../../../../../lists/server'; + +const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4()); + +describe('filterEventsAgainstList', () => { + it('should respond with eventSearchResult if exceptionList is empty', async () => { + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: undefined, + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + expect(res.hits.hits.length).toEqual(4); + }); + + it('should throw an error if malformed exception list present', async () => { + let message = ''; + try { + await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: undefined, + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + } catch (exc) { + message = exc.message; + } + expect(message).toEqual( + 'Failed to query lists index. Reason: Malformed exception list provided' + ); + }); + + it('should throw an error if unsupported exception type', async () => { + let message = ''; + try { + await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'unsupportedListPluginType', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + } catch (exc) { + message = exc.message; + } + expect(message).toEqual( + 'Failed to query lists index. Reason: Unsupported list type used, please use one of ip,keyword' + ); + }); + + describe('operator_type is includes', () => { + it('should respond with same list if no items match value list', async () => { + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)), + }); + expect(res.hits.hits.length).toEqual(4); + }); + it('should respond with less items in the list if some values match', async () => { + let outerType = ''; + let outerListId = ''; + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async ({ + value, + type, + listId, + }: { + type: string; + listId: string; + value: string[]; + }) => { + outerType = type; + outerListId = listId; + return value.slice(0, 2).map((item) => ({ + value: item, + })); + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + expect(outerType).toEqual('ip'); + expect(outerListId).toEqual('ci-badguys.txt'); + expect(res.hits.hits.length).toEqual(2); + }); + }); + describe('operator type is excluded', () => { + it('should respond with empty list if no items match value list', async () => { + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)), + }); + expect(res.hits.hits.length).toEqual(0); + }); + it('should respond with less items in the list if some values match', async () => { + let outerType = ''; + let outerListId = ''; + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async ({ + value, + type, + listId, + }: { + type: string; + listId: string; + value: string[]; + }) => { + outerType = type; + outerListId = listId; + return value.slice(0, 2).map((item) => ({ + value: item, + })); + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + expect(outerType).toEqual('ip'); + expect(outerListId).toEqual('ci-badguys.txt'); + expect(res.hits.hits.length).toEqual(2); + }); + }); +}); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts new file mode 100644 index 000000000000..400bb5dda46e --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { get } from 'lodash/fp'; +import { Logger } from 'src/core/server'; + +import { type } from '../../../../../lists/common/schemas/common'; +import { ListClient } from '../../../../../lists/server'; +import { SignalSearchResponse, SearchTypes } from './types'; +import { RuleAlertParams } from '../types'; +import { List } from '../routes/schemas/types/lists_default_array'; + +interface FilterEventsAgainstList { + listClient: ListClient; + exceptionsList: RuleAlertParams['exceptions_list']; + logger: Logger; + eventSearchResult: SignalSearchResponse; +} + +export const filterEventsAgainstList = async ({ + listClient, + exceptionsList, + logger, + eventSearchResult, +}: FilterEventsAgainstList): Promise => { + try { + if (exceptionsList == null || exceptionsList.length === 0) { + return eventSearchResult; + } + + // narrow unioned type to be single + const isStringableType = (val: SearchTypes) => + ['string', 'number', 'boolean'].includes(typeof val); + // grab the signals with values found in the given exception lists. + const filteredHitsPromises = exceptionsList + .filter((exceptionItem: List) => exceptionItem.values_type === 'list') + .map(async (exceptionItem: List) => { + if (exceptionItem.values == null || exceptionItem.values.length === 0) { + throw new Error('Malformed exception list provided'); + } + if (!type.is(exceptionItem.values[0].name)) { + throw new Error( + `Unsupported list type used, please use one of ${Object.keys(type.keys).join()}` + ); + } + if (!exceptionItem.values[0].id) { + throw new Error(`Missing list id for exception on field ${exceptionItem.field}`); + } + // acquire the list values we are checking for. + const valuesOfGivenType = eventSearchResult.hits.hits.reduce((acc, searchResultItem) => { + const valueField = get(exceptionItem.field, searchResultItem._source); + if (valueField != null && isStringableType(valueField)) { + acc.add(valueField.toString()); + } + return acc; + }, new Set()); + + // matched will contain any list items that matched with the + // values passed in from the Set. + const matchedListItems = await listClient.getListItemByValues({ + listId: exceptionItem.values[0].id, + type: exceptionItem.values[0].name, + value: [...valuesOfGivenType], + }); + + // create a set of list values that were a hit - easier to work with + const matchedListItemsSet = new Set( + matchedListItems.map((item) => item.value) + ); + + // do a single search after with these values. + // painless script to do nested query in elasticsearch + // filter out the search results that match with the values found in the list. + const operator = exceptionItem.values_operator; + const filteredEvents = eventSearchResult.hits.hits.filter((item) => { + const eventItem = get(exceptionItem.field, item._source); + if (operator === 'included') { + if (eventItem != null) { + return !matchedListItemsSet.has(eventItem); + } + } else if (operator === 'excluded') { + if (eventItem != null) { + return matchedListItemsSet.has(eventItem); + } + } + return false; + }); + const diff = eventSearchResult.hits.hits.length - filteredEvents.length; + logger.debug(`Lists filtered out ${diff} events`); + return filteredEvents; + }); + + const filteredHits = await Promise.all(filteredHitsPromises); + const toReturn: SignalSearchResponse = { + took: eventSearchResult.took, + timed_out: eventSearchResult.timed_out, + _shards: eventSearchResult._shards, + hits: { + total: filteredHits.length, + max_score: eventSearchResult.hits.max_score, + hits: filteredHits.flat(), + }, + }; + + return toReturn; + } catch (exc) { + throw new Error(`Failed to query lists index. Reason: ${exc.message}`); + } +}; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 208f0e680722..7479ab54af6e 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -10,58 +10,28 @@ import { sampleRuleGuid, mockLogger, repeatedSearchResultsWithSortId, - sampleBulkCreateDuplicateResult, - sampleDocSearchResultsNoSortId, - sampleDocSearchResultsNoSortIdNoHits, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; import uuid from 'uuid'; +import { ListClient } from '../../../../../lists/server'; +import { ListItemArraySchema } from '../../../../../lists/common/schemas'; describe('searchAfterAndBulkCreate', () => { let mockService: AlertServicesMock; let inputIndexPattern: string[] = []; + const someGuids = Array.from({ length: 13 }).map(() => uuid.v4()); beforeEach(() => { jest.clearAllMocks(); inputIndexPattern = ['auditbeat-*']; mockService = alertsMock.createAlertServices(); }); - test('if successful with empty search results', async () => { - const sampleParams = sampleRuleAlertParams(); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: sampleEmptyDocSearchResults(), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(mockService.callCluster).toHaveBeenCalledTimes(0); - expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(0); - expect(lastLookBackDate).toBeNull(); - }); - - test('if successful iteration of while loop with maxDocs', async () => { + test('should return success with number of searches less than max signals', async () => { const sampleParams = sampleRuleAlertParams(30); - const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4()); mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3))) .mockResolvedValueOnce({ took: 100, errors: false, @@ -76,7 +46,7 @@ describe('searchAfterAndBulkCreate', () => { }, ], }) - .mockResolvedValueOnce(repeatedSearchResultsWithSortId(3, 1, someGuids.slice(0, 3))) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6))) .mockResolvedValueOnce({ took: 100, errors: false, @@ -91,7 +61,22 @@ describe('searchAfterAndBulkCreate', () => { }, ], }) - .mockResolvedValueOnce(repeatedSearchResultsWithSortId(3, 1, someGuids.slice(3, 6))) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(9, 12))) .mockResolvedValueOnce({ took: 100, errors: false, @@ -107,8 +92,23 @@ describe('searchAfterAndBulkCreate', () => { ], }); const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(3, 1, someGuids.slice(6, 9)), ruleParams: sampleParams, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], services: mockService, logger: mockLogger, id: sampleRuleGuid, @@ -128,18 +128,184 @@ describe('searchAfterAndBulkCreate', () => { tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); - expect(mockService.callCluster).toHaveBeenCalledTimes(5); expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(3); + expect(mockService.callCluster).toHaveBeenCalledTimes(8); + expect(createdSignalsCount).toEqual(4); + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); + + test('should return success when no search results are in the allowlist', async () => { + const sampleParams = sampleRuleAlertParams(30); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + ], + }); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + ruleParams: sampleParams, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + services: mockService, + logger: mockLogger, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + pageSize: 1, + filter: undefined, + refresh: false, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + expect(success).toEqual(true); + expect(mockService.callCluster).toHaveBeenCalledTimes(2); + expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); + + test('should return success when no exceptions list provided', async () => { + const sampleParams = sampleRuleAlertParams(30); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + ], + }); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + ruleParams: sampleParams, + listClient: ({ + getListItemByValues: async ({ + value, + }: { + type: string; + listId: string; + value: string[]; + }) => { + return value.map((item) => ({ value: item })); + }, + } as unknown) as ListClient, + exceptionsList: undefined, + services: mockService, + logger: mockLogger, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + pageSize: 1, + filter: undefined, + refresh: false, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + expect(success).toEqual(true); + expect(mockService.callCluster).toHaveBeenCalledTimes(2); + expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); test('if unsuccessful first bulk create', async () => { - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); const sampleParams = sampleRuleAlertParams(10); - mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3))) + .mockRejectedValue(new Error('bulk failed')); // Added this recently const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), + listClient: ({ + getListItemByValues: async () => { + return ([] as unknown) as ListItemArraySchema; + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -162,72 +328,38 @@ describe('searchAfterAndBulkCreate', () => { }); expect(mockLogger.error).toHaveBeenCalled(); expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(1); + expect(createdSignalsCount).toEqual(0); expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); - test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => { + test('should return success with 0 total hits', async () => { const sampleParams = sampleRuleAlertParams(); - mockService.callCluster.mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', + mockService.callCluster.mockResolvedValueOnce(sampleEmptyDocSearchResults()); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + listClient: ({ + getListItemByValues: async ({ + value, + }: { + type: string; + listId: string; + value: string[]; + }) => { + return value.map((item) => ({ value: item })); }, + } as unknown) as ListClient, + exceptionsList: [ { - create: { - status: 201, - }, + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], }, ], - }); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(mockLogger.error).toHaveBeenCalled(); - expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - - test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => { - const sampleParams = sampleRuleAlertParams(); - mockService.callCluster.mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - { - create: { - status: 201, - }, - }, - ], - }); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortIdNoHits(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -249,105 +381,12 @@ describe('searchAfterAndBulkCreate', () => { throttle: 'no_actions', }); expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - - test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { - const sampleParams = sampleRuleAlertParams(10); - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); - mockService.callCluster - .mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - { - create: { - status: 201, - }, - }, - ], - }) - .mockResolvedValueOnce(sampleDocSearchResultsNoSortId()); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - - test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => { - const sampleParams = sampleRuleAlertParams(10); - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); - mockService.callCluster - .mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - { - create: { - status: 201, - }, - }, - ], - }) - .mockResolvedValueOnce(sampleEmptyDocSearchResults()); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + expect(createdSignalsCount).toEqual(0); + expect(lastLookBackDate).toEqual(null); }); test('if returns false when singleSearchAfter throws an exception', async () => { const sampleParams = sampleRuleAlertParams(10); - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); mockService.callCluster .mockResolvedValueOnce({ took: 100, @@ -367,7 +406,30 @@ describe('searchAfterAndBulkCreate', () => { throw Error('Fake Error'); }); const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), + listClient: ({ + getListItemByValues: async ({ + value, + }: { + type: string; + listId: string; + value: string[]; + }) => { + return value.map((item) => ({ value: item })); + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -389,7 +451,7 @@ describe('searchAfterAndBulkCreate', () => { throttle: 'no_actions', }); expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + expect(createdSignalsCount).toEqual(0); // should not create signals if search threw error + expect(lastLookBackDate).toEqual(null); }); }); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts index acf3e9bfb055..05cdccedbc2c 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -4,18 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ListClient } from '../../../../../lists/server'; import { AlertServices } from '../../../../../alerting/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams, RefreshTypes } from '../types'; +import { RuleTypeParams, RefreshTypes, RuleAlertParams } from '../types'; import { Logger } from '../../../../../../../src/core/server'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; import { SignalSearchResponse } from './types'; +import { filterEventsAgainstList } from './filter_events_with_list'; interface SearchAfterAndBulkCreateParams { - someResult: SignalSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; + listClient: ListClient | undefined; // TODO: undefined is for temporary development, remove before merged + exceptionsList: RuleAlertParams['exceptions_list']; logger: Logger; id: string; inputIndexPattern: string[]; @@ -45,9 +48,10 @@ export interface SearchAfterAndBulkCreateReturnType { // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ - someResult, ruleParams, + exceptionsList, services, + listClient, logger, id, inputIndexPattern, @@ -73,71 +77,31 @@ export const searchAfterAndBulkCreate = async ({ lastLookBackDate: null, createdSignalsCount: 0, }; - if (someResult.hits.hits.length === 0) { - toReturn.success = true; - return toReturn; - } - logger.debug('[+] starting bulk insertion'); - const { bulkCreateDuration, createdItemsCount } = await singleBulkCreate({ - someResult, - ruleParams, - services, - logger, - id, - signalsIndex, - actions, - name, - createdAt, - createdBy, - updatedAt, - updatedBy, - interval, - enabled, - refresh, - tags, - throttle, - }); + let sortId; // tells us where to start our next search_after query + let searchResultSize = 0; - if (createdItemsCount > 0) { - toReturn.createdSignalsCount = createdItemsCount; - toReturn.lastLookBackDate = - someResult.hits.hits.length > 0 - ? new Date(someResult.hits.hits[someResult.hits.hits.length - 1]?._source['@timestamp']) - : null; - } + /* + The purpose of `maxResults` is to ensure we do not perform + extra search_after's. This will be reset on each + iteration, although it really only matters for the first + iteration of the loop. + e.g. if maxSignals = 100 but our search result only yields + 27 documents, there is no point in performing another search + since we know there are no more events that match our rule, + and thus, no more signals we could possibly generate. + However, if maxSignals = 500 and our search yields a total + of 3050 results we don't want to make 3050 signals, + we only want 500. So maxResults will help us control how + many times we perform a search_after + */ + let maxResults = ruleParams.maxSignals; - if (bulkCreateDuration) { - toReturn.bulkCreateTimes.push(bulkCreateDuration); - } - const totalHits = - typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value; - // maxTotalHitsSize represents the total number of docs to - // query for, no matter the size of each individual page of search results. - // If the total number of hits for the overall search result is greater than - // maxSignals, default to requesting a total of maxSignals, otherwise use the - // totalHits in the response from the searchAfter query. - const maxTotalHitsSize = Math.min(totalHits, ruleParams.maxSignals); + // Get - // number of docs in the current search result - let hitsSize = someResult.hits.hits.length; - logger.debug(`first size: ${hitsSize}`); - let sortIds = someResult.hits.hits[0].sort; - if (sortIds == null && totalHits > 0) { - logger.error('sortIds was empty on first search but expected more'); - toReturn.success = false; - return toReturn; - } else if (sortIds == null && totalHits === 0) { - toReturn.success = true; - return toReturn; - } - let sortId; - if (sortIds != null) { - sortId = sortIds[0]; - } - while (hitsSize < maxTotalHitsSize && hitsSize !== 0) { + while (searchResultSize < maxResults) { try { - logger.debug(`sortIds: ${sortIds}`); + logger.debug(`sortIds: ${sortId}`); const { searchResult, searchDuration, @@ -152,25 +116,60 @@ export const searchAfterAndBulkCreate = async ({ pageSize, // maximum number of docs to receive per search result. }); toReturn.searchAfterTimes.push(searchDuration); + toReturn.lastLookBackDate = + searchResult.hits.hits.length > 0 + ? new Date( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp'] + ) + : null; + const totalHits = + typeof searchResult.hits.total === 'number' + ? searchResult.hits.total + : searchResult.hits.total.value; + logger.debug(`totalHits: ${totalHits}`); + + // re-calculate maxResults to ensure if our search results + // are less than max signals, we are not attempting to + // create more signals than there are total search results. + maxResults = Math.min(totalHits, ruleParams.maxSignals); + searchResultSize += searchResult.hits.hits.length; if (searchResult.hits.hits.length === 0) { toReturn.success = true; return toReturn; } - hitsSize += searchResult.hits.hits.length; - logger.debug(`size adjusted: ${hitsSize}`); - sortIds = searchResult.hits.hits[0].sort; - if (sortIds == null) { - logger.debug('sortIds was empty on search'); + + // filter out the search results that match with the values found in the list. + // the resulting set are valid signals that are not on the allowlist. + const filteredEvents = + listClient != null + ? await filterEventsAgainstList({ + listClient, + exceptionsList, + logger, + eventSearchResult: searchResult, + }) + : searchResult; + + if (filteredEvents.hits.hits.length === 0) { + // everything in the events were allowed, so no need to generate signals toReturn.success = true; - return toReturn; // no more search results + return toReturn; + } + + // cap max signals created to be no more than maxSignals + if (toReturn.createdSignalsCount + filteredEvents.hits.hits.length > ruleParams.maxSignals) { + const tempSignalsToIndex = filteredEvents.hits.hits.slice( + 0, + ruleParams.maxSignals - toReturn.createdSignalsCount + ); + filteredEvents.hits.hits = tempSignalsToIndex; } - sortId = sortIds[0]; logger.debug('next bulk index'); const { bulkCreateDuration: bulkDuration, createdItemsCount: createdCount, } = await singleBulkCreate({ - someResult: searchResult, + filteredEvents, ruleParams, services, logger, @@ -189,17 +188,25 @@ export const searchAfterAndBulkCreate = async ({ throttle, }); logger.debug('finished next bulk index'); + logger.debug(`created ${createdCount} signals`); toReturn.createdSignalsCount += createdCount; if (bulkDuration) { toReturn.bulkCreateTimes.push(bulkDuration); } + + if (filteredEvents.hits.hits[0].sort == null) { + logger.debug('sortIds was empty on search'); + toReturn.success = true; + return toReturn; // no more search results + } + sortId = filteredEvents.hits.hits[0].sort[0]; } catch (exc) { logger.error(`[-] search_after and bulk threw an error ${exc}`); toReturn.success = false; return toReturn; } } - logger.debug(`[+] completed bulk index of ${maxTotalHitsSize}`); + logger.debug(`[+] completed bulk index of ${toReturn.createdSignalsCount}`); toReturn.success = true; return toReturn; }; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 0c7f0839f8da..ea7255b8a925 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -17,6 +17,7 @@ import { scheduleNotificationActions } from '../notifications/schedule_notificat import { RuleAlertType } from '../rules/types'; import { findMlSignals } from './find_ml_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; +import { ListPluginSetup } from '../../../../../lists/server/types'; jest.mock('./rule_status_saved_objects_client'); jest.mock('./rule_status_service'); @@ -68,6 +69,11 @@ describe('rules_notification_alert_type', () => { modulesProvider: jest.fn(), resultsServiceProvider: jest.fn(), }; + const listMock = { + getListClient: () => ({ + getListItemByValues: () => [], + }), + }; let payload: jest.Mocked; let alert: ReturnType; let logger: ReturnType; @@ -110,6 +116,7 @@ describe('rules_notification_alert_type', () => { logger, version, ml: mlMock, + lists: (listMock as unknown) as ListPluginSetup, }); }); @@ -199,6 +206,7 @@ describe('rules_notification_alert_type', () => { logger, version, ml: undefined, + lists: undefined, }); await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); @@ -358,7 +366,7 @@ describe('rules_notification_alert_type', () => { }); it('when error was thrown', async () => { - (searchAfterAndBulkCreate as jest.Mock).mockResolvedValue({}); + (searchAfterAndBulkCreate as jest.Mock).mockRejectedValue({}); await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain('An error occurred during rule execution'); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8cef4c8ea0e6..6885b4c81467 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -4,14 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { performance } from 'perf_hooks'; +/* eslint-disable complexity */ + import { Logger, KibanaRequest } from 'src/core/server'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE } from '../../../../common/constants'; import { isJobStarted, isMlRule } from '../../../../common/machine_learning/helpers'; import { SetupPlugins } from '../../../plugin'; -import { buildEventsSearchQuery } from './build_events_query'; +import { ListClient } from '../../../../../lists/server'; + import { getInputIndex } from './get_input_output_index'; import { searchAfterAndBulkCreate, @@ -19,7 +21,7 @@ import { } from './search_after_bulk_create'; import { getFilter } from './get_filter'; import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; -import { getGapBetweenRuns, makeFloatString, parseScheduleDates } from './utils'; +import { getGapBetweenRuns, parseScheduleDates } from './utils'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; @@ -32,15 +34,18 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { hasListsFeature } from '../feature_flags'; export const signalRulesAlertType = ({ logger, version, ml, + lists, }: { logger: Logger; version: string; ml: SetupPlugins['ml']; + lists: SetupPlugins['lists'] | undefined; }): SignalRuleAlertTypeDefinition => { return { id: SIGNALS_ID, @@ -51,7 +56,14 @@ export const signalRulesAlertType = ({ params: signalParamsSchema(), }, producer: 'siem', - async executor({ previousStartedAt, alertId, services, params }) { + async executor({ + previousStartedAt, + alertId, + services, + params, + spaceId, + updatedBy: updatedByUser, + }) { const { anomalyThreshold, from, @@ -67,7 +79,7 @@ export const signalRulesAlertType = ({ query, to, type, - exceptions_list, + exceptions_list: exceptionsList, } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; @@ -123,7 +135,6 @@ export const signalRulesAlertType = ({ hasError = true; await ruleStatusService.error(gapMessage, { gap: gapString }); } - try { if (isMlRule(type)) { if (ml == null) { @@ -199,6 +210,18 @@ export const signalRulesAlertType = ({ result.bulkCreateTimes.push(bulkCreateDuration); } } else { + let listClient: ListClient | undefined; + if (hasListsFeature()) { + if (lists == null) { + throw new Error('lists plugin unavailable during rule execution'); + } + listClient = await lists.getListClient( + services.callCluster, + spaceId, + updatedByUser ?? 'elastic' + ); + } + const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, @@ -208,34 +231,13 @@ export const signalRulesAlertType = ({ savedId, services, index: inputIndex, - lists: exceptions_list, + // temporary filter out list type + lists: exceptionsList?.filter((item) => item.values_type !== 'list'), }); - const noReIndex = buildEventsSearchQuery({ - index: inputIndex, - from, - to, - filter: esFilter, - size: searchAfterSize, - searchAfterSortId: undefined, - }); - - logger.debug(buildRuleMessage('[+] Initial search call')); - const start = performance.now(); - const noReIndexResult = await services.callCluster('search', noReIndex); - const end = performance.now(); - - const signalCount = noReIndexResult.hits.total.value; - if (signalCount !== 0) { - logger.info( - buildRuleMessage( - `Found ${signalCount} signals from the indexes of "[${inputIndex.join(', ')}]"` - ) - ); - } - result = await searchAfterAndBulkCreate({ - someResult: noReIndexResult, + listClient, + exceptionsList, ruleParams: params, services, logger, @@ -256,7 +258,6 @@ export const signalRulesAlertType = ({ tags, throttle, }); - result.searchAfterTimes.push(makeFloatString(end - start)); } if (result.success) { @@ -293,6 +294,11 @@ export const signalRulesAlertType = ({ } logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); + logger.debug( + buildRuleMessage( + `[+] Finished indexing ${result.createdSignalsCount} signals into ${outputIndex}` + ) + ); if (!hasError) { await ruleStatusService.success('succeeded', { bulkCreateTimeDurations: result.bulkCreateTimes, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts index 6f3cc6e708fc..265f98653313 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -141,7 +141,7 @@ describe('singleBulkCreate', () => { ], }); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(), + filteredEvents: sampleDocSearchResultsNoSortId(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -175,7 +175,7 @@ describe('singleBulkCreate', () => { ], }); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleDocSearchResultsNoSortIdNoVersion(), + filteredEvents: sampleDocSearchResultsNoSortIdNoVersion(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -201,7 +201,7 @@ describe('singleBulkCreate', () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(false); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleEmptyDocSearchResults(), + filteredEvents: sampleEmptyDocSearchResults(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -228,7 +228,7 @@ describe('singleBulkCreate', () => { const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleSearchResult(), + filteredEvents: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -257,7 +257,7 @@ describe('singleBulkCreate', () => { const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockResolvedValue(sampleBulkCreateErrorResult); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleSearchResult(), + filteredEvents: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -352,7 +352,7 @@ describe('singleBulkCreate', () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(), + filteredEvents: sampleDocSearchResultsNoSortId(), ruleParams: sampleParams, services: mockService, logger: mockLogger, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts index c162c8855b09..39aecde470e0 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts @@ -15,7 +15,7 @@ import { buildBulkBody } from './build_bulk_body'; import { Logger } from '../../../../../../../src/core/server'; interface SingleBulkCreateParams { - someResult: SignalSearchResponse; + filteredEvents: SignalSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; logger: Logger; @@ -64,7 +64,7 @@ export interface SingleBulkCreateResponse { // Bulk Index documents. export const singleBulkCreate = async ({ - someResult, + filteredEvents, ruleParams, services, logger, @@ -82,8 +82,8 @@ export const singleBulkCreate = async ({ tags, throttle, }: SingleBulkCreateParams): Promise => { - someResult.hits.hits = filterDuplicateRules(id, someResult); - if (someResult.hits.hits.length === 0) { + filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents); + if (filteredEvents.hits.hits.length === 0) { return { success: true, createdItemsCount: 0 }; } // index documents after creating an ID based on the @@ -95,7 +95,7 @@ export const singleBulkCreate = async ({ // while preventing duplicates from being added to the // signals index if rules are re-run over the same time // span. Also allow for versioning. - const bulkBody = someResult.hits.hits.flatMap((doc) => [ + const bulkBody = filteredEvents.hits.hits.flatMap((doc) => [ { create: { _index: signalsIndex, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts index 580080966457..2aa42234460d 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts @@ -22,18 +22,17 @@ describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId); - await expect( - singleSearchAfter({ - searchAfterSortId, - index: [], - from: 'now-360s', - to: 'now', - services: mockService, - logger: mockLogger, - pageSize: 1, - filter: undefined, - }) - ).rejects.toThrow('Attempted to search after with empty sort id'); + const { searchResult } = await singleSearchAfter({ + searchAfterSortId, + index: [], + from: 'now-360s', + to: 'now', + services: mockService, + logger: mockLogger, + pageSize: 1, + filter: undefined, + }); + expect(searchResult).toEqual(sampleDocSearchResultsNoSortId); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts index 8071c18713c1..a7086a4fb229 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts @@ -36,9 +36,6 @@ export const singleSearchAfter = async ({ searchResult: SignalSearchResponse; searchDuration: string; }> => { - if (searchAfterSortId == null) { - throw Error('Attempted to search after with empty sort id'); - } try { const searchAfterQuery = buildEventsSearchQuery({ index, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts index b493bab8b461..32b13c5251a6 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts @@ -100,6 +100,7 @@ export interface GetResponse { _source: SearchTypes; } +export type EventSearchResponse = SearchResponse; export type SignalSearchResponse = SearchResponse; export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; diff --git a/x-pack/plugins/siem/server/plugin.ts b/x-pack/plugins/siem/server/plugin.ts index 3c336991f3d9..5a47efd45888 100644 --- a/x-pack/plugins/siem/server/plugin.ts +++ b/x-pack/plugins/siem/server/plugin.ts @@ -19,6 +19,7 @@ import { PluginSetupContract as AlertingSetup } from '../../alerting/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; +import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; @@ -52,6 +53,7 @@ export interface SetupPlugins { security?: SecuritySetup; spaces?: SpacesSetup; ml?: MlSetup; + lists?: ListPluginSetup; } export interface StartPlugins { @@ -194,6 +196,7 @@ export class Plugin implements IPlugin `--xpack.${key}.enabled=false`), `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'alerts')}`, `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'actions')}`,