From 177cda42bd1e3b53b8fb7a1e08600e75dc43fdb4 Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Thu, 28 May 2020 15:45:46 -0400 Subject: [PATCH] [SIEM] [Detection Engine] Incorporate large lists to rule execution. (#65372) * introduce lists plugin for use by executor * adds getListClient function on setup * refactors searchAfterBulkCreate to integrate with the lists plugin so we only generate signals from events not in the list * fixes type check issues * fixes unit tests, adds field and other parameters for using lists in executor. * cleaning up types and exports, updates to match new contracts with lists client from master * prior to this commit the refactored while loop was doing more search after loops than it needed to and this fixes two bugs in the list filter function where we were returning the wrong count, and we were not accessing the right field on the event * exception lists are optional * use exceptions list format, this works with given sample query in scripts * updates tests and fixes type issues * updates README doc in detection engine with example for rule with list exception * adds one test and removes commented out code * fix sample rule json from 30s to 5m * fix sample rule json from 30s to 5m * remove unused import * more cleanup * e2e test for prepackaged rules was failing because lists was undefined in the siem plugin and was preventing the registration of the rule alert type. I removed this but once lists is ready for prime time we should consider adding the null check back * can't reuse the same env var since the tests are setting the ELASTIC_XPACK_SIEM_LISTS_FEATURE env var to true without enabling the lists plugin * fixes from pr review, still needs more TLC * exports listspluginsetup type from top-level in lists plugin, fixes logic for empty exceptions list, updates types * utilize type.is to remove as casting, also do null checks and throw an error when exceptionItem is malformed. This will change in the very near future once the new json format for exception lists is incorporated * fix type issues after merging master into branch * update mock * remove bad null check for ml plugin before registering rule alert type in siem plugin * prettier linting * adds test for filter events with list * pr comments * adds logic for included vs excluded and updates tests * update test cases for search after bulk create to default to included for exception lists * filter out non-list exception items from the loop --- x-pack/plugins/lists/server/index.ts | 4 + x-pack/plugins/siem/kibana.json | 3 +- .../server/lib/detection_engine/README.md | 9 + .../queries/lists/query_with_list_plugin.json | 24 + .../signals/__mocks__/es_results.ts | 13 +- .../signals/bulk_create_ml_signals.ts | 2 +- .../signals/filter_events_with_list.test.ts | 242 +++++++++ .../signals/filter_events_with_list.ts | 111 +++++ .../signals/search_after_bulk_create.test.ts | 460 ++++++++++-------- .../signals/search_after_bulk_create.ts | 151 +++--- .../signals/signal_rule_alert_type.test.ts | 10 +- .../signals/signal_rule_alert_type.ts | 70 +-- .../signals/single_bulk_create.test.ts | 12 +- .../signals/single_bulk_create.ts | 10 +- .../signals/single_search_after.test.ts | 23 +- .../signals/single_search_after.ts | 3 - .../lib/detection_engine/signals/types.ts | 1 + x-pack/plugins/siem/server/plugin.ts | 3 + .../common/config.ts | 1 + 19 files changed, 817 insertions(+), 335 deletions(-) create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts create mode 100644 x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts diff --git a/x-pack/plugins/lists/server/index.ts b/x-pack/plugins/lists/server/index.ts index c1e577aa6019..33f58ba65d3c 100644 --- a/x-pack/plugins/lists/server/index.ts +++ b/x-pack/plugins/lists/server/index.ts @@ -9,6 +9,10 @@ import { PluginInitializerContext } from '../../../../src/core/server'; import { ConfigSchema } from './config'; import { ListPlugin } from './plugin'; +// exporting these since its required at top level in siem plugin +export { ListClient } from './services/lists/list_client'; +export { ListPluginSetup } from './types'; + export const config = { schema: ConfigSchema }; export const plugin = (initializerContext: PluginInitializerContext): ListPlugin => new ListPlugin(initializerContext); diff --git a/x-pack/plugins/siem/kibana.json b/x-pack/plugins/siem/kibana.json index 1106781fd45e..6b43b41df8ee 100644 --- a/x-pack/plugins/siem/kibana.json +++ b/x-pack/plugins/siem/kibana.json @@ -24,7 +24,8 @@ "newsfeed", "security", "spaces", - "usageCollection" + "usageCollection", + "lists" ], "server": true, "ui": true diff --git a/x-pack/plugins/siem/server/lib/detection_engine/README.md b/x-pack/plugins/siem/server/lib/detection_engine/README.md index 610e82fd5f6e..695165e1990a 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/plugins/siem/server/lib/detection_engine/README.md @@ -165,3 +165,12 @@ go about doing so. `./signals/set_status_with_id.sh open` will update the status of the sample signal to open `./signals/set_status_with_query.sh closed` will update the status of the signals in the result of the query to closed. `./signals/set_status_with_query.sh open` will update the status of the signals in the result of the query to open. + +### Large List Exceptions + +To test out the functionality of large lists with rules, the user will need to import a list and post a rule with a reference to that exception list. The following outlines an example using the sample json rule provided in the repo. + +* First, set the appropriate env var in order to enable exceptions features`export ELASTIC_XPACK_SIEM_LISTS_FEATURE=true` and `export ELASTIC_XPACK_SIEM_EXCEPTIONS_LISTS=true` and start kibana +* Second, import a list of ips from a file called `ci-badguys.txt`. The command should look like this: +`cd $HOME/kibana/x-pack/plugins/lists/server/scripts && ./import_list_items_by_filename.sh ip ~/ci-badguys.txt` +* Then, from the detection engine scripts folder (`cd kibana/x-pack/plugins/siem/server/lib/detection_engine/scripts`) run `./post_rule.sh rules/queries/lists/query_with_list_plugin.json` diff --git a/x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json b/x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json new file mode 100644 index 000000000000..fa6fe6ac7111 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/scripts/rules/queries/lists/query_with_list_plugin.json @@ -0,0 +1,24 @@ +{ + "name": "Query with a list", + "description": "Query with a list only generate signals if source.ip is not in list", + "rule_id": "query-with-list", + "risk_score": 2, + "severity": "high", + "type": "query", + "query": "host.name: *", + "interval": "30s", + "language": "kuery", + "exceptions_list": [ + { + "field": "source.ip", + "values_operator": "excluded", + "values_type": "list", + "values": [ + { + "id": "ci-badguys.txt", + "name": "ip" + } + ] + } + ] +} diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts index 251a1e6d118f..2d75ba4f42d1 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -101,7 +101,10 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig }, }); -export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ +export const sampleDocWithSortId = ( + someUuid: string = sampleIdGuid, + ip?: string +): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', _score: 100, @@ -110,6 +113,9 @@ export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSour _source: { someKey: 'someValue', '@timestamp': '2020-04-20T21:27:45+0000', + source: { + ip: ip ?? '127.0.0.1', + }, }, sort: ['1234567891111'], }); @@ -313,7 +319,8 @@ export const sampleDocSearchResultsNoSortIdNoHits = ( export const repeatedSearchResultsWithSortId = ( total: number, pageSize: number, - guids: string[] + guids: string[], + ips?: string[] ) => ({ took: 10, timed_out: false, @@ -327,7 +334,7 @@ export const repeatedSearchResultsWithSortId = ( total, max_score: 100, hits: Array.from({ length: pageSize }).map((x, index) => ({ - ...sampleDocWithSortId(guids[index]), + ...sampleDocWithSortId(guids[index], ips ? ips[index] : '127.0.0.1'), })), }, }); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 9ac4d4087016..5862e6c48143 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -86,5 +86,5 @@ export const bulkCreateMlSignals = async ( const anomalyResults = params.someResult; const ecsResults = transformAnomalyResultsToEcs(anomalyResults); - return singleBulkCreate({ ...params, someResult: ecsResults }); + return singleBulkCreate({ ...params, filteredEvents: ecsResults }); }; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts new file mode 100644 index 000000000000..86efdb660349 --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.test.ts @@ -0,0 +1,242 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { filterEventsAgainstList } from './filter_events_with_list'; +import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results'; + +import { ListClient } from '../../../../../lists/server'; + +const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4()); + +describe('filterEventsAgainstList', () => { + it('should respond with eventSearchResult if exceptionList is empty', async () => { + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: undefined, + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + expect(res.hits.hits.length).toEqual(4); + }); + + it('should throw an error if malformed exception list present', async () => { + let message = ''; + try { + await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: undefined, + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + } catch (exc) { + message = exc.message; + } + expect(message).toEqual( + 'Failed to query lists index. Reason: Malformed exception list provided' + ); + }); + + it('should throw an error if unsupported exception type', async () => { + let message = ''; + try { + await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'unsupportedListPluginType', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + } catch (exc) { + message = exc.message; + } + expect(message).toEqual( + 'Failed to query lists index. Reason: Unsupported list type used, please use one of ip,keyword' + ); + }); + + describe('operator_type is includes', () => { + it('should respond with same list if no items match value list', async () => { + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)), + }); + expect(res.hits.hits.length).toEqual(4); + }); + it('should respond with less items in the list if some values match', async () => { + let outerType = ''; + let outerListId = ''; + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async ({ + value, + type, + listId, + }: { + type: string; + listId: string; + value: string[]; + }) => { + outerType = type; + outerListId = listId; + return value.slice(0, 2).map((item) => ({ + value: item, + })); + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + expect(outerType).toEqual('ip'); + expect(outerListId).toEqual('ci-badguys.txt'); + expect(res.hits.hits.length).toEqual(2); + }); + }); + describe('operator type is excluded', () => { + it('should respond with empty list if no items match value list', async () => { + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)), + }); + expect(res.hits.hits.length).toEqual(0); + }); + it('should respond with less items in the list if some values match', async () => { + let outerType = ''; + let outerListId = ''; + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient: ({ + getListItemByValues: async ({ + value, + type, + listId, + }: { + type: string; + listId: string; + value: string[]; + }) => { + outerType = type; + outerListId = listId; + return value.slice(0, 2).map((item) => ({ + value: item, + })); + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'excluded', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '7.7.7.7', + ]), + }); + expect(outerType).toEqual('ip'); + expect(outerListId).toEqual('ci-badguys.txt'); + expect(res.hits.hits.length).toEqual(2); + }); + }); +}); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts new file mode 100644 index 000000000000..400bb5dda46e --- /dev/null +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { get } from 'lodash/fp'; +import { Logger } from 'src/core/server'; + +import { type } from '../../../../../lists/common/schemas/common'; +import { ListClient } from '../../../../../lists/server'; +import { SignalSearchResponse, SearchTypes } from './types'; +import { RuleAlertParams } from '../types'; +import { List } from '../routes/schemas/types/lists_default_array'; + +interface FilterEventsAgainstList { + listClient: ListClient; + exceptionsList: RuleAlertParams['exceptions_list']; + logger: Logger; + eventSearchResult: SignalSearchResponse; +} + +export const filterEventsAgainstList = async ({ + listClient, + exceptionsList, + logger, + eventSearchResult, +}: FilterEventsAgainstList): Promise => { + try { + if (exceptionsList == null || exceptionsList.length === 0) { + return eventSearchResult; + } + + // narrow unioned type to be single + const isStringableType = (val: SearchTypes) => + ['string', 'number', 'boolean'].includes(typeof val); + // grab the signals with values found in the given exception lists. + const filteredHitsPromises = exceptionsList + .filter((exceptionItem: List) => exceptionItem.values_type === 'list') + .map(async (exceptionItem: List) => { + if (exceptionItem.values == null || exceptionItem.values.length === 0) { + throw new Error('Malformed exception list provided'); + } + if (!type.is(exceptionItem.values[0].name)) { + throw new Error( + `Unsupported list type used, please use one of ${Object.keys(type.keys).join()}` + ); + } + if (!exceptionItem.values[0].id) { + throw new Error(`Missing list id for exception on field ${exceptionItem.field}`); + } + // acquire the list values we are checking for. + const valuesOfGivenType = eventSearchResult.hits.hits.reduce((acc, searchResultItem) => { + const valueField = get(exceptionItem.field, searchResultItem._source); + if (valueField != null && isStringableType(valueField)) { + acc.add(valueField.toString()); + } + return acc; + }, new Set()); + + // matched will contain any list items that matched with the + // values passed in from the Set. + const matchedListItems = await listClient.getListItemByValues({ + listId: exceptionItem.values[0].id, + type: exceptionItem.values[0].name, + value: [...valuesOfGivenType], + }); + + // create a set of list values that were a hit - easier to work with + const matchedListItemsSet = new Set( + matchedListItems.map((item) => item.value) + ); + + // do a single search after with these values. + // painless script to do nested query in elasticsearch + // filter out the search results that match with the values found in the list. + const operator = exceptionItem.values_operator; + const filteredEvents = eventSearchResult.hits.hits.filter((item) => { + const eventItem = get(exceptionItem.field, item._source); + if (operator === 'included') { + if (eventItem != null) { + return !matchedListItemsSet.has(eventItem); + } + } else if (operator === 'excluded') { + if (eventItem != null) { + return matchedListItemsSet.has(eventItem); + } + } + return false; + }); + const diff = eventSearchResult.hits.hits.length - filteredEvents.length; + logger.debug(`Lists filtered out ${diff} events`); + return filteredEvents; + }); + + const filteredHits = await Promise.all(filteredHitsPromises); + const toReturn: SignalSearchResponse = { + took: eventSearchResult.took, + timed_out: eventSearchResult.timed_out, + _shards: eventSearchResult._shards, + hits: { + total: filteredHits.length, + max_score: eventSearchResult.hits.max_score, + hits: filteredHits.flat(), + }, + }; + + return toReturn; + } catch (exc) { + throw new Error(`Failed to query lists index. Reason: ${exc.message}`); + } +}; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 208f0e680722..7479ab54af6e 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -10,58 +10,28 @@ import { sampleRuleGuid, mockLogger, repeatedSearchResultsWithSortId, - sampleBulkCreateDuplicateResult, - sampleDocSearchResultsNoSortId, - sampleDocSearchResultsNoSortIdNoHits, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; import uuid from 'uuid'; +import { ListClient } from '../../../../../lists/server'; +import { ListItemArraySchema } from '../../../../../lists/common/schemas'; describe('searchAfterAndBulkCreate', () => { let mockService: AlertServicesMock; let inputIndexPattern: string[] = []; + const someGuids = Array.from({ length: 13 }).map(() => uuid.v4()); beforeEach(() => { jest.clearAllMocks(); inputIndexPattern = ['auditbeat-*']; mockService = alertsMock.createAlertServices(); }); - test('if successful with empty search results', async () => { - const sampleParams = sampleRuleAlertParams(); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: sampleEmptyDocSearchResults(), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(mockService.callCluster).toHaveBeenCalledTimes(0); - expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(0); - expect(lastLookBackDate).toBeNull(); - }); - - test('if successful iteration of while loop with maxDocs', async () => { + test('should return success with number of searches less than max signals', async () => { const sampleParams = sampleRuleAlertParams(30); - const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4()); mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3))) .mockResolvedValueOnce({ took: 100, errors: false, @@ -76,7 +46,7 @@ describe('searchAfterAndBulkCreate', () => { }, ], }) - .mockResolvedValueOnce(repeatedSearchResultsWithSortId(3, 1, someGuids.slice(0, 3))) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6))) .mockResolvedValueOnce({ took: 100, errors: false, @@ -91,7 +61,22 @@ describe('searchAfterAndBulkCreate', () => { }, ], }) - .mockResolvedValueOnce(repeatedSearchResultsWithSortId(3, 1, someGuids.slice(3, 6))) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(9, 12))) .mockResolvedValueOnce({ took: 100, errors: false, @@ -107,8 +92,23 @@ describe('searchAfterAndBulkCreate', () => { ], }); const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(3, 1, someGuids.slice(6, 9)), ruleParams: sampleParams, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], services: mockService, logger: mockLogger, id: sampleRuleGuid, @@ -128,18 +128,184 @@ describe('searchAfterAndBulkCreate', () => { tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); - expect(mockService.callCluster).toHaveBeenCalledTimes(5); expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(3); + expect(mockService.callCluster).toHaveBeenCalledTimes(8); + expect(createdSignalsCount).toEqual(4); + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); + + test('should return success when no search results are in the allowlist', async () => { + const sampleParams = sampleRuleAlertParams(30); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + ], + }); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + ruleParams: sampleParams, + listClient: ({ + getListItemByValues: async () => [], + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], + services: mockService, + logger: mockLogger, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + pageSize: 1, + filter: undefined, + refresh: false, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + expect(success).toEqual(true); + expect(mockService.callCluster).toHaveBeenCalledTimes(2); + expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); + + test('should return success when no exceptions list provided', async () => { + const sampleParams = sampleRuleAlertParams(30); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + { + create: { + status: 201, + }, + }, + ], + }); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + ruleParams: sampleParams, + listClient: ({ + getListItemByValues: async ({ + value, + }: { + type: string; + listId: string; + value: string[]; + }) => { + return value.map((item) => ({ value: item })); + }, + } as unknown) as ListClient, + exceptionsList: undefined, + services: mockService, + logger: mockLogger, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + pageSize: 1, + filter: undefined, + refresh: false, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + expect(success).toEqual(true); + expect(mockService.callCluster).toHaveBeenCalledTimes(2); + expect(createdSignalsCount).toEqual(4); // should not create any signals because all events were in the allowlist expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); test('if unsuccessful first bulk create', async () => { - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); const sampleParams = sampleRuleAlertParams(10); - mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3))) + .mockRejectedValue(new Error('bulk failed')); // Added this recently const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), + listClient: ({ + getListItemByValues: async () => { + return ([] as unknown) as ListItemArraySchema; + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -162,72 +328,38 @@ describe('searchAfterAndBulkCreate', () => { }); expect(mockLogger.error).toHaveBeenCalled(); expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(1); + expect(createdSignalsCount).toEqual(0); expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); - test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => { + test('should return success with 0 total hits', async () => { const sampleParams = sampleRuleAlertParams(); - mockService.callCluster.mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', + mockService.callCluster.mockResolvedValueOnce(sampleEmptyDocSearchResults()); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + listClient: ({ + getListItemByValues: async ({ + value, + }: { + type: string; + listId: string; + value: string[]; + }) => { + return value.map((item) => ({ value: item })); }, + } as unknown) as ListClient, + exceptionsList: [ { - create: { - status: 201, - }, + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], }, ], - }); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(mockLogger.error).toHaveBeenCalled(); - expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - - test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => { - const sampleParams = sampleRuleAlertParams(); - mockService.callCluster.mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - { - create: { - status: 201, - }, - }, - ], - }); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: sampleDocSearchResultsNoSortIdNoHits(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -249,105 +381,12 @@ describe('searchAfterAndBulkCreate', () => { throttle: 'no_actions', }); expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - - test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { - const sampleParams = sampleRuleAlertParams(10); - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); - mockService.callCluster - .mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - { - create: { - status: 201, - }, - }, - ], - }) - .mockResolvedValueOnce(sampleDocSearchResultsNoSortId()); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - }); - - test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => { - const sampleParams = sampleRuleAlertParams(10); - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); - mockService.callCluster - .mockResolvedValueOnce({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - { - create: { - status: 201, - }, - }, - ], - }) - .mockResolvedValueOnce(sampleEmptyDocSearchResults()); - const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), - ruleParams: sampleParams, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - inputIndexPattern, - signalsIndex: DEFAULT_SIGNALS_INDEX, - name: 'rule-name', - actions: [], - createdAt: '2020-01-28T15:58:34.810Z', - updatedAt: '2020-01-28T15:59:14.004Z', - createdBy: 'elastic', - updatedBy: 'elastic', - interval: '5m', - enabled: true, - pageSize: 1, - filter: undefined, - refresh: false, - tags: ['some fake tag 1', 'some fake tag 2'], - throttle: 'no_actions', - }); - expect(success).toEqual(true); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + expect(createdSignalsCount).toEqual(0); + expect(lastLookBackDate).toEqual(null); }); test('if returns false when singleSearchAfter throws an exception', async () => { const sampleParams = sampleRuleAlertParams(10); - const someGuids = Array.from({ length: 4 }).map((x) => uuid.v4()); mockService.callCluster .mockResolvedValueOnce({ took: 100, @@ -367,7 +406,30 @@ describe('searchAfterAndBulkCreate', () => { throw Error('Fake Error'); }); const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - someResult: repeatedSearchResultsWithSortId(4, 1, someGuids), + listClient: ({ + getListItemByValues: async ({ + value, + }: { + type: string; + listId: string; + value: string[]; + }) => { + return value.map((item) => ({ value: item })); + }, + } as unknown) as ListClient, + exceptionsList: [ + { + field: 'source.ip', + values_operator: 'included', + values_type: 'list', + values: [ + { + id: 'ci-badguys.txt', + name: 'ip', + }, + ], + }, + ], ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -389,7 +451,7 @@ describe('searchAfterAndBulkCreate', () => { throttle: 'no_actions', }); expect(success).toEqual(false); - expect(createdSignalsCount).toEqual(1); - expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + expect(createdSignalsCount).toEqual(0); // should not create signals if search threw error + expect(lastLookBackDate).toEqual(null); }); }); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts index acf3e9bfb055..05cdccedbc2c 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -4,18 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ListClient } from '../../../../../lists/server'; import { AlertServices } from '../../../../../alerting/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams, RefreshTypes } from '../types'; +import { RuleTypeParams, RefreshTypes, RuleAlertParams } from '../types'; import { Logger } from '../../../../../../../src/core/server'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; import { SignalSearchResponse } from './types'; +import { filterEventsAgainstList } from './filter_events_with_list'; interface SearchAfterAndBulkCreateParams { - someResult: SignalSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; + listClient: ListClient | undefined; // TODO: undefined is for temporary development, remove before merged + exceptionsList: RuleAlertParams['exceptions_list']; logger: Logger; id: string; inputIndexPattern: string[]; @@ -45,9 +48,10 @@ export interface SearchAfterAndBulkCreateReturnType { // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ - someResult, ruleParams, + exceptionsList, services, + listClient, logger, id, inputIndexPattern, @@ -73,71 +77,31 @@ export const searchAfterAndBulkCreate = async ({ lastLookBackDate: null, createdSignalsCount: 0, }; - if (someResult.hits.hits.length === 0) { - toReturn.success = true; - return toReturn; - } - logger.debug('[+] starting bulk insertion'); - const { bulkCreateDuration, createdItemsCount } = await singleBulkCreate({ - someResult, - ruleParams, - services, - logger, - id, - signalsIndex, - actions, - name, - createdAt, - createdBy, - updatedAt, - updatedBy, - interval, - enabled, - refresh, - tags, - throttle, - }); + let sortId; // tells us where to start our next search_after query + let searchResultSize = 0; - if (createdItemsCount > 0) { - toReturn.createdSignalsCount = createdItemsCount; - toReturn.lastLookBackDate = - someResult.hits.hits.length > 0 - ? new Date(someResult.hits.hits[someResult.hits.hits.length - 1]?._source['@timestamp']) - : null; - } + /* + The purpose of `maxResults` is to ensure we do not perform + extra search_after's. This will be reset on each + iteration, although it really only matters for the first + iteration of the loop. + e.g. if maxSignals = 100 but our search result only yields + 27 documents, there is no point in performing another search + since we know there are no more events that match our rule, + and thus, no more signals we could possibly generate. + However, if maxSignals = 500 and our search yields a total + of 3050 results we don't want to make 3050 signals, + we only want 500. So maxResults will help us control how + many times we perform a search_after + */ + let maxResults = ruleParams.maxSignals; - if (bulkCreateDuration) { - toReturn.bulkCreateTimes.push(bulkCreateDuration); - } - const totalHits = - typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value; - // maxTotalHitsSize represents the total number of docs to - // query for, no matter the size of each individual page of search results. - // If the total number of hits for the overall search result is greater than - // maxSignals, default to requesting a total of maxSignals, otherwise use the - // totalHits in the response from the searchAfter query. - const maxTotalHitsSize = Math.min(totalHits, ruleParams.maxSignals); + // Get - // number of docs in the current search result - let hitsSize = someResult.hits.hits.length; - logger.debug(`first size: ${hitsSize}`); - let sortIds = someResult.hits.hits[0].sort; - if (sortIds == null && totalHits > 0) { - logger.error('sortIds was empty on first search but expected more'); - toReturn.success = false; - return toReturn; - } else if (sortIds == null && totalHits === 0) { - toReturn.success = true; - return toReturn; - } - let sortId; - if (sortIds != null) { - sortId = sortIds[0]; - } - while (hitsSize < maxTotalHitsSize && hitsSize !== 0) { + while (searchResultSize < maxResults) { try { - logger.debug(`sortIds: ${sortIds}`); + logger.debug(`sortIds: ${sortId}`); const { searchResult, searchDuration, @@ -152,25 +116,60 @@ export const searchAfterAndBulkCreate = async ({ pageSize, // maximum number of docs to receive per search result. }); toReturn.searchAfterTimes.push(searchDuration); + toReturn.lastLookBackDate = + searchResult.hits.hits.length > 0 + ? new Date( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?._source['@timestamp'] + ) + : null; + const totalHits = + typeof searchResult.hits.total === 'number' + ? searchResult.hits.total + : searchResult.hits.total.value; + logger.debug(`totalHits: ${totalHits}`); + + // re-calculate maxResults to ensure if our search results + // are less than max signals, we are not attempting to + // create more signals than there are total search results. + maxResults = Math.min(totalHits, ruleParams.maxSignals); + searchResultSize += searchResult.hits.hits.length; if (searchResult.hits.hits.length === 0) { toReturn.success = true; return toReturn; } - hitsSize += searchResult.hits.hits.length; - logger.debug(`size adjusted: ${hitsSize}`); - sortIds = searchResult.hits.hits[0].sort; - if (sortIds == null) { - logger.debug('sortIds was empty on search'); + + // filter out the search results that match with the values found in the list. + // the resulting set are valid signals that are not on the allowlist. + const filteredEvents = + listClient != null + ? await filterEventsAgainstList({ + listClient, + exceptionsList, + logger, + eventSearchResult: searchResult, + }) + : searchResult; + + if (filteredEvents.hits.hits.length === 0) { + // everything in the events were allowed, so no need to generate signals toReturn.success = true; - return toReturn; // no more search results + return toReturn; + } + + // cap max signals created to be no more than maxSignals + if (toReturn.createdSignalsCount + filteredEvents.hits.hits.length > ruleParams.maxSignals) { + const tempSignalsToIndex = filteredEvents.hits.hits.slice( + 0, + ruleParams.maxSignals - toReturn.createdSignalsCount + ); + filteredEvents.hits.hits = tempSignalsToIndex; } - sortId = sortIds[0]; logger.debug('next bulk index'); const { bulkCreateDuration: bulkDuration, createdItemsCount: createdCount, } = await singleBulkCreate({ - someResult: searchResult, + filteredEvents, ruleParams, services, logger, @@ -189,17 +188,25 @@ export const searchAfterAndBulkCreate = async ({ throttle, }); logger.debug('finished next bulk index'); + logger.debug(`created ${createdCount} signals`); toReturn.createdSignalsCount += createdCount; if (bulkDuration) { toReturn.bulkCreateTimes.push(bulkDuration); } + + if (filteredEvents.hits.hits[0].sort == null) { + logger.debug('sortIds was empty on search'); + toReturn.success = true; + return toReturn; // no more search results + } + sortId = filteredEvents.hits.hits[0].sort[0]; } catch (exc) { logger.error(`[-] search_after and bulk threw an error ${exc}`); toReturn.success = false; return toReturn; } } - logger.debug(`[+] completed bulk index of ${maxTotalHitsSize}`); + logger.debug(`[+] completed bulk index of ${toReturn.createdSignalsCount}`); toReturn.success = true; return toReturn; }; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 0c7f0839f8da..ea7255b8a925 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -17,6 +17,7 @@ import { scheduleNotificationActions } from '../notifications/schedule_notificat import { RuleAlertType } from '../rules/types'; import { findMlSignals } from './find_ml_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; +import { ListPluginSetup } from '../../../../../lists/server/types'; jest.mock('./rule_status_saved_objects_client'); jest.mock('./rule_status_service'); @@ -68,6 +69,11 @@ describe('rules_notification_alert_type', () => { modulesProvider: jest.fn(), resultsServiceProvider: jest.fn(), }; + const listMock = { + getListClient: () => ({ + getListItemByValues: () => [], + }), + }; let payload: jest.Mocked; let alert: ReturnType; let logger: ReturnType; @@ -110,6 +116,7 @@ describe('rules_notification_alert_type', () => { logger, version, ml: mlMock, + lists: (listMock as unknown) as ListPluginSetup, }); }); @@ -199,6 +206,7 @@ describe('rules_notification_alert_type', () => { logger, version, ml: undefined, + lists: undefined, }); await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); @@ -358,7 +366,7 @@ describe('rules_notification_alert_type', () => { }); it('when error was thrown', async () => { - (searchAfterAndBulkCreate as jest.Mock).mockResolvedValue({}); + (searchAfterAndBulkCreate as jest.Mock).mockRejectedValue({}); await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain('An error occurred during rule execution'); diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8cef4c8ea0e6..6885b4c81467 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -4,14 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { performance } from 'perf_hooks'; +/* eslint-disable complexity */ + import { Logger, KibanaRequest } from 'src/core/server'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE } from '../../../../common/constants'; import { isJobStarted, isMlRule } from '../../../../common/machine_learning/helpers'; import { SetupPlugins } from '../../../plugin'; -import { buildEventsSearchQuery } from './build_events_query'; +import { ListClient } from '../../../../../lists/server'; + import { getInputIndex } from './get_input_output_index'; import { searchAfterAndBulkCreate, @@ -19,7 +21,7 @@ import { } from './search_after_bulk_create'; import { getFilter } from './get_filter'; import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; -import { getGapBetweenRuns, makeFloatString, parseScheduleDates } from './utils'; +import { getGapBetweenRuns, parseScheduleDates } from './utils'; import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; @@ -32,15 +34,18 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { hasListsFeature } from '../feature_flags'; export const signalRulesAlertType = ({ logger, version, ml, + lists, }: { logger: Logger; version: string; ml: SetupPlugins['ml']; + lists: SetupPlugins['lists'] | undefined; }): SignalRuleAlertTypeDefinition => { return { id: SIGNALS_ID, @@ -51,7 +56,14 @@ export const signalRulesAlertType = ({ params: signalParamsSchema(), }, producer: 'siem', - async executor({ previousStartedAt, alertId, services, params }) { + async executor({ + previousStartedAt, + alertId, + services, + params, + spaceId, + updatedBy: updatedByUser, + }) { const { anomalyThreshold, from, @@ -67,7 +79,7 @@ export const signalRulesAlertType = ({ query, to, type, - exceptions_list, + exceptions_list: exceptionsList, } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; @@ -123,7 +135,6 @@ export const signalRulesAlertType = ({ hasError = true; await ruleStatusService.error(gapMessage, { gap: gapString }); } - try { if (isMlRule(type)) { if (ml == null) { @@ -199,6 +210,18 @@ export const signalRulesAlertType = ({ result.bulkCreateTimes.push(bulkCreateDuration); } } else { + let listClient: ListClient | undefined; + if (hasListsFeature()) { + if (lists == null) { + throw new Error('lists plugin unavailable during rule execution'); + } + listClient = await lists.getListClient( + services.callCluster, + spaceId, + updatedByUser ?? 'elastic' + ); + } + const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, @@ -208,34 +231,13 @@ export const signalRulesAlertType = ({ savedId, services, index: inputIndex, - lists: exceptions_list, + // temporary filter out list type + lists: exceptionsList?.filter((item) => item.values_type !== 'list'), }); - const noReIndex = buildEventsSearchQuery({ - index: inputIndex, - from, - to, - filter: esFilter, - size: searchAfterSize, - searchAfterSortId: undefined, - }); - - logger.debug(buildRuleMessage('[+] Initial search call')); - const start = performance.now(); - const noReIndexResult = await services.callCluster('search', noReIndex); - const end = performance.now(); - - const signalCount = noReIndexResult.hits.total.value; - if (signalCount !== 0) { - logger.info( - buildRuleMessage( - `Found ${signalCount} signals from the indexes of "[${inputIndex.join(', ')}]"` - ) - ); - } - result = await searchAfterAndBulkCreate({ - someResult: noReIndexResult, + listClient, + exceptionsList, ruleParams: params, services, logger, @@ -256,7 +258,6 @@ export const signalRulesAlertType = ({ tags, throttle, }); - result.searchAfterTimes.push(makeFloatString(end - start)); } if (result.success) { @@ -293,6 +294,11 @@ export const signalRulesAlertType = ({ } logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); + logger.debug( + buildRuleMessage( + `[+] Finished indexing ${result.createdSignalsCount} signals into ${outputIndex}` + ) + ); if (!hasError) { await ruleStatusService.success('succeeded', { bulkCreateTimeDurations: result.bulkCreateTimes, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts index 6f3cc6e708fc..265f98653313 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -141,7 +141,7 @@ describe('singleBulkCreate', () => { ], }); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(), + filteredEvents: sampleDocSearchResultsNoSortId(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -175,7 +175,7 @@ describe('singleBulkCreate', () => { ], }); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleDocSearchResultsNoSortIdNoVersion(), + filteredEvents: sampleDocSearchResultsNoSortIdNoVersion(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -201,7 +201,7 @@ describe('singleBulkCreate', () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(false); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleEmptyDocSearchResults(), + filteredEvents: sampleEmptyDocSearchResults(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -228,7 +228,7 @@ describe('singleBulkCreate', () => { const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleSearchResult(), + filteredEvents: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -257,7 +257,7 @@ describe('singleBulkCreate', () => { const sampleSearchResult = sampleDocSearchResultsNoSortId; mockService.callCluster.mockResolvedValue(sampleBulkCreateErrorResult); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleSearchResult(), + filteredEvents: sampleSearchResult(), ruleParams: sampleParams, services: mockService, logger: mockLogger, @@ -352,7 +352,7 @@ describe('singleBulkCreate', () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); const { success, createdItemsCount } = await singleBulkCreate({ - someResult: sampleDocSearchResultsNoSortId(), + filteredEvents: sampleDocSearchResultsNoSortId(), ruleParams: sampleParams, services: mockService, logger: mockLogger, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts index c162c8855b09..39aecde470e0 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts @@ -15,7 +15,7 @@ import { buildBulkBody } from './build_bulk_body'; import { Logger } from '../../../../../../../src/core/server'; interface SingleBulkCreateParams { - someResult: SignalSearchResponse; + filteredEvents: SignalSearchResponse; ruleParams: RuleTypeParams; services: AlertServices; logger: Logger; @@ -64,7 +64,7 @@ export interface SingleBulkCreateResponse { // Bulk Index documents. export const singleBulkCreate = async ({ - someResult, + filteredEvents, ruleParams, services, logger, @@ -82,8 +82,8 @@ export const singleBulkCreate = async ({ tags, throttle, }: SingleBulkCreateParams): Promise => { - someResult.hits.hits = filterDuplicateRules(id, someResult); - if (someResult.hits.hits.length === 0) { + filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents); + if (filteredEvents.hits.hits.length === 0) { return { success: true, createdItemsCount: 0 }; } // index documents after creating an ID based on the @@ -95,7 +95,7 @@ export const singleBulkCreate = async ({ // while preventing duplicates from being added to the // signals index if rules are re-run over the same time // span. Also allow for versioning. - const bulkBody = someResult.hits.hits.flatMap((doc) => [ + const bulkBody = filteredEvents.hits.hits.flatMap((doc) => [ { create: { _index: signalsIndex, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts index 580080966457..2aa42234460d 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts @@ -22,18 +22,17 @@ describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; mockService.callCluster.mockResolvedValue(sampleDocSearchResultsNoSortId); - await expect( - singleSearchAfter({ - searchAfterSortId, - index: [], - from: 'now-360s', - to: 'now', - services: mockService, - logger: mockLogger, - pageSize: 1, - filter: undefined, - }) - ).rejects.toThrow('Attempted to search after with empty sort id'); + const { searchResult } = await singleSearchAfter({ + searchAfterSortId, + index: [], + from: 'now-360s', + to: 'now', + services: mockService, + logger: mockLogger, + pageSize: 1, + filter: undefined, + }); + expect(searchResult).toEqual(sampleDocSearchResultsNoSortId); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts index 8071c18713c1..a7086a4fb229 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts @@ -36,9 +36,6 @@ export const singleSearchAfter = async ({ searchResult: SignalSearchResponse; searchDuration: string; }> => { - if (searchAfterSortId == null) { - throw Error('Attempted to search after with empty sort id'); - } try { const searchAfterQuery = buildEventsSearchQuery({ index, diff --git a/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts index b493bab8b461..32b13c5251a6 100644 --- a/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/siem/server/lib/detection_engine/signals/types.ts @@ -100,6 +100,7 @@ export interface GetResponse { _source: SearchTypes; } +export type EventSearchResponse = SearchResponse; export type SignalSearchResponse = SearchResponse; export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; diff --git a/x-pack/plugins/siem/server/plugin.ts b/x-pack/plugins/siem/server/plugin.ts index 3c336991f3d9..5a47efd45888 100644 --- a/x-pack/plugins/siem/server/plugin.ts +++ b/x-pack/plugins/siem/server/plugin.ts @@ -19,6 +19,7 @@ import { PluginSetupContract as AlertingSetup } from '../../alerting/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; +import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; @@ -52,6 +53,7 @@ export interface SetupPlugins { security?: SecuritySetup; spaces?: SpacesSetup; ml?: MlSetup; + lists?: ListPluginSetup; } export interface StartPlugins { @@ -194,6 +196,7 @@ export class Plugin implements IPlugin `--xpack.${key}.enabled=false`), `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'alerts')}`, `--plugin-path=${path.join(__dirname, 'fixtures', 'plugins', 'actions')}`,