diff --git a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts index c8017c9c1279..c3097d0d9e37 100644 --- a/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts @@ -70,6 +70,7 @@ export const getSearchListItemMock = (): SearchResponse _score: 0, _source: getSearchEsListItemMock(), _type: '', + matched_queries: ['0.0'], }, ], max_score: 0, diff --git a/x-pack/plugins/lists/common/schemas/response/index.ts b/x-pack/plugins/lists/common/schemas/response/index.ts index deca06ad99fe..5e739ccf3a0a 100644 --- a/x-pack/plugins/lists/common/schemas/response/index.ts +++ b/x-pack/plugins/lists/common/schemas/response/index.ts @@ -15,3 +15,4 @@ export * from './found_list_schema'; export * from './list_item_schema'; export * from './list_schema'; export * from './list_item_index_exist_schema'; +export * from './search_list_item_schema'; diff --git a/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.mock.ts new file mode 100644 index 000000000000..1ad241ffca07 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.mock.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchListItemSchema } from '../../../common/schemas'; +import { VALUE } from '../../../common/constants.mock'; + +import { getListItemResponseMock } from './list_item_schema.mock'; + +export const getSearchListItemResponseMock = (): SearchListItemSchema => ({ + items: [getListItemResponseMock()], + value: VALUE, +}); diff --git a/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.test.ts b/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.test.ts new file mode 100644 index 000000000000..132c3f16688f --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { left } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; + +import { exactCheck, foldLeftRight, getPaths } from '../../shared_imports'; + +import { getSearchListItemResponseMock } from './search_list_item_schema.mock'; +import { SearchListItemSchema, searchListItemSchema } from './search_list_item_schema'; + +describe('search_list_item_schema', () => { + test('it should validate a typical search list item response', () => { + const payload = getSearchListItemResponseMock(); + const decoded = searchListItemSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it should NOT validate with an "undefined" for "items"', () => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { items, ...noItems } = getSearchListItemResponseMock(); + const decoded = searchListItemSchema.decode(noItems); + const checked = exactCheck(noItems, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "undefined" supplied to "items"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should not allow an extra key to be sent in', () => { + const payload: SearchListItemSchema & { extraKey?: string } = getSearchListItemResponseMock(); + payload.extraKey = 'some new value'; + const decoded = searchListItemSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual(['invalid keys "extraKey"']); + expect(message.schema).toEqual({}); + }); +}); diff --git a/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.ts new file mode 100644 index 000000000000..5177098a6f67 --- /dev/null +++ b/x-pack/plugins/lists/common/schemas/response/search_list_item_schema.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { listItemArraySchema } from './list_item_schema'; + +/** + * NOTE: Although this is defined within "response" this does not expose a REST API + * endpoint right now for this particular response. Instead this is only used internally + * for the plugins at this moment. If this changes, please remove this message. + */ +export const searchListItemSchema = t.exact( + t.type({ + items: listItemArraySchema, + value: t.unknown, + }) +); + +export type SearchListItemSchema = t.TypeOf; + +export const searchListItemArraySchema = t.array(searchListItemSchema); +export type SearchListItemArraySchema = t.TypeOf; diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts b/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts index f658a51730d9..1120f99bf917 100644 --- a/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts +++ b/x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts @@ -47,7 +47,15 @@ describe('delete_list_item_by_value', () => { body: { query: { bool: { - filter: [{ term: { list_id: 'some-list-id' } }, { terms: { ip: ['127.0.0.1'] } }], + filter: [ + { term: { list_id: 'some-list-id' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '0.0', value: '127.0.0.1' } } }], + }, + }, + ], }, }, }, diff --git a/x-pack/plugins/lists/server/services/items/index.ts b/x-pack/plugins/lists/server/services/items/index.ts index bc04ba88b943..31003771679a 100644 --- a/x-pack/plugins/lists/server/services/items/index.ts +++ b/x-pack/plugins/lists/server/services/items/index.ts @@ -11,6 +11,7 @@ export * from './delete_list_item_by_value'; export * from './delete_list_item'; export * from './find_list_item'; export * from './get_list_item_by_value'; +export * from './get_list_item_by_values'; export * from './get_list_item'; export * from './get_list_item_by_values'; export * from './get_list_item_template'; @@ -18,3 +19,4 @@ export * from './get_list_item_index'; export * from './update_list_item'; export * from './write_lines_to_bulk_list_items'; export * from './write_list_items_to_stream'; +export * from './search_list_item_by_values'; diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.mock.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.mock.ts new file mode 100644 index 000000000000..40b5fbb3ab8f --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.mock.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getCallClusterMock } from '../../../common/get_call_cluster.mock'; +import { SearchListItemByValuesOptions } from '../items'; +import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE, VALUE_2 } from '../../../common/constants.mock'; + +export const searchListItemByValuesOptionsMocks = (): SearchListItemByValuesOptions => ({ + callCluster: getCallClusterMock(), + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [VALUE, VALUE_2], +}); diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts new file mode 100644 index 000000000000..b2a89dfe321a --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchListItemArraySchema } from '../../../common/schemas'; +import { getSearchListItemMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; +import { getCallClusterMock } from '../../../common/get_call_cluster.mock'; +import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE, VALUE_2 } from '../../../common/constants.mock'; + +import { searchListItemByValues } from './search_list_item_by_values'; + +describe('search_list_item_by_values', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('Returns a an empty array of items if the value is empty', async () => { + const data = getSearchListItemMock(); + data.hits.hits = []; + const callCluster = getCallClusterMock(data); + const listItem = await searchListItemByValues({ + callCluster, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [], + }); + + expect(listItem).toEqual([]); + }); + + test('Returns a an empty array of items if the ES query is also empty', async () => { + const data = getSearchListItemMock(); + data.hits.hits = []; + const callCluster = getCallClusterMock(data); + const listItem = await searchListItemByValues({ + callCluster, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [VALUE, VALUE_2], + }); + + const expected: SearchListItemArraySchema = [ + { items: [], value: VALUE }, + { items: [], value: VALUE_2 }, + ]; + expect(listItem).toEqual(expected); + }); + + test('Returns transformed list item if the data exists within ES', async () => { + const data = getSearchListItemMock(); + const callCluster = getCallClusterMock(data); + const listItem = await searchListItemByValues({ + callCluster, + listId: LIST_ID, + listItemIndex: LIST_ITEM_INDEX, + type: TYPE, + value: [VALUE, VALUE_2], + }); + + const expected: SearchListItemArraySchema = [ + { + items: [ + { + _version: undefined, + created_at: '2020-04-20T15:25:31.830Z', + created_by: 'some user', + deserializer: undefined, + id: 'some-list-item-id', + list_id: 'some-list-id', + meta: {}, + serializer: undefined, + tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e', + type: 'ip', + updated_at: '2020-04-20T15:25:31.830Z', + updated_by: 'some user', + value: '127.0.0.1', + }, + ], + value: '127.0.0.1', + }, + { + items: [], + value: VALUE_2, + }, + ]; + expect(listItem).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts new file mode 100644 index 000000000000..33025a6a177f --- /dev/null +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LegacyAPICaller } from 'kibana/server'; + +import { SearchEsListItemSchema, SearchListItemArraySchema, Type } from '../../../common/schemas'; +import { getQueryFilterFromTypeValue, transformElasticNamedSearchToListItem } from '../utils'; + +export interface SearchListItemByValuesOptions { + listId: string; + callCluster: LegacyAPICaller; + listItemIndex: string; + type: Type; + value: unknown[]; +} + +export const searchListItemByValues = async ({ + listId, + callCluster, + listItemIndex, + type, + value, +}: SearchListItemByValuesOptions): Promise => { + const response = await callCluster('search', { + body: { + query: { + bool: { + filter: getQueryFilterFromTypeValue({ listId, type, value }), + }, + }, + }, + ignoreUnavailable: true, + index: listItemIndex, + size: 10000, // TODO: This has a limit on the number which is 10,000 the default of Elastic but we might want to provide a way to increase that number + }); + return transformElasticNamedSearchToListItem({ response, type, value }); +}; diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 590bfef6625f..b0640ac8d6ba 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -12,6 +12,7 @@ import { ListItemArraySchema, ListItemSchema, ListSchema, + SearchListItemArraySchema, } from '../../../common/schemas'; import { ConfigType } from '../../config'; import { @@ -35,6 +36,7 @@ import { getListItemIndex, getListItemTemplate, importListItemsToStream, + searchListItemByValues, updateListItem, } from '../../services/items'; import { @@ -67,6 +69,7 @@ import { GetListItemsByValueOptions, GetListOptions, ImportListItemsToStreamOptions, + SearchListItemByValuesOptions, UpdateListItemOptions, UpdateListOptions, } from './list_client_types'; @@ -472,6 +475,22 @@ export class ListClient { }); }; + public searchListItemByValues = async ({ + type, + listId, + value, + }: SearchListItemByValuesOptions): Promise => { + const { callCluster } = this; + const listItemIndex = this.getListItemIndex(); + return searchListItemByValues({ + callCluster, + listId, + listItemIndex, + type, + value, + }); + }; + public findList = async ({ filter, currentIndexPosition, diff --git a/x-pack/plugins/lists/server/services/lists/list_client_types.ts b/x-pack/plugins/lists/server/services/lists/list_client_types.ts index ea983b38c7e5..fd9066cfe240 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client_types.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client_types.ts @@ -160,3 +160,9 @@ export interface FindListItemOptions { sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; } + +export interface SearchListItemByValuesOptions { + type: Type; + listId: string; + value: unknown[]; +} diff --git a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.test.ts b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.test.ts index 3d48e44e26ea..aec9ef629788 100644 --- a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.test.ts +++ b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.test.ts @@ -4,7 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { QueryFilterType, getQueryFilterFromTypeValue } from './get_query_filter_from_type_value'; +import { + QueryFilterType, + getEmptyQuery, + getQueryFilterFromTypeValue, + getShouldQuery, + getTermsQuery, + getTextQuery, +} from './get_query_filter_from_type_value'; describe('get_query_filter_from_type_value', () => { beforeEach(() => { @@ -15,78 +22,813 @@ describe('get_query_filter_from_type_value', () => { jest.clearAllMocks(); }); - test('it returns an ip if given an ip', () => { - const queryFilter = getQueryFilterFromTypeValue({ - listId: 'list-123', - type: 'ip', - value: ['127.0.0.1'], + describe('getQueryFilterFromTypeValue', () => { + test('it returns an ip if given an ip', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'ip', + value: ['127.0.0.1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '0.0', value: '127.0.0.1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two ip if given two ip', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'ip', + value: ['127.0.0.1', '127.0.0.2'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { term: { ip: { _name: '0.0', value: '127.0.0.1' } } }, + { term: { ip: { _name: '1.0', value: '127.0.0.2' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns a keyword if given a keyword', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'keyword', + value: ['host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { keyword: { _name: '0.0', value: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two keywords if given two values', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'keyword', + value: ['host-name-1', 'host-name-2'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { term: { keyword: { _name: '0.0', value: 'host-name-1' } } }, + { term: { keyword: { _name: '1.0', value: 'host-name-2' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns an empty query given an empty value', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'keyword', + value: [], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_none: { + _name: 'empty', + }, + }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns an empty query object given an empty array', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'ip', + value: [], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_none: { + _name: 'empty', + }, + }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns an empty query object given an array with only null values', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'ip', + value: [null, null], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_none: { + _name: 'empty', + }, + }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a null value if mixed with a string value for non-text based query', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'ip', + value: [null, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out an object value if mixed with a string value for non-text based query', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'ip', + value: [{}, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a null value if mixed with a string value for text based query', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'text', + value: [null, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { text: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out object values if mixed with a string value for text based query', () => { + const query = getQueryFilterFromTypeValue({ + listId: 'list-123', + type: 'text', + value: [{}, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { text: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); }); - const expected: QueryFilterType = [ - { term: { list_id: 'list-123' } }, - { terms: { ip: ['127.0.0.1'] } }, - ]; - expect(queryFilter).toEqual(expected); }); - test('it returns two ip if given two ip', () => { - const queryFilter = getQueryFilterFromTypeValue({ - listId: 'list-123', - type: 'ip', - value: ['127.0.0.1', '127.0.0.2'], + describe('getEmptyQuery', () => { + test('it returns an empty query given a list_id', () => { + const emptyQuery = getEmptyQuery({ listId: 'list-123' }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { bool: { minimum_should_match: 1, should: [{ match_none: { _name: 'empty' } }] } }, + ]; + expect(emptyQuery).toEqual(expected); }); - const expected: QueryFilterType = [ - { term: { list_id: 'list-123' } }, - { terms: { ip: ['127.0.0.1', '127.0.0.2'] } }, - ]; - expect(queryFilter).toEqual(expected); }); - test('it returns a keyword if given a keyword', () => { - const queryFilter = getQueryFilterFromTypeValue({ - listId: 'list-123', - type: 'keyword', - value: ['host-name-1'], + describe('getTermsQuery', () => { + describe('scalar values', () => { + test('it returns a expected terms query give a single string value, listId, and type', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: ['127.0.0.1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '0.0', value: '127.0.0.1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected terms query given two string values, listId, and type', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: ['127.0.0.1', '127.0.0.2'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { term: { ip: { _name: '0.0', value: '127.0.0.1' } } }, + { term: { ip: { _name: '1.0', value: '127.0.0.2' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected numeric terms without converting them into strings', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [5, 3], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { term: { ip: { _name: '0.0', value: 5 } } }, + { term: { ip: { _name: '1.0', value: 3 } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns a string and a numeric without converting them into a homogenous type', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [5, '3'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { term: { ip: { _name: '0.0', value: 5 } } }, + { term: { ip: { _name: '1.0', value: '3' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a null value if mixed with a string value', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [null, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out an object value if mixed with a string value', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [{}, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + }); + + describe('array values', () => { + test('it returns a expected terms query give a single string value, listId, and type', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [['127.0.0.1']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ terms: { _name: '0.0', ip: ['127.0.0.1'] } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected terms query given two string values, listId, and type', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [['127.0.0.1'], ['127.0.0.2']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { terms: { _name: '0.0', ip: ['127.0.0.1'] } }, + { terms: { _name: '1.0', ip: ['127.0.0.2'] } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected numeric terms without converting them into strings', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [[5], [3]], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ terms: { _name: '0.0', ip: [5] } }, { terms: { _name: '1.0', ip: [3] } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns a string and a numeric without converting them into a homogenous type', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [[5], ['3']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { terms: { _name: '0.0', ip: [5] } }, + { terms: { _name: '1.0', ip: ['3'] } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a null value if mixed with a string value', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [[null], ['host-name-1']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ terms: { _name: '1.0', ip: ['host-name-1'] } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out an object value if mixed with a string value', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [[{}], ['host-name-1']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ terms: { _name: '1.0', ip: ['host-name-1'] } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it flattens and removes null values correctly in a deeply nested set of arrays', () => { + const query = getTermsQuery({ + listId: 'list-123', + type: 'ip', + value: [ + [null], + [ + 'host-name-1', + ['host-name-2', [null], ['host-name-3'], ['host-name-4', null, 'host-name-5']], + ], + ['host-name-6'], + ], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + terms: { + _name: '1.0', + ip: ['host-name-1', 'host-name-2', 'host-name-3', 'host-name-4', 'host-name-5'], + }, + }, + { terms: { _name: '2.0', ip: ['host-name-6'] } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); }); - const expected: QueryFilterType = [ - { term: { list_id: 'list-123' } }, - { terms: { keyword: ['host-name-1'] } }, - ]; - expect(queryFilter).toEqual(expected); }); - test('it returns two keywords if given two values', () => { - const queryFilter = getQueryFilterFromTypeValue({ - listId: 'list-123', - type: 'keyword', - value: ['host-name-1', 'host-name-2'], + describe('getTextQuery', () => { + describe('scalar values', () => { + test('it returns a expected terms query give a single string value, listId, and type', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: ['127.0.0.1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected terms query given two string values, listId, and type', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: ['127.0.0.1', '127.0.0.2'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } }, + { match: { ip: { _name: '1.0', operator: 'and', query: '127.0.0.2' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected numeric terms without converting them into strings', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [5, 3], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } }, + { match: { ip: { _name: '1.0', operator: 'and', query: 3 } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns a string and a numeric without converting them into a homogenous type', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [5, '3'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } }, + { match: { ip: { _name: '1.0', operator: 'and', query: '3' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a null value if mixed with a string value', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [null, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out an object value if mixed with a string value', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [{}, 'host-name-1'], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + }); + + describe('array values', () => { + test('it returns a expected terms query give a single string value, listId, and type', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [['127.0.0.1']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected terms query given two string values, listId, and type', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [['127.0.0.1'], ['127.0.0.2']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } }, + { match: { ip: { _name: '1.0', operator: 'and', query: '127.0.0.2' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns two expected numeric terms without converting them into strings', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [[5], [3]], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } }, + { match: { ip: { _name: '1.0', operator: 'and', query: 3 } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it returns a string and a numeric without converting them into a homogenous type', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [[5], ['3']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } }, + { match: { ip: { _name: '1.0', operator: 'and', query: '3' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a null value if mixed with a string value', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [[null], ['host-name-1']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it filters out a object value if mixed with a string value', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [[{}], ['host-name-1']], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }], + }, + }, + ]; + expect(query).toEqual(expected); + }); + + test('it flattens and removes null values correctly in a deeply nested set of arrays', () => { + const query = getTextQuery({ + listId: 'list-123', + type: 'ip', + value: [ + [null], + [ + 'host-name-1', + ['host-name-2', [null], ['host-name-3'], ['host-name-4', null, 'host-name-5']], + ], + ['host-name-6'], + ], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }, + { match: { ip: { _name: '1.1', operator: 'and', query: 'host-name-2' } } }, + { match: { ip: { _name: '1.2', operator: 'and', query: 'host-name-3' } } }, + { match: { ip: { _name: '1.3', operator: 'and', query: 'host-name-4' } } }, + { match: { ip: { _name: '1.4', operator: 'and', query: 'host-name-5' } } }, + { match: { ip: { _name: '2.0', operator: 'and', query: 'host-name-6' } } }, + ], + }, + }, + ]; + expect(query).toEqual(expected); + }); }); - const expected: QueryFilterType = [ - { term: { list_id: 'list-123' } }, - { terms: { keyword: ['host-name-1', 'host-name-2'] } }, - ]; - expect(queryFilter).toEqual(expected); }); - test('it returns an empty keyword given an empty value', () => { - const queryFilter = getQueryFilterFromTypeValue({ - listId: 'list-123', - type: 'keyword', - value: [], + describe('getShouldQuery', () => { + test('it returns a should as-is when passed one', () => { + const query = getShouldQuery({ + listId: 'list-123', + should: [ + { + bool: { + minimum_should_match: 1, + should: [{ terms: { _name: '0.0', ip: ['127.0.0.1'] } }], + }, + }, + ], + }); + const expected: QueryFilterType = [ + { term: { list_id: 'list-123' } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + bool: { + minimum_should_match: 1, + should: [{ terms: { _name: '0.0', ip: ['127.0.0.1'] } }], + }, + }, + ], + }, + }, + ]; + expect(query).toEqual(expected); }); - const expected: QueryFilterType = [ - { term: { list_id: 'list-123' } }, - { terms: { keyword: [] } }, - ]; - expect(queryFilter).toEqual(expected); - }); - - test('it returns an empty ip given an empty value', () => { - const queryFilter = getQueryFilterFromTypeValue({ - listId: 'list-123', - type: 'ip', - value: [], - }); - const expected: QueryFilterType = [{ term: { list_id: 'list-123' } }, { terms: { ip: [] } }]; - expect(queryFilter).toEqual(expected); }); }); diff --git a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts index 3baba07aa988..cf332cd6dd95 100644 --- a/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts +++ b/x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts @@ -4,19 +4,170 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEmpty, isObject } from 'lodash/fp'; + import { Type } from '../../../common/schemas'; export type QueryFilterType = [ - { term: Record }, - { terms: Record } + { term: Record }, + { terms: Record } | { bool: {} } ]; +/** + * Given a type, value, and listId, this will return a valid query. If the type is + * "text" it will return a "text" match, otherwise it returns a terms query. If an + * array or array of arrays is passed, this will flatten, remove any "null" values, + * and then the result. + * @param type The type of list + * @param value The unknown value + * @param listId The list id + */ export const getQueryFilterFromTypeValue = ({ type, value, listId, }: { type: Type; - value: string[]; + value: unknown[]; listId: string; -}): QueryFilterType => [{ term: { list_id: listId } }, { terms: { [type]: value } }]; +}): QueryFilterType => { + const valueFlattened = value + .flat(Infinity) + .filter((singleValue) => singleValue != null && !isObject(singleValue)); + if (isEmpty(valueFlattened)) { + return getEmptyQuery({ listId }); + } else if (type === 'text') { + return getTextQuery({ listId, type, value }); + } else { + return getTermsQuery({ listId, type, value }); + } +}; + +/** + * Returns an empty named query that should not match anything + * @param listId The list id to associate with the empty query + */ +export const getEmptyQuery = ({ listId }: { listId: string }): QueryFilterType => [ + { term: { list_id: listId } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_none: { + _name: 'empty', + }, + }, + ], + }, + }, +]; + +/** + * Returns a terms query against a large value based list. If it detects that an array or item has a "null" + * value it will filter that value out. If it has arrays within arrays it will flatten those out as well. + * @param value The value which can be unknown + * @param type The list type type + * @param listId The list id + */ +export const getTermsQuery = ({ + value, + type, + listId, +}: { + value: unknown[]; + type: Type; + listId: string; +}): QueryFilterType => { + const should = value.reduce((accum, item, index) => { + if (Array.isArray(item)) { + const itemFlattened = item + .flat(Infinity) + .filter((singleValue) => singleValue != null && !isObject(singleValue)); + if (itemFlattened.length === 0) { + return accum; + } else { + return [...accum, { terms: { _name: `${index}.0`, [type]: itemFlattened } }]; + } + } else { + if (item == null || isObject(item)) { + return accum; + } else { + return [...accum, { term: { [type]: { _name: `${index}.0`, value: item } } }]; + } + } + }, []); + return getShouldQuery({ listId, should }); +}; + +/** + * Returns a text query against a large value based list. If it detects that an array or item has a "null" + * value it will filter that value out. If it has arrays within arrays it will flatten those out as well. + * @param value The value which can be unknown + * @param type The list type type + * @param listId The list id + */ +export const getTextQuery = ({ + value, + type, + listId, +}: { + value: unknown[]; + type: Type; + listId: string; +}): QueryFilterType => { + const should = value.reduce((accum, item, index) => { + if (Array.isArray(item)) { + const itemFlattened = item + .flat(Infinity) + .filter((singleValue) => singleValue != null && !isObject(singleValue)); + if (itemFlattened.length === 0) { + return accum; + } else { + return [ + ...accum, + ...itemFlattened.map((flatItem, secondIndex) => ({ + match: { + [type]: { _name: `${index}.${secondIndex}`, operator: 'and', query: flatItem }, + }, + })), + ]; + } + } else { + if (item == null || isObject(item)) { + return accum; + } else { + return [ + ...accum, + { match: { [type]: { _name: `${index}.0`, operator: 'and', query: item } } }, + ]; + } + } + }, []); + + return getShouldQuery({ listId, should }); +}; + +/** + * Given an unknown should this constructs a simple bool and terms with the should + * clause/query. + * @param listId The list id to query against + * @param should The unknown should to construct the query against + */ +export const getShouldQuery = ({ + listId, + should, +}: { + listId: string; + should: unknown; +}): QueryFilterType => { + return [ + { term: { list_id: listId } }, + { + bool: { + minimum_should_match: 1, + should, + }, + }, + ]; +}; diff --git a/x-pack/plugins/lists/server/services/utils/index.ts b/x-pack/plugins/lists/server/services/utils/index.ts index f7ed118ea585..57f37a1d6bfc 100644 --- a/x-pack/plugins/lists/server/services/utils/index.ts +++ b/x-pack/plugins/lists/server/services/utils/index.ts @@ -15,6 +15,7 @@ export * from './get_search_after_with_tie_breaker'; export * from './get_sort_with_tie_breaker'; export * from './get_source_with_tie_breaker'; export * from './scroll_to_start_page'; +export * from './transform_elastic_named_search_to_list_item'; export * from './transform_elastic_to_list'; export * from './transform_elastic_to_list_item'; export * from './transform_list_item_to_elastic_query'; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.test.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.test.ts new file mode 100644 index 000000000000..83a486b5d154 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getSearchListItemResponseMock } from '../../../common/schemas/response/search_list_item_schema.mock'; +import { LIST_INDEX, LIST_ITEM_ID, TYPE, VALUE } from '../../../common/constants.mock'; +import { + getSearchEsListItemMock, + getSearchListItemMock, +} from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock'; +import { SearchListItemArraySchema } from '../../../common/schemas'; + +import { transformElasticNamedSearchToListItem } from './transform_elastic_named_search_to_list_item'; + +describe('transform_elastic_named_search_to_list_item', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('if given an empty array for values, it returns an empty array', () => { + const response = getSearchListItemMock(); + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [], + }); + const expected: SearchListItemArraySchema = []; + expect(queryFilter).toEqual(expected); + }); + + test('if given an empty array for hits, it returns an empty match', () => { + const response = getSearchListItemMock(); + response.hits.hits = []; + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [VALUE], + }); + const expected: SearchListItemArraySchema = [{ items: [], value: VALUE }]; + expect(queryFilter).toEqual(expected); + }); + + test('it transforms a single elastic type to a search list item type', () => { + const response = getSearchListItemMock(); + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [VALUE], + }); + const expected: SearchListItemArraySchema = [getSearchListItemResponseMock()]; + expect(queryFilter).toEqual(expected); + }); + + test('it transforms two elastic types to a search list item type', () => { + const response = getSearchListItemMock(); + response.hits.hits = [ + ...response.hits.hits, + { + _id: LIST_ITEM_ID, + _index: LIST_INDEX, + _score: 0, + _source: getSearchEsListItemMock(), + _type: '', + matched_queries: ['1.0'], + }, + ]; + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [VALUE, VALUE], + }); + const expected: SearchListItemArraySchema = [ + getSearchListItemResponseMock(), + getSearchListItemResponseMock(), + ]; + expect(queryFilter).toEqual(expected); + }); + + test('it transforms only 1 elastic type to a search list item type if only 1 is found as a value', () => { + const response = getSearchListItemMock(); + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [VALUE, '127.0.0.2'], + }); + const expected: SearchListItemArraySchema = [ + getSearchListItemResponseMock(), + { items: [], value: '127.0.0.2' }, + ]; + expect(queryFilter).toEqual(expected); + }); + + test('it attaches two found results if the value is found in two hits from Elastic Search', () => { + const response = getSearchListItemMock(); + response.hits.hits = [ + ...response.hits.hits, + { + _id: LIST_ITEM_ID, + _index: LIST_INDEX, + _score: 0, + _source: getSearchEsListItemMock(), + _type: '', + matched_queries: ['0.0'], + }, + ]; + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [VALUE], + }); + const { + items: [firstItem], + value, + } = getSearchListItemResponseMock(); + const expected: SearchListItemArraySchema = [ + { + items: [firstItem, firstItem], + value, + }, + ]; + expect(queryFilter).toEqual(expected); + }); + + test('it will return an empty array if no values are passed in', () => { + const response = getSearchListItemMock(); + response.hits.hits = [ + ...response.hits.hits, + { + _id: LIST_ITEM_ID, + _index: LIST_INDEX, + _score: 0, + _source: getSearchEsListItemMock(), + _type: '', + matched_queries: ['1.0'], + }, + ]; + const queryFilter = transformElasticNamedSearchToListItem({ + response, + type: TYPE, + value: [], + }); + expect(queryFilter).toEqual([]); + }); +}); diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts new file mode 100644 index 000000000000..0326d22aa843 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { SearchEsListItemSchema, SearchListItemArraySchema, Type } from '../../../common/schemas'; + +import { transformElasticHitsToListItem } from './transform_elastic_to_list_item'; + +export interface TransformElasticMSearchToListItemOptions { + response: SearchResponse; + type: Type; + value: unknown[]; +} + +/** + * Given an Elasticsearch response this will look to see if the named query matches the + * index found. The named query will have to be in the format of, "1.0", "1.1", "2.0" where the + * major number "1,2,n" will match with the index. + * Ref: https://www.elastic.co/guide/en/elasticsearch//reference/7.9/query-dsl-bool-query.html#named-queries + * @param response The elastic response + * @param type The list type + * @param value The values to check against the named queries. + */ +export const transformElasticNamedSearchToListItem = ({ + response, + type, + value, +}: TransformElasticMSearchToListItemOptions): SearchListItemArraySchema => { + return value.map((singleValue, index) => { + const matchingHits = response.hits.hits.filter((hit) => { + if (hit.matched_queries != null) { + return hit.matched_queries.some((matchedQuery) => { + const [matchedQueryIndex] = matchedQuery.split('.'); + return matchedQueryIndex === `${index}`; + }); + } else { + return false; + } + }); + const items = transformElasticHitsToListItem({ hits: matchingHits, type }); + return { + items, + value: singleValue, + }; + }); +}; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts index 8a5554c3865c..09e5ecd74b0d 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts @@ -8,7 +8,10 @@ import { getSearchListItemMock } from '../../../common/schemas/elastic_response/ import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; import { ListItemArraySchema } from '../../../common/schemas'; -import { transformElasticToListItem } from './transform_elastic_to_list_item'; +import { + transformElasticHitsToListItem, + transformElasticToListItem, +} from './transform_elastic_to_list_item'; describe('transform_elastic_to_list_item', () => { beforeEach(() => { @@ -19,28 +22,61 @@ describe('transform_elastic_to_list_item', () => { jest.clearAllMocks(); }); - test('it transforms an elastic type to a list item type', () => { - const response = getSearchListItemMock(); - const queryFilter = transformElasticToListItem({ - response, - type: 'ip', + describe('transformElasticToListItem', () => { + test('it transforms an elastic type to a list item type', () => { + const response = getSearchListItemMock(); + const queryFilter = transformElasticToListItem({ + response, + type: 'ip', + }); + const expected: ListItemArraySchema = [getListItemResponseMock()]; + expect(queryFilter).toEqual(expected); + }); + + test('it transforms an elastic keyword type to a list item type', () => { + const response = getSearchListItemMock(); + response.hits.hits[0]._source.ip = undefined; + response.hits.hits[0]._source.keyword = 'host-name-example'; + const queryFilter = transformElasticToListItem({ + response, + type: 'keyword', + }); + const listItemResponse = getListItemResponseMock(); + listItemResponse.type = 'keyword'; + listItemResponse.value = 'host-name-example'; + const expected: ListItemArraySchema = [listItemResponse]; + expect(queryFilter).toEqual(expected); }); - const expected: ListItemArraySchema = [getListItemResponseMock()]; - expect(queryFilter).toEqual(expected); }); - test('it transforms an elastic keyword type to a list item type', () => { - const response = getSearchListItemMock(); - response.hits.hits[0]._source.ip = undefined; - response.hits.hits[0]._source.keyword = 'host-name-example'; - const queryFilter = transformElasticToListItem({ - response, - type: 'keyword', + describe('transformElasticHitsToListItem', () => { + test('it transforms an elastic type to a list item type', () => { + const { + hits: { hits }, + } = getSearchListItemMock(); + const queryFilter = transformElasticHitsToListItem({ + hits, + type: 'ip', + }); + const expected: ListItemArraySchema = [getListItemResponseMock()]; + expect(queryFilter).toEqual(expected); + }); + + test('it transforms an elastic keyword type to a list item type', () => { + const { + hits: { hits }, + } = getSearchListItemMock(); + hits[0]._source.ip = undefined; + hits[0]._source.keyword = 'host-name-example'; + const queryFilter = transformElasticHitsToListItem({ + hits, + type: 'keyword', + }); + const listItemResponse = getListItemResponseMock(); + listItemResponse.type = 'keyword'; + listItemResponse.value = 'host-name-example'; + const expected: ListItemArraySchema = [listItemResponse]; + expect(queryFilter).toEqual(expected); }); - const listItemResponse = getListItemResponseMock(); - listItemResponse.type = 'keyword'; - listItemResponse.value = 'host-name-example'; - const expected: ListItemArraySchema = [listItemResponse]; - expect(queryFilter).toEqual(expected); }); }); diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts index 14794870bf67..db16f213adec 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts @@ -17,11 +17,23 @@ export interface TransformElasticToListItemOptions { type: Type; } +export interface TransformElasticHitToListItemOptions { + hits: SearchResponse['hits']['hits']; + type: Type; +} + export const transformElasticToListItem = ({ response, type, }: TransformElasticToListItemOptions): ListItemArraySchema => { - return response.hits.hits.map((hit) => { + return transformElasticHitsToListItem({ hits: response.hits.hits, type }); +}; + +export const transformElasticHitsToListItem = ({ + hits, + type, +}: TransformElasticHitToListItemOptions): ListItemArraySchema => { + return hits.map((hit) => { const { _id, _source: { diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx index 28a73801c8c0..c82ef392ce3d 100644 --- a/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx +++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx @@ -9,7 +9,7 @@ import { EuiFormRow, EuiComboBoxOptionOption, EuiComboBox } from '@elastic/eui'; import { IFieldType } from '../../../../../../../src/plugins/data/common'; import { useFindLists, ListSchema } from '../../../lists_plugin_deps'; import { useKibana } from '../../../common/lib/kibana'; -import { getGenericComboBoxProps } from './helpers'; +import { filterFieldToList, getGenericComboBoxProps } from './helpers'; import * as i18n from './translations'; interface AutocompleteFieldListsProps { @@ -41,17 +41,10 @@ export const AutocompleteFieldListsComponent: React.FC name, []); - const optionsMemo = useMemo(() => { - if ( - selectedField != null && - selectedField.esTypes != null && - selectedField.esTypes.length > 0 - ) { - return lists.filter(({ type }) => selectedField.esTypes?.includes(type)); - } else { - return []; - } - }, [lists, selectedField]); + const optionsMemo = useMemo(() => filterFieldToList(lists, selectedField), [ + lists, + selectedField, + ]); const selectedOptionsMemo = useMemo(() => { if (selectedValue != null) { const list = lists.filter(({ id }) => id === selectedValue); diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts index f78740f76420..abbeec2b64d7 100644 --- a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts @@ -6,6 +6,7 @@ import moment from 'moment'; import '../../../common/mock/match_media'; import { getField } from '../../../../../../../src/plugins/data/common/index_patterns/fields/fields.mocks'; +import { IFieldType } from '../../../../../../../src/plugins/data/common'; import * as i18n from './translations'; import { @@ -15,7 +16,16 @@ import { existsOperator, doesNotExistOperator, } from './operators'; -import { getOperators, checkEmptyValue, paramIsValid, getGenericComboBoxProps } from './helpers'; +import { + getOperators, + checkEmptyValue, + paramIsValid, + getGenericComboBoxProps, + typeMatch, + filterFieldToList, +} from './helpers'; +import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock'; +import { ListSchema } from '../../../../../lists/common'; describe('helpers', () => { // @ts-ignore @@ -260,4 +270,117 @@ describe('helpers', () => { }); }); }); + + describe('#typeMatch', () => { + test('ip -> ip is true', () => { + expect(typeMatch('ip', 'ip')).toEqual(true); + }); + + test('keyword -> keyword is true', () => { + expect(typeMatch('keyword', 'keyword')).toEqual(true); + }); + + test('text -> text is true', () => { + expect(typeMatch('text', 'text')).toEqual(true); + }); + + test('ip_range -> ip is true', () => { + expect(typeMatch('ip_range', 'ip')).toEqual(true); + }); + + test('date_range -> date is true', () => { + expect(typeMatch('date_range', 'date')).toEqual(true); + }); + + test('double_range -> double is true', () => { + expect(typeMatch('double_range', 'double')).toEqual(true); + }); + + test('float_range -> float is true', () => { + expect(typeMatch('float_range', 'float')).toEqual(true); + }); + + test('integer_range -> integer is true', () => { + expect(typeMatch('integer_range', 'integer')).toEqual(true); + }); + + test('long_range -> long is true', () => { + expect(typeMatch('long_range', 'long')).toEqual(true); + }); + + test('ip -> date is false', () => { + expect(typeMatch('ip', 'date')).toEqual(false); + }); + + test('long -> float is false', () => { + expect(typeMatch('long', 'float')).toEqual(false); + }); + + test('integer -> long is false', () => { + expect(typeMatch('integer', 'long')).toEqual(false); + }); + }); + + describe('#filterFieldToList', () => { + test('it returns empty array if given a undefined for field', () => { + const filter = filterFieldToList([], undefined); + expect(filter).toEqual([]); + }); + + test('it returns empty array if filed does not contain esTypes', () => { + const field: IFieldType = { name: 'some-name', type: 'some-type' }; + const filter = filterFieldToList([], field); + expect(filter).toEqual([]); + }); + + test('it returns single filtered list of ip_range -> ip', () => { + const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] }; + const listItem: ListSchema = { ...getListResponseMock(), type: 'ip_range' }; + const filter = filterFieldToList([listItem], field); + const expected: ListSchema[] = [listItem]; + expect(filter).toEqual(expected); + }); + + test('it returns single filtered list of ip -> ip', () => { + const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] }; + const listItem: ListSchema = { ...getListResponseMock(), type: 'ip' }; + const filter = filterFieldToList([listItem], field); + const expected: ListSchema[] = [listItem]; + expect(filter).toEqual(expected); + }); + + test('it returns single filtered list of keyword -> keyword', () => { + const field: IFieldType = { name: 'some-name', type: 'keyword', esTypes: ['keyword'] }; + const listItem: ListSchema = { ...getListResponseMock(), type: 'keyword' }; + const filter = filterFieldToList([listItem], field); + const expected: ListSchema[] = [listItem]; + expect(filter).toEqual(expected); + }); + + test('it returns single filtered list of text -> text', () => { + const field: IFieldType = { name: 'some-name', type: 'text', esTypes: ['text'] }; + const listItem: ListSchema = { ...getListResponseMock(), type: 'text' }; + const filter = filterFieldToList([listItem], field); + const expected: ListSchema[] = [listItem]; + expect(filter).toEqual(expected); + }); + + test('it returns 2 filtered lists of ip_range -> ip', () => { + const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] }; + const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' }; + const listItem2: ListSchema = { ...getListResponseMock(), type: 'ip_range' }; + const filter = filterFieldToList([listItem1, listItem2], field); + const expected: ListSchema[] = [listItem1, listItem2]; + expect(filter).toEqual(expected); + }); + + test('it returns 1 filtered lists of ip_range -> ip if the 2nd is not compatible type', () => { + const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] }; + const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' }; + const listItem2: ListSchema = { ...getListResponseMock(), type: 'text' }; + const filter = filterFieldToList([listItem1, listItem2], field); + const expected: ListSchema[] = [listItem1]; + expect(filter).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts index 1ad296e0299b..44e5adde6565 100644 --- a/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts +++ b/x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts @@ -18,6 +18,7 @@ import { } from './operators'; import { GetGenericComboBoxPropsReturn, OperatorOption } from './types'; import * as i18n from './translations'; +import { ListSchema, Type } from '../../../lists_plugin_deps'; /** * Returns the appropriate operators given a field type @@ -138,3 +139,36 @@ export function getGenericComboBoxProps({ selectedComboOptions: newSelectedComboOptions, }; } + +/** + * Given an array of lists and optionally a field this will return all + * the lists that match against the field based on the types from the field + * @param lists The lists to match against the field + * @param field The field to check against the list to see if they are compatible + */ +export const filterFieldToList = (lists: ListSchema[], field?: IFieldType): ListSchema[] => { + if (field != null) { + const { esTypes = [] } = field; + return lists.filter(({ type }) => esTypes.some((esType) => typeMatch(type, esType))); + } else { + return []; + } +}; + +/** + * Given an input list type and a string based ES type this will match + * if they're exact or if they are compatible with a range + * @param type The type to match against the esType + * @param esType The ES type to match with + */ +export const typeMatch = (type: Type, esType: string): boolean => { + return ( + type === esType || + (type === 'ip_range' && esType === 'ip') || + (type === 'date_range' && esType === 'date') || + (type === 'double_range' && esType === 'double') || + (type === 'float_range' && esType === 'float') || + (type === 'integer_range' && esType === 'integer') || + (type === 'long_range' && esType === 'long') + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts deleted file mode 100644 index 1c13de16d9b1..000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { get } from 'lodash/fp'; -import { Logger } from 'src/core/server'; - -import { ListClient } from '../../../../../lists/server'; -import { BuildRuleMessage } from './rule_messages'; -import { - EntryList, - ExceptionListItemSchema, - entriesList, - Type, -} from '../../../../../lists/common/schemas'; -import { hasLargeValueList } from '../../../../common/detection_engine/utils'; -import { SearchTypes } from '../../../../common/detection_engine/types'; -import { SearchResponse } from '../../types'; - -// narrow unioned type to be single -const isStringableType = (val: SearchTypes): val is string | number | boolean => - ['string', 'number', 'boolean'].includes(typeof val); - -const isStringableArray = (val: SearchTypes): val is Array => { - if (!Array.isArray(val)) { - return false; - } - // TS does not allow .every to be called on val as-is, even though every type in the union - // is an array. https://github.com/microsoft/TypeScript/issues/36390 - // @ts-expect-error - return val.every((subVal) => isStringableType(subVal)); -}; - -export const createSetToFilterAgainst = async ({ - events, - field, - listId, - listType, - listClient, - logger, - buildRuleMessage, -}: { - events: SearchResponse['hits']['hits']; - field: string; - listId: string; - listType: Type; - listClient: ListClient; - logger: Logger; - buildRuleMessage: BuildRuleMessage; -}): Promise> => { - const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => { - const valueField = get(field, searchResultItem._source); - if (valueField != null) { - if (isStringableType(valueField)) { - acc.add(valueField.toString()); - } else if (isStringableArray(valueField)) { - valueField.forEach((subVal) => acc.add(subVal.toString())); - } - } - return acc; - }, new Set()); - logger.debug( - `number of distinct values from ${field}: ${[...valuesFromSearchResultField].length}` - ); - - // matched will contain any list items that matched with the - // values passed in from the Set. - const matchedListItems = await listClient.getListItemByValues({ - listId, - type: listType, - value: [...valuesFromSearchResultField], - }); - - logger.debug(`number of matched items from list with id ${listId}: ${matchedListItems.length}`); - // create a set of list values that were a hit - easier to work with - const matchedListItemsSet = new Set(matchedListItems.map((item) => item.value)); - return matchedListItemsSet; -}; - -export const filterEventsAgainstList = async ({ - listClient, - exceptionsList, - logger, - eventSearchResult, - buildRuleMessage, -}: { - listClient: ListClient; - exceptionsList: ExceptionListItemSchema[]; - logger: Logger; - eventSearchResult: SearchResponse; - buildRuleMessage: BuildRuleMessage; -}): Promise> => { - try { - if (exceptionsList == null || exceptionsList.length === 0) { - logger.debug(buildRuleMessage('about to return original search result')); - return eventSearchResult; - } - - const exceptionItemsWithLargeValueLists = exceptionsList.reduce( - (acc, exception) => { - const { entries } = exception; - if (hasLargeValueList(entries)) { - return [...acc, exception]; - } - - return acc; - }, - [] - ); - - if (exceptionItemsWithLargeValueLists.length === 0) { - logger.debug( - buildRuleMessage('no exception items of type list found - returning original search result') - ); - return eventSearchResult; - } - - const valueListExceptionItems = exceptionsList.filter((listItem: ExceptionListItemSchema) => { - return listItem.entries.every((entry) => entriesList.is(entry)); - }); - - // now that we have all the exception items which are value lists (whether single entry or have multiple entries) - const res = await valueListExceptionItems.reduce['hits']['hits']>>( - async ( - filteredAccum: Promise['hits']['hits']>, - exceptionItem: ExceptionListItemSchema - ) => { - // 1. acquire the values from the specified fields to check - // e.g. if the value list is checking against source.ip, gather - // all the values for source.ip from the search response events. - - // 2. search against the value list with the values found in the search result - // and see if there are any matches. For every match, add that value to a set - // that represents the "matched" values - - // 3. filter the search result against the set from step 2 using the - // given operator (included vs excluded). - // acquire the list values we are checking for in the field. - const filtered = await filteredAccum; - const typedEntries = exceptionItem.entries.filter((entry): entry is EntryList => - entriesList.is(entry) - ); - const fieldAndSetTuples = await Promise.all( - typedEntries.map(async (entry) => { - const { list, field, operator } = entry; - const { id, type } = list; - const matchedSet = await createSetToFilterAgainst({ - events: filtered, - field, - listId: id, - listType: type, - listClient, - logger, - buildRuleMessage, - }); - - return Promise.resolve({ field, operator, matchedSet }); - }) - ); - - // check if for each tuple, the entry is not in both for when two value list entries exist. - // need to re-write this as a reduce. - const filteredEvents = filtered.filter((item) => { - const vals = fieldAndSetTuples.map((tuple) => { - const eventItem = get(tuple.field, item._source); - if (tuple.operator === 'included') { - // only create a signal if the field value is not in the value list - if (eventItem != null) { - if (isStringableType(eventItem)) { - return !tuple.matchedSet.has(eventItem); - } else if (isStringableArray(eventItem)) { - return !eventItem.some((val) => tuple.matchedSet.has(val)); - } - } - return true; - } else if (tuple.operator === 'excluded') { - // only create a signal if the field value is in the value list - if (eventItem != null) { - if (isStringableType(eventItem)) { - return tuple.matchedSet.has(eventItem); - } else if (isStringableArray(eventItem)) { - return eventItem.some((val) => tuple.matchedSet.has(val)); - } - } - return true; - } - return false; - }); - return vals.some((value) => value); - }); - const diff = eventSearchResult.hits.hits.length - filteredEvents.length; - logger.debug( - buildRuleMessage(`Exception with id ${exceptionItem.id} filtered out ${diff} events`) - ); - const toReturn = filteredEvents; - return toReturn; - }, - Promise.resolve['hits']['hits']>(eventSearchResult.hits.hits) - ); - - const toReturn: SearchResponse = { - took: eventSearchResult.took, - timed_out: eventSearchResult.timed_out, - _shards: eventSearchResult._shards, - hits: { - total: res.length, - max_score: eventSearchResult.hits.max_score, - hits: res, - }, - }; - return toReturn; - } catch (exc) { - throw new Error(`Failed to query lists index. Reason: ${exc.message}`); - } -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts new file mode 100644 index 000000000000..9192eeb35d0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts @@ -0,0 +1,287 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createFieldAndSetTuples } from './create_field_and_set_tuples'; +import { mockLogger, sampleDocWithSortId } from '../__mocks__/es_results'; + +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { listMock } from '../../../../../../lists/server/mocks'; +import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock'; +import { EntryList } from '../../../../../../lists/common'; +import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'; + +describe('filterEventsAgainstList', () => { + let listClient = listMock.getListClient(); + let exceptionItem = getExceptionListItemSchemaMock(); + let events = [sampleDocWithSortId('123', '1.1.1.1')]; + + beforeEach(() => { + jest.clearAllMocks(); + listClient = listMock.getListClient(); + listClient.searchListItemByValues = jest.fn(({ value }) => + Promise.resolve( + value.map((item) => ({ + ...getSearchListItemResponseMock(), + value: item, + })) + ) + ); + exceptionItem = { + ...getExceptionListItemSchemaMock(), + entries: [ + { + field: 'source.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + ], + }; + events = [sampleDocWithSortId('123', '1.1.1.1')]; + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns an empty array if exceptionItem entries are empty', async () => { + exceptionItem.entries = []; + const field = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(field).toEqual([]); + }); + + test('it returns a single field and set tuple if entries has a single item', async () => { + const field = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(field.length).toEqual(1); + }); + + test('it returns "included" if the operator is "included"', async () => { + (exceptionItem.entries[0] as EntryList).operator = 'included'; + const [{ operator }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(operator).toEqual('included'); + }); + + test('it returns "excluded" if the operator is "excluded"', async () => { + (exceptionItem.entries[0] as EntryList).operator = 'excluded'; + const [{ operator }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(operator).toEqual('excluded'); + }); + + test('it returns "field" if the "field is "source.ip"', async () => { + (exceptionItem.entries[0] as EntryList).field = 'source.ip'; + const [{ field }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(field).toEqual('source.ip'); + }); + + test('it returns a single matched set as a JSON.stringify() set from the "events"', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1')]; + (exceptionItem.entries[0] as EntryList).field = 'source.ip'; + const [{ matchedSet }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1')]); + }); + + test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + (exceptionItem.entries[0] as EntryList).field = 'source.ip'; + const [{ matchedSet }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + }); + + test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => { + events = [sampleDocWithSortId('123', ['1.1.1.1', '2.2.2.2'])]; + (exceptionItem.entries[0] as EntryList).field = 'source.ip'; + const [{ matchedSet }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1', '2.2.2.2'])]); + }); + + test('it returns 2 fields when given two exception list items', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + exceptionItem.entries = [ + { + field: 'source.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + { + field: 'destination.ip', + operator: 'excluded', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + ]; + const fields = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(fields.length).toEqual(2); + }); + + test('it returns two matched sets from two different events, one excluded, and one included', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + exceptionItem.entries = [ + { + field: 'source.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + { + field: 'destination.ip', + operator: 'excluded', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + ]; + const [{ operator: operator1 }, { operator: operator2 }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(operator1).toEqual('included'); + expect(operator2).toEqual('excluded'); + }); + + test('it returns two fields from two different events', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')]; + exceptionItem.entries = [ + { + field: 'source.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + { + field: 'destination.ip', + operator: 'excluded', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + ]; + const [{ field: field1 }, { field: field2 }] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect(field1).toEqual('source.ip'); + expect(field2).toEqual('destination.ip'); + }); + + test('it returns two matches from two different events', async () => { + events = [ + sampleDocWithSortId('123', '1.1.1.1', '3.3.3.3'), + sampleDocWithSortId('456', '2.2.2.2', '5.5.5.5'), + ]; + exceptionItem.entries = [ + { + field: 'source.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + { + field: 'destination.ip', + operator: 'excluded', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + ]; + const [ + { matchedSet: matchedSet1 }, + { matchedSet: matchedSet2 }, + ] = await createFieldAndSetTuples({ + listClient, + logger: mockLogger, + events, + exceptionItem, + buildRuleMessage, + }); + expect([...matchedSet1]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + expect([...matchedSet2]).toEqual([JSON.stringify('3.3.3.3'), JSON.stringify('5.5.5.5')]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.ts new file mode 100644 index 000000000000..d31b7a0eb613 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EntryList, entriesList } from '../../../../../../lists/common'; +import { createSetToFilterAgainst } from './create_set_to_filter_against'; +import { CreateFieldAndSetTuplesOptions, FieldSet } from './types'; + +export const createFieldAndSetTuples = async ({ + events, + exceptionItem, + listClient, + logger, + buildRuleMessage, +}: CreateFieldAndSetTuplesOptions): Promise => { + const typedEntries = exceptionItem.entries.filter((entry): entry is EntryList => + entriesList.is(entry) + ); + const fieldAndSetTuples = await Promise.all( + typedEntries.map(async (entry) => { + const { list, field, operator } = entry; + const { id, type } = list; + const matchedSet = await createSetToFilterAgainst({ + events, + field, + listId: id, + listType: type, + listClient, + logger, + buildRuleMessage, + }); + + return { field, operator, matchedSet }; + }) + ); + return fieldAndSetTuples; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts new file mode 100644 index 000000000000..0ac09713cc8a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { mockLogger, sampleDocWithSortId } from '../__mocks__/es_results'; + +import { listMock } from '../../../../../../lists/server/mocks'; +import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock'; +import { createSetToFilterAgainst } from './create_set_to_filter_against'; +import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'; + +describe('createSetToFilterAgainst', () => { + let listClient = listMock.getListClient(); + let events = [sampleDocWithSortId('123', '1.1.1.1')]; + + beforeEach(() => { + jest.clearAllMocks(); + listClient = listMock.getListClient(); + listClient.searchListItemByValues = jest.fn(({ value }) => + Promise.resolve( + value.map((item) => ({ + ...getSearchListItemResponseMock(), + value: item, + })) + ) + ); + events = [sampleDocWithSortId('123', '1.1.1.1')]; + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it returns an empty array if list return is empty', async () => { + listClient.searchListItemByValues = jest.fn().mockResolvedValue([]); + const field = await createSetToFilterAgainst({ + events, + field: 'source.ip', + listId: 'list-123', + listType: 'ip', + listClient, + logger: mockLogger, + buildRuleMessage, + }); + expect([...field]).toEqual([]); + }); + + test('it returns 1 field if the list returns a single item', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1')]; + const field = await createSetToFilterAgainst({ + events, + field: 'source.ip', + listId: 'list-123', + listType: 'ip', + listClient, + logger: mockLogger, + buildRuleMessage, + }); + expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ + listId: 'list-123', + type: 'ip', + value: ['1.1.1.1'], + }); + expect([...field]).toEqual([JSON.stringify('1.1.1.1')]); + }); + + test('it returns 2 fields if the list returns 2 items', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')]; + const field = await createSetToFilterAgainst({ + events, + field: 'source.ip', + listId: 'list-123', + listType: 'ip', + listClient, + logger: mockLogger, + buildRuleMessage, + }); + expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ + listId: 'list-123', + type: 'ip', + value: ['1.1.1.1', '2.2.2.2'], + }); + expect([...field]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]); + }); + + test('it returns 0 fields if the field does not match up to a valid field within the event', async () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')]; + const field = await createSetToFilterAgainst({ + events, + field: 'nonexistent.field', // field does not exist + listId: 'list-123', + listType: 'ip', + listClient, + logger: mockLogger, + buildRuleMessage, + }); + expect(listClient.searchListItemByValues).toHaveBeenCalledWith({ + listId: 'list-123', + type: 'ip', + value: [], + }); + expect([...field]).toEqual([]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts new file mode 100644 index 000000000000..c546654676c8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash/fp'; +import { CreateSetToFilterAgainstOptions } from './types'; + +/** + * Creates a field set to filter against using the stringed version of the + * data type for compare. Creates a set of list values that are stringified that + * are easier to work with as well as ensures that deep values can work since it turns + * things into a string using JSON.stringify(). + * + * @param events The events to filter against + * @param field The field checking against the list + * @param listId The list id for the list function call + * @param listType The type of list for the list function call + * @param listClient The list client API + * @param logger logger for errors, debug, etc... + */ +export const createSetToFilterAgainst = async ({ + events, + field, + listId, + listType, + listClient, + logger, + buildRuleMessage, +}: CreateSetToFilterAgainstOptions): Promise> => { + const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => { + const valueField = get(field, searchResultItem._source); + if (valueField != null) { + acc.add(valueField); + } + return acc; + }, new Set()); + + logger.debug( + buildRuleMessage( + `number of distinct values from ${field}: ${[...valuesFromSearchResultField].length}` + ) + ); + + const matchedListItems = await listClient.searchListItemByValues({ + listId, + type: listType, + value: [...valuesFromSearchResultField], + }); + + logger.debug( + buildRuleMessage( + `number of matched items from list with id ${listId}: ${matchedListItems.length}` + ) + ); + + return new Set( + matchedListItems + .filter((item) => item.items.length !== 0) + .map((item) => JSON.stringify(item.value)) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts new file mode 100644 index 000000000000..6a045f6694da --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { sampleDocWithSortId } from '../__mocks__/es_results'; + +import { listMock } from '../../../../../../lists/server/mocks'; +import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock'; +import { filterEvents } from './filter_events'; +import { FieldSet } from './types'; + +describe('filterEvents', () => { + let listClient = listMock.getListClient(); + let events = [sampleDocWithSortId('123', '1.1.1.1')]; + + beforeEach(() => { + jest.clearAllMocks(); + listClient = listMock.getListClient(); + listClient.searchListItemByValues = jest.fn(({ value }) => + Promise.resolve( + value.map((item) => ({ + ...getSearchListItemResponseMock(), + value: item, + })) + ) + ); + events = [sampleDocWithSortId('123', '1.1.1.1')]; + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('it filters out the event if it is "included"', () => { + events = [sampleDocWithSortId('123', '1.1.1.1')]; + const fieldAndSetTuples: FieldSet[] = [ + { + field: 'source.ip', + operator: 'included', + matchedSet: new Set([JSON.stringify('1.1.1.1')]), + }, + ]; + const field = filterEvents({ + events, + fieldAndSetTuples, + }); + expect([...field]).toEqual([]); + }); + + test('it does not filter out the event if it is "excluded"', () => { + events = [sampleDocWithSortId('123', '1.1.1.1')]; + const fieldAndSetTuples: FieldSet[] = [ + { + field: 'source.ip', + operator: 'excluded', + matchedSet: new Set([JSON.stringify('1.1.1.1')]), + }, + ]; + const field = filterEvents({ + events, + fieldAndSetTuples, + }); + expect([...field]).toEqual(events); + }); + + test('it does NOT filter out the event if the field is not found', () => { + events = [sampleDocWithSortId('123', '1.1.1.1')]; + const fieldAndSetTuples: FieldSet[] = [ + { + field: 'madeup.nonexistent', // field does not exist + operator: 'included', + matchedSet: new Set([JSON.stringify('1.1.1.1')]), + }, + ]; + const field = filterEvents({ + events, + fieldAndSetTuples, + }); + expect([...field]).toEqual(events); + }); + + test('it does NOT filter out the event if it is in both an inclusion and exclusion list', () => { + events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')]; + const fieldAndSetTuples: FieldSet[] = [ + { + field: 'source.ip', + operator: 'included', + matchedSet: new Set([JSON.stringify('1.1.1.1')]), + }, + { + field: 'source.ip', + operator: 'excluded', + matchedSet: new Set([JSON.stringify('1.1.1.1')]), + }, + ]; + + const field = filterEvents({ + events, + fieldAndSetTuples, + }); + expect([...field]).toEqual(events); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts new file mode 100644 index 000000000000..e8667510da68 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash/fp'; +import { SearchResponse } from '../../../types'; +import { FilterEventsOptions } from './types'; + +/** + * Check if for each tuple, the entry is not in both for when two or more value list entries exist. + * If the entry is in both an inclusion and exclusion list it will not be filtered out. + * @param events The events to check against + * @param fieldAndSetTuples The field and set tuples + */ +export const filterEvents = ({ + events, + fieldAndSetTuples, +}: FilterEventsOptions): SearchResponse['hits']['hits'] => { + return events.filter((item) => { + return fieldAndSetTuples + .map((tuple) => { + const eventItem = get(tuple.field, item._source); + if (eventItem == null) { + return true; + } else if (tuple.operator === 'included') { + // only create a signal if the event is not in the value list + return !tuple.matchedSet.has(JSON.stringify(eventItem)); + } else if (tuple.operator === 'excluded') { + // only create a signal if the event is in the value list + return tuple.matchedSet.has(JSON.stringify(eventItem)); + } else { + return false; + } + }) + .some((value) => value); + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts similarity index 77% rename from x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts index 01e7e7160e1a..eb6e905c0303 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts @@ -5,27 +5,27 @@ */ import uuid from 'uuid'; -import { filterEventsAgainstList } from './filter_events_with_list'; -import { buildRuleMessageFactory } from './rule_messages'; -import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results'; +import { filterEventsAgainstList } from './filter_events_against_list'; +import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'; +import { mockLogger, repeatedSearchResultsWithSortId } from '../__mocks__/es_results'; -import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock'; -import { listMock } from '../../../../../lists/server/mocks'; +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { listMock } from '../../../../../../lists/server/mocks'; +import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock'; const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4()); -const buildRuleMessage = buildRuleMessageFactory({ - id: 'fake id', - ruleId: 'fake rule id', - index: 'fakeindex', - name: 'fake name', -}); + describe('filterEventsAgainstList', () => { let listClient = listMock.getListClient(); + beforeEach(() => { jest.clearAllMocks(); listClient = listMock.getListClient(); - listClient.getListItemByValues = jest.fn().mockResolvedValue([]); + listClient.searchListItemByValues = jest.fn().mockResolvedValue([]); + }); + + afterEach(() => { + jest.clearAllMocks(); }); it('should respond with eventSearchResult if exceptionList is empty array', async () => { @@ -87,6 +87,7 @@ describe('filterEventsAgainstList', () => { }); expect(res.hits.hits.length).toEqual(4); }); + it('should respond with less items in the list if some values match', async () => { const exceptionItem = getExceptionListItemSchemaMock(); exceptionItem.entries = [ @@ -100,10 +101,10 @@ describe('filterEventsAgainstList', () => { }, }, ]; - listClient.getListItemByValues = jest.fn(({ value }) => + listClient.searchListItemByValues = jest.fn(({ value }) => Promise.resolve( value.slice(0, 2).map((item) => ({ - ...getListItemResponseMock(), + ...getSearchListItemResponseMock(), value: item, })) ) @@ -120,8 +121,8 @@ describe('filterEventsAgainstList', () => { ]), buildRuleMessage, }); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip'); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual( + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip'); + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual( 'ci-badguys.txt' ); expect(res.hits.hits.length).toEqual(2); @@ -159,13 +160,13 @@ describe('filterEventsAgainstList', () => { ]; // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '2.2.2.2' }, - { ...getListItemResponseMock(), value: '4.4.4.4' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: '4.4.4.4' }, ]); // this call represents an exception list with a value list containing ['6.6.6.6'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '6.6.6.6' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: '6.6.6.6' }, ]); const res = await filterEventsAgainstList({ @@ -185,7 +186,7 @@ describe('filterEventsAgainstList', () => { ]), buildRuleMessage, }); - expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); expect(res.hits.hits.length).toEqual(6); // @ts-expect-error @@ -221,12 +222,12 @@ describe('filterEventsAgainstList', () => { ]; // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '2.2.2.2' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, ]); // this call represents an exception list with a value list containing ['6.6.6.6'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '6.6.6.6' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: '6.6.6.6' }, ]); const res = await filterEventsAgainstList({ @@ -246,7 +247,7 @@ describe('filterEventsAgainstList', () => { ]), buildRuleMessage, }); - expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); // @ts-expect-error const ipVals = res.hits.hits.map((item) => item._source.source.ip); expect(res.hits.hits.length).toEqual(7); @@ -280,12 +281,12 @@ describe('filterEventsAgainstList', () => { ]; // this call represents an exception list with a value list containing ['2.2.2.2'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '2.2.2.2' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, ]); // this call represents an exception list with a value list containing ['4.4.4.4'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '4.4.4.4' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: '4.4.4.4' }, ]); const res = await filterEventsAgainstList({ @@ -321,7 +322,7 @@ describe('filterEventsAgainstList', () => { ), buildRuleMessage, }); - expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); expect(res.hits.hits.length).toEqual(8); // @ts-expect-error @@ -362,8 +363,8 @@ describe('filterEventsAgainstList', () => { ]; // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValue([ - { ...getListItemResponseMock(), value: '2.2.2.2' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValue([ + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, ]); const res = await filterEventsAgainstList({ @@ -383,7 +384,7 @@ describe('filterEventsAgainstList', () => { ]), buildRuleMessage, }); - expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); expect(res.hits.hits.length).toEqual(9); // @ts-expect-error @@ -401,7 +402,7 @@ describe('filterEventsAgainstList', () => { ]).toEqual(ipVals); }); - it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => { + it('should respond with same items in the list given one exception item with two entries of type list and array of values in document', async () => { const exceptionItem = getExceptionListItemSchemaMock(); exceptionItem.entries = [ { @@ -425,12 +426,12 @@ describe('filterEventsAgainstList', () => { ]; // this call represents an exception list with a value list containing ['2.2.2.2'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '2.2.2.2' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: ['2.2.2.2', '3.3.3.3'] }, ]); // this call represents an exception list with a value list containing ['4.4.4.4'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '4.4.4.4' }, + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: ['3.3.3.3', '4.4.4.4'] }, ]); const res = await filterEventsAgainstList({ @@ -454,17 +455,16 @@ describe('filterEventsAgainstList', () => { ), buildRuleMessage, }); - expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([ - '1.1.1.1', - '2.2.2.2', - '3.3.3.3', + expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([ + ['1.1.1.1', '1.1.1.1'], + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], ]); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([ - '1.1.1.1', - '2.2.2.2', - '3.3.3.3', - '4.4.4.4', + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([ + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ['3.3.3.3', '4.4.4.4'], ]); expect(res.hits.hits.length).toEqual(2); @@ -505,6 +505,7 @@ describe('filterEventsAgainstList', () => { }); expect(res.hits.hits.length).toEqual(0); }); + it('should respond with less items in the list if some values match', async () => { const exceptionItem = getExceptionListItemSchemaMock(); exceptionItem.entries = [ @@ -518,10 +519,10 @@ describe('filterEventsAgainstList', () => { }, }, ]; - listClient.getListItemByValues = jest.fn(({ value }) => + listClient.searchListItemByValues = jest.fn(({ value }) => Promise.resolve( value.slice(0, 2).map((item) => ({ - ...getListItemResponseMock(), + ...getSearchListItemResponseMock(), value: item, })) ) @@ -538,14 +539,14 @@ describe('filterEventsAgainstList', () => { ]), buildRuleMessage, }); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip'); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual( + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip'); + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual( 'ci-badguys.txt' ); expect(res.hits.hits.length).toEqual(2); }); - it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => { + it('should respond with the same items in the list given one exception item with two entries of type list and array of values in document', async () => { const exceptionItem = getExceptionListItemSchemaMock(); exceptionItem.entries = [ { @@ -568,13 +569,16 @@ describe('filterEventsAgainstList', () => { }, ]; - // this call represents an exception list with a value list containing ['2.2.2.2'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '2.2.2.2' }, + // this call represents an exception list with a value list containing ['2.2.2.2', '3.3.3.3'] + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { + ...getSearchListItemResponseMock(), + value: ['1.1.1.1', '2.2.2.2'], + }, ]); - // this call represents an exception list with a value list containing ['4.4.4.4'] - (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ - { ...getListItemResponseMock(), value: '4.4.4.4' }, + // this call represents an exception list with a value list containing ['3.3.3.3', '4.4.4.4'] + (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getSearchListItemResponseMock(), value: ['3.3.3.3', '4.4.4.4'] }, ]); const res = await filterEventsAgainstList({ @@ -598,17 +602,16 @@ describe('filterEventsAgainstList', () => { ), buildRuleMessage, }); - expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([ - '1.1.1.1', - '2.2.2.2', - '3.3.3.3', + expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([ + ['1.1.1.1', '1.1.1.1'], + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], ]); - expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([ - '1.1.1.1', - '2.2.2.2', - '3.3.3.3', - '4.4.4.4', + expect((listClient.searchListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([ + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ['3.3.3.3', '4.4.4.4'], ]); expect(res.hits.hits.length).toEqual(2); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.ts new file mode 100644 index 000000000000..e6c20713afd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { ExceptionListItemSchema, entriesList } from '../../../../../../lists/common/schemas'; +import { hasLargeValueList } from '../../../../../common/detection_engine/utils'; +import { FilterEventsAgainstListOptions } from './types'; +import { filterEvents } from './filter_events'; +import { createFieldAndSetTuples } from './create_field_and_set_tuples'; +import { SearchResponse } from '../../../types'; + +/** + * Filters events against a large value based list. It does this through these + * steps below. + * + * 1. acquire the values from the specified fields to check + * e.g. if the value list is checking against source.ip, gather + * all the values for source.ip from the search response events. + * + * 2. search against the value list with the values found in the search result + * and see if there are any matches. For every match, add that value to a set + * that represents the "matched" values + * + * 3. filter the search result against the set from step 2 using the + * given operator (included vs excluded). + * acquire the list values we are checking for in the field. + * + * @param listClient The list client to use for queries + * @param exceptionsList The exception list + * @param logger Logger for messages + * @param eventSearchResult The current events from the search + */ +export const filterEventsAgainstList = async ({ + listClient, + exceptionsList, + logger, + eventSearchResult, + buildRuleMessage, +}: FilterEventsAgainstListOptions): Promise> => { + try { + const atLeastOneLargeValueList = exceptionsList.some(({ entries }) => + hasLargeValueList(entries) + ); + + if (!atLeastOneLargeValueList) { + logger.debug( + buildRuleMessage('no exception items of type list found - returning original search result') + ); + return eventSearchResult; + } + + const valueListExceptionItems = exceptionsList.filter((listItem: ExceptionListItemSchema) => { + return listItem.entries.every((entry) => entriesList.is(entry)); + }); + + const res = await valueListExceptionItems.reduce['hits']['hits']>>( + async ( + filteredAccum: Promise['hits']['hits']>, + exceptionItem: ExceptionListItemSchema + ) => { + const events = await filteredAccum; + const fieldAndSetTuples = await createFieldAndSetTuples({ + events, + exceptionItem, + listClient, + logger, + buildRuleMessage, + }); + const filteredEvents = filterEvents({ events, fieldAndSetTuples }); + const diff = eventSearchResult.hits.hits.length - filteredEvents.length; + logger.debug( + buildRuleMessage(`Exception with id ${exceptionItem.id} filtered out ${diff} events`) + ); + return filteredEvents; + }, + Promise.resolve['hits']['hits']>(eventSearchResult.hits.hits) + ); + + return { + took: eventSearchResult.took, + timed_out: eventSearchResult.timed_out, + _shards: eventSearchResult._shards, + hits: { + total: res.length, + max_score: eventSearchResult.hits.max_score, + hits: res, + }, + }; + } catch (exc) { + throw new Error(`Failed to query large value based lists index. Reason: ${exc.message}`); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/types.ts new file mode 100644 index 000000000000..673719d87dcd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'src/core/server'; + +import { ListClient } from '../../../../../../lists/server'; +import { BuildRuleMessage } from '../rule_messages'; +import { ExceptionListItemSchema, Type } from '../../../../../../lists/common/schemas'; +import { SearchResponse } from '../../../types'; + +export interface FilterEventsAgainstListOptions { + listClient: ListClient; + exceptionsList: ExceptionListItemSchema[]; + logger: Logger; + eventSearchResult: SearchResponse; + buildRuleMessage: BuildRuleMessage; +} + +export interface CreateSetToFilterAgainstOptions { + events: SearchResponse['hits']['hits']; + field: string; + listId: string; + listType: Type; + listClient: ListClient; + logger: Logger; + buildRuleMessage: BuildRuleMessage; +} + +export interface FilterEventsOptions { + events: SearchResponse['hits']['hits']; + fieldAndSetTuples: FieldSet[]; +} + +export interface CreateFieldAndSetTuplesOptions { + events: SearchResponse['hits']['hits']; + exceptionItem: ExceptionListItemSchema; + listClient: ListClient; + logger: Logger; + buildRuleMessage: BuildRuleMessage; +} + +export interface FieldSet { + field: string; + operator: 'excluded' | 'included'; + matchedSet: Set; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_messages.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_messages.mock.ts new file mode 100644 index 000000000000..9478ed18d472 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_messages.mock.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { buildRuleMessageFactory } from './rule_messages'; + +export const buildRuleMessageMock = buildRuleMessageFactory({ + id: 'fake id', + ruleId: 'fake rule id', + index: 'fakeindex', + name: 'fake name', +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index c82c1fe969ee..46722c69e53e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -19,10 +19,11 @@ import { buildRuleMessageFactory } from './rule_messages'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; import uuid from 'uuid'; -import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock'; import { listMock } from '../../../../../lists/server/mocks'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { BulkResponse } from './types'; +import { SearchListItemArraySchema } from '../../../../../lists/common/schemas'; +import { getSearchListItemResponseMock } from '../../../../../lists/common/schemas/response/search_list_item_schema.mock'; const buildRuleMessage = buildRuleMessageFactory({ id: 'fake id', @@ -39,7 +40,7 @@ describe('searchAfterAndBulkCreate', () => { beforeEach(() => { jest.clearAllMocks(); listClient = listMock.getListClient(); - listClient.getListItemByValues = jest.fn().mockResolvedValue([]); + listClient.searchListItemByValues = jest.fn().mockResolvedValue([]); inputIndexPattern = ['auditbeat-*']; mockService = alertsMock.createAlertServices(); }); @@ -362,9 +363,12 @@ describe('searchAfterAndBulkCreate', () => { }); test('should return success when all search results are in the allowlist and with sortId present', async () => { - listClient.getListItemByValues = jest - .fn() - .mockResolvedValue([{ value: '1.1.1.1' }, { value: '2.2.2.2' }, { value: '3.3.3.3' }]); + const searchListItems: SearchListItemArraySchema = [ + { ...getSearchListItemResponseMock(), value: '1.1.1.1' }, + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: '3.3.3.3' }, + ]; + listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); const sampleParams = sampleRuleAlertParams(30); mockService.callCluster .mockResolvedValueOnce( @@ -423,9 +427,14 @@ describe('searchAfterAndBulkCreate', () => { }); test('should return success when all search results are in the allowlist and no sortId present', async () => { - listClient.getListItemByValues = jest - .fn() - .mockResolvedValue([{ value: '1.1.1.1' }, { value: '2.2.2.2' }, { value: '3.3.3.3' }]); + const searchListItems: SearchListItemArraySchema = [ + { ...getSearchListItemResponseMock(), value: '1.1.1.1' }, + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + { ...getSearchListItemResponseMock(), value: '2.2.2.2' }, + ]; + + listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems); const sampleParams = sampleRuleAlertParams(30); mockService.callCluster.mockResolvedValueOnce( repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3), [ @@ -605,10 +614,10 @@ describe('searchAfterAndBulkCreate', () => { }) .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits()); - listClient.getListItemByValues = jest.fn(({ value }) => + listClient.searchListItemByValues = jest.fn(({ value }) => Promise.resolve( value.slice(0, 2).map((item) => ({ - ...getListItemResponseMock(), + ...getSearchListItemResponseMock(), value: item, })) ) @@ -711,10 +720,10 @@ describe('searchAfterAndBulkCreate', () => { ]; const sampleParams = sampleRuleAlertParams(30); mockService.callCluster.mockResolvedValueOnce(sampleEmptyDocSearchResults()); - listClient.getListItemByValues = jest.fn(({ value }) => + listClient.searchListItemByValues = jest.fn(({ value }) => Promise.resolve( value.slice(0, 2).map((item) => ({ - ...getListItemResponseMock(), + ...getSearchListItemResponseMock(), value: item, })) ) @@ -771,10 +780,10 @@ describe('searchAfterAndBulkCreate', () => { .mockImplementation(() => { throw Error('Fake Error'); // throws the exception we are testing }); - listClient.getListItemByValues = jest.fn(({ value }) => + listClient.searchListItemByValues = jest.fn(({ value }) => Promise.resolve( value.slice(0, 2).map((item) => ({ - ...getListItemResponseMock(), + ...getSearchListItemResponseMock(), value: item, })) ) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 0e6ddbc766fa..32865e117cba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -6,7 +6,7 @@ import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; -import { filterEventsAgainstList } from './filter_events_with_list'; +import { filterEventsAgainstList } from './filters/filter_events_against_list'; import { sendAlertTelemetryEvents } from './send_telemetry_events'; import { createSearchAfterReturnType, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 7965a09efefa..6be4a83d237a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -66,7 +66,7 @@ import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk import { createThreatSignals } from './threat_mapping/create_threat_signals'; import { getIndexVersion } from '../routes/index/get_index_version'; import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template'; -import { filterEventsAgainstList } from './filter_events_with_list'; +import { filterEventsAgainstList } from './filters/filter_events_against_list'; import { isOutdated } from '../migrations/helpers'; export const signalRulesAlertType = ({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts index e29487880de6..a5793489cd8d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts @@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => { }); describe('"is in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a double against an index that has the doubles stored as real doubles. - describe.skip('working against double values in the data set', () => { + describe('working against double values in the data set', () => { it('will return 3 results if we have a list that includes 1 double', async () => { await importFile(supertest, 'double', ['1.0'], 'list_items.txt'); const rule = getRuleForSignalTesting(['double']); @@ -545,17 +543,19 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql([]); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 1 result if we have a list which contains the double range of 1.0-1.2', async () => { - await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt'); + it('will return 1 result if we have a list which contains the double range of 1.0-1.2', async () => { + await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt', [ + '1.0', + '1.2', + ]); const rule = getRuleForSignalTesting(['double_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'double', list: { id: 'list_items.txt', - type: 'ip', + type: 'double_range', }, operator: 'included', type: 'list', @@ -565,16 +565,14 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess(supertest, id); await waitForSignalsToBePresent(supertest, 1, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.double).sort(); expect(hits).to.eql(['1.3']); }); }); }); describe('"is not in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a double against an index that has the doubles stored as real doubles. - describe.skip('working against double values in the data set', () => { + describe('working against double values in the data set', () => { it('will return 1 result if we have a list that excludes 1 double', async () => { await importFile(supertest, 'double', ['1.0'], 'list_items.txt'); const rule = getRuleForSignalTesting(['double']); @@ -715,17 +713,19 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql(['1.0', '1.1', '1.2', '1.3']); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 3 results if we have a list which contains the double range of 1.0-1.2', async () => { - await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt'); + it('will return 3 results if we have a list which contains the double range of 1.0-1.2', async () => { + await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt', [ + '1.0', + '1.2', + ]); const rule = getRuleForSignalTesting(['double_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'double', list: { id: 'list_items.txt', - type: 'ip', + type: 'double_range', }, operator: 'excluded', type: 'list', @@ -733,9 +733,9 @@ export default ({ getService }: FtrProviderContext) => { ], ]); await waitForRuleSuccess(supertest, id); - await waitForSignalsToBePresent(supertest, 1, [id]); + await waitForSignalsToBePresent(supertest, 3, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.double).sort(); expect(hits).to.eql(['1.0', '1.1', '1.2']); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts index d68f0f6a6927..955d27c08646 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts @@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => { }); describe('"is in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a float against an index that has the floats stored as real floats. - describe.skip('working against float values in the data set', () => { + describe('working against float values in the data set', () => { it('will return 3 results if we have a list that includes 1 float', async () => { await importFile(supertest, 'float', ['1.0'], 'list_items.txt'); const rule = getRuleForSignalTesting(['float']); @@ -545,17 +543,16 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql([]); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 1 result if we have a list which contains the float range of 1.0-1.2', async () => { - await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt'); + it('will return 1 result if we have a list which contains the float range of 1.0-1.2', async () => { + await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt', ['1.0', '1.2']); const rule = getRuleForSignalTesting(['float_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'float', list: { id: 'list_items.txt', - type: 'ip', + type: 'float_range', }, operator: 'included', type: 'list', @@ -565,16 +562,14 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess(supertest, id); await waitForSignalsToBePresent(supertest, 1, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.float).sort(); expect(hits).to.eql(['1.3']); }); }); }); describe('"is not in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a float against an index that has the floats stored as real floats. - describe.skip('working against float values in the data set', () => { + describe('working against float values in the data set', () => { it('will return 1 result if we have a list that excludes 1 float', async () => { await importFile(supertest, 'float', ['1.0'], 'list_items.txt'); const rule = getRuleForSignalTesting(['float']); @@ -715,17 +710,16 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql(['1.0', '1.1', '1.2', '1.3']); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 3 results if we have a list which contains the float range of 1.0-1.2', async () => { - await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt'); + it('will return 3 results if we have a list which contains the float range of 1.0-1.2', async () => { + await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt', ['1.0', '1.2']); const rule = getRuleForSignalTesting(['float_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'float', list: { id: 'list_items.txt', - type: 'ip', + type: 'float_range', }, operator: 'excluded', type: 'list', @@ -733,9 +727,9 @@ export default ({ getService }: FtrProviderContext) => { ], ]); await waitForRuleSuccess(supertest, id); - await waitForSignalsToBePresent(supertest, 1, [id]); + await waitForSignalsToBePresent(supertest, 3, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.float).sort(); expect(hits).to.eql(['1.0', '1.1', '1.2']); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts index 0fbb97d28442..6b32eb19c83d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts @@ -16,8 +16,11 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./float')); loadTestFile(require.resolve('./integer')); loadTestFile(require.resolve('./ip')); + loadTestFile(require.resolve('./ip_array')); loadTestFile(require.resolve('./keyword')); + loadTestFile(require.resolve('./keyword_array')); loadTestFile(require.resolve('./long')); loadTestFile(require.resolve('./text')); + loadTestFile(require.resolve('./text_array')); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts index 9b38f0f7cbb4..a1275afe288b 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts @@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => { }); describe('"is in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a integer against an index that has the integers stored as real integers. - describe.skip('working against integer values in the data set', () => { + describe('working against integer values in the data set', () => { it('will return 3 results if we have a list that includes 1 integer', async () => { await importFile(supertest, 'integer', ['1'], 'list_items.txt'); const rule = getRuleForSignalTesting(['integer']); @@ -545,17 +543,16 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql([]); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 1 result if we have a list which contains the integer range of 1-3', async () => { - await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt'); + it('will return 1 result if we have a list which contains the integer range of 1-3', async () => { + await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt', ['1', '2']); const rule = getRuleForSignalTesting(['integer_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'integer', list: { id: 'list_items.txt', - type: 'ip', + type: 'integer_range', }, operator: 'included', type: 'list', @@ -565,16 +562,14 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess(supertest, id); await waitForSignalsToBePresent(supertest, 1, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.integer).sort(); expect(hits).to.eql(['4']); }); }); }); describe('"is not in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a integer against an index that has the integers stored as real integers. - describe.skip('working against integer values in the data set', () => { + describe('working against integer values in the data set', () => { it('will return 1 result if we have a list that excludes 1 integer', async () => { await importFile(supertest, 'integer', ['1'], 'list_items.txt'); const rule = getRuleForSignalTesting(['integer']); @@ -715,17 +710,16 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql(['1', '2', '3', '4']); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 3 results if we have a list which contains the integer range of 1-3', async () => { - await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt'); + it('will return 3 results if we have a list which contains the integer range of 1-3', async () => { + await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt', ['1', '2', '3']); const rule = getRuleForSignalTesting(['integer_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'integer', list: { id: 'list_items.txt', - type: 'ip', + type: 'integer_range', }, operator: 'excluded', type: 'list', @@ -733,9 +727,9 @@ export default ({ getService }: FtrProviderContext) => { ], ]); await waitForRuleSuccess(supertest, id); - await waitForSignalsToBePresent(supertest, 1, [id]); + await waitForSignalsToBePresent(supertest, 3, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.integer).sort(); expect(hits).to.eql(['1', '2', '3']); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts index c3537efc12de..311354c63ca4 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts @@ -180,7 +180,7 @@ export default ({ getService }: FtrProviderContext) => { expect(ips).to.eql([]); }); - it('should filter a CIDR range of 127.0.0.1/30', async () => { + it('should filter a CIDR range of "127.0.0.1/30"', async () => { const rule = getRuleForSignalTesting(['ip']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -494,9 +494,12 @@ export default ({ getService }: FtrProviderContext) => { expect(ips).to.eql([]); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 1 result if we have a list which contains the CIDR range of 127.0.0.1/30', async () => { - await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt'); + it('will return 1 result if we have a list which contains the CIDR range of "127.0.0.1/30"', async () => { + await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt', [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + ]); const rule = getRuleForSignalTesting(['ip']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -504,7 +507,63 @@ export default ({ getService }: FtrProviderContext) => { field: 'ip', list: { id: 'list_items.txt', - type: 'ip', + type: 'ip_range', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql(['127.0.0.4']); + }); + + it('will return 1 result if we have a list which contains the range syntax of "127.0.0.1-127.0.0.3"', async () => { + await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.3'], 'list_items.txt', [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + ]); + const rule = getRuleForSignalTesting(['ip']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql(['127.0.0.4']); + }); + + it('will return 1 result if we have a list which contains the range mixed syntax of "127.0.0.1/32,127.0.0.2-127.0.0.3"', async () => { + await importFile( + supertest, + 'ip_range', + ['127.0.0.1/32', '127.0.0.2-127.0.0.3'], + 'list_items.txt', + ['127.0.0.1', '127.0.0.2', '127.0.0.3'] + ); + const rule = getRuleForSignalTesting(['ip']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', }, operator: 'included', type: 'list', @@ -594,9 +653,12 @@ export default ({ getService }: FtrProviderContext) => { expect(ips).to.eql(['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 3 results if we have a list which contains the CIDR range of 127.0.0.1/30', async () => { - await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt'); + it('will return 3 results if we have a list which contains the CIDR range of "127.0.0.1/30"', async () => { + await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt', [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + ]); const rule = getRuleForSignalTesting(['ip']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -604,9 +666,36 @@ export default ({ getService }: FtrProviderContext) => { field: 'ip', list: { id: 'list_items.txt', - type: 'ip', + type: 'ip_range', }, - operator: 'included', + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql(['127.0.0.1', '127.0.0.2', '127.0.0.3']); + }); + + it('will return 3 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.3"', async () => { + await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.3'], 'list_items.txt', [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + ]); + const rule = getRuleForSignalTesting(['ip']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', + }, + operator: 'excluded', type: 'list', }, ], diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts new file mode 100644 index 000000000000..8f4827ec6e71 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts @@ -0,0 +1,735 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { + createListsIndex, + deleteAllExceptions, + deleteListsIndex, + importFile, +} from '../../../../lists_api_integration/utils'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { + createRule, + createRuleWithExceptionEntries, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getRuleForSignalTesting, + getSignalsById, + waitForRuleSuccess, + waitForSignalsToBePresent, +} from '../../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('Rule exception operators for data type ip', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await createListsIndex(supertest); + await esArchiver.load('rule_exceptions/ip_as_array'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await deleteAllExceptions(es); + await deleteListsIndex(supertest); + await esArchiver.unload('rule_exceptions/ip_as_array'); + }); + + describe('"is" operator', () => { + it('should find all the ips from the data set when no exceptions are set on the rule', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + [], + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + + it('should filter 1 single ip if it is set as an exception', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.1', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + [], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + + it('should filter 2 ips if both are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.1', + }, + ], + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.5', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + }); + + it('should filter 3 ips if all 3 are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.1', + }, + ], + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.5', + }, + ], + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.8', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[]]); + }); + + it('should filter a CIDR range of "127.0.0.1/30"', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.1/30', // CIDR IP Range is 127.0.0.0 - 127.0.0.3 + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + [], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + + it('should filter a CIDR range of "127.0.0.4/31"', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match', + value: '127.0.0.4/31', // CIDR IP Range is 127.0.0.4 - 127.0.0.5 + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + }); + }); + + describe('"is not" operator', () => { + it('will return 0 results if it cannot find what it is excluding', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'match', + value: '192.168.0.1', // this value does not exist + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([]); + }); + + it('will return just 1 result we excluded', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'match', + value: '127.0.0.1', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']]); + }); + + it('will return just 1 result we excluded 2 from the same array elements', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'match', + value: '127.0.0.1', + }, + { + field: 'ip', + operator: 'excluded', + type: 'match', + value: '127.0.0.2', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']]); + }); + + it('will return 0 results if we exclude two ips', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'match', + value: '127.0.0.1', + }, + ], + [ + { + field: 'ip', + operator: 'excluded', + type: 'match', + value: '127.0.0.5', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([]); + }); + }); + + describe('"is one of" operator', () => { + it('should filter 1 single ip if it is set as an exception', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match_any', + value: ['127.0.0.1'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + [], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + + it('should filter 2 ips if both are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match_any', + value: ['127.0.0.1', '127.0.0.5'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + }); + + it('should filter 3 ips if all 3 are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'match_any', + value: ['127.0.0.1', '127.0.0.5', '127.0.0.8'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[]]); + }); + }); + + describe('"is not one of" operator', () => { + it('will return 0 results if it cannot find what it is excluding', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'match_any', + value: ['192.168.0.1', '192.168.0.2'], // These values do not exist + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([]); + }); + + it('will return just the result we excluded', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'match_any', + value: ['127.0.0.1', '127.0.0.5'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ]); + }); + }); + + describe('"exists" operator', () => { + it('will return 1 empty result if matching against ip', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'included', + type: 'exists', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[]]); + }); + }); + + describe('"does not exist" operator', () => { + it('will return 3 results if matching against ip', async () => { + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + operator: 'excluded', + type: 'exists', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + }); + + describe('"is in list" operator', () => { + it('will return 3 results if we have a list that includes 1 ip', async () => { + await importFile(supertest, 'ip', ['127.0.0.1'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + [], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + + it('will return 2 results if we have a list that includes 2 ips', async () => { + await importFile(supertest, 'ip', ['127.0.0.1', '127.0.0.5'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + }); + + it('will return 1 result if we have a list that includes all ips', async () => { + await importFile( + supertest, + 'ip', + ['127.0.0.1', '127.0.0.5', '127.0.0.8'], + 'list_items.txt' + ); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[]]); + }); + + it('will return 2 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => { + await importFile( + supertest, + 'ip_range', + ['127.0.0.1/32', '127.0.0.2/31', '127.0.0.4/30'], + 'list_items.txt', + [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + '127.0.0.4', + '127.0.0.5', + '127.0.0.6', + '127.0.0.7', + ] + ); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + }); + + it('will return 2 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.7', async () => { + await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.7'], 'list_items.txt', [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + '127.0.0.4', + '127.0.0.5', + '127.0.0.6', + '127.0.0.7', + ]); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + }); + }); + + describe('"is not in list" operator', () => { + it('will return 1 result if we have a list that excludes 1 ip', async () => { + await importFile(supertest, 'ip', ['127.0.0.1'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']]); + }); + + it('will return 2 results if we have a list that excludes 2 ips', async () => { + await importFile(supertest, 'ip', ['127.0.0.1', '127.0.0.5'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ]); + }); + + it('will return 3 results if we have a list that excludes all ips', async () => { + await importFile( + supertest, + 'ip', + ['127.0.0.1', '127.0.0.5', '127.0.0.8'], + 'list_items.txt' + ); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + ]); + }); + + it('will return 3 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => { + await importFile( + supertest, + 'ip_range', + ['127.0.0.1/32', '127.0.0.2/31', '127.0.0.4/30'], + 'list_items.txt', + [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + '127.0.0.4', + '127.0.0.5', + '127.0.0.6', + '127.0.0.7', + ] + ); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ]); + }); + + it('will return 3 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.7"', async () => { + await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.7'], 'list_items.txt', [ + '127.0.0.1', + '127.0.0.2', + '127.0.0.3', + '127.0.0.4', + '127.0.0.5', + '127.0.0.6', + '127.0.0.7', + ]); + const rule = getRuleForSignalTesting(['ip_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'ip', + list: { + id: 'list_items.txt', + type: 'ip_range', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + expect(ips).to.eql([ + ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], + ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], + ]); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts index 0c227c9acc38..e4e80cb1b65e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts @@ -402,6 +402,39 @@ export default ({ getService }: FtrProviderContext) => { }); describe('"is in list" operator', () => { + it('will return 4 results if we have two lists with an AND contradiction keyword === "word one" AND keyword === "word two"', async () => { + await importFile(supertest, 'keyword', ['word one'], 'list_items_1.txt'); + await importFile(supertest, 'keyword', ['word two'], 'list_items_2.txt'); + const rule = getRuleForSignalTesting(['keyword']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items_1.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + { + field: 'keyword', + list: { + id: 'list_items_2.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql(['word four', 'word one', 'word three', 'word two']); + }); + it('will return 3 results if we have a list that includes 1 keyword', async () => { await importFile(supertest, 'keyword', ['word one'], 'list_items.txt'); const rule = getRuleForSignalTesting(['keyword']); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts new file mode 100644 index 000000000000..01e301c35085 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts @@ -0,0 +1,624 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { + createListsIndex, + deleteAllExceptions, + deleteListsIndex, + importFile, +} from '../../../../lists_api_integration/utils'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { + createRule, + createRuleWithExceptionEntries, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getRuleForSignalTesting, + getSignalsById, + waitForRuleSuccess, + waitForSignalsToBePresent, +} from '../../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('Rule exception operators for data type keyword', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await createListsIndex(supertest); + await esArchiver.load('rule_exceptions/keyword_as_array'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await deleteAllExceptions(es); + await deleteListsIndex(supertest); + await esArchiver.unload('rule_exceptions/keyword_as_array'); + }); + + describe('"is" operator', () => { + it('should find all the keyword from the data set when no exceptions are set on the rule', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + + it('should filter 1 single keyword if it is set as an exception', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'word one', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('should filter 2 keyword if both are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'word one', + }, + ], + [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'word seven', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + }); + + it('should filter 3 keyword if all 3 are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'word one', + }, + ], + [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'word six', + }, + ], + [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'word nine', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"is not" operator', () => { + it('will return 0 results if it cannot find what it is excluding', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'excluded', + type: 'match', + value: '500.0', // this value is not in the data set + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([]); + }); + + it('will return just 1 result we excluded', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'excluded', + type: 'match', + value: 'word one', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]); + }); + + it('will return 0 results if we exclude two keyword', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'excluded', + type: 'match', + value: 'word one', + }, + ], + [ + { + field: 'keyword', + operator: 'excluded', + type: 'match', + value: 'word five', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([]); + }); + }); + + describe('"is one of" operator', () => { + it('should filter 1 single keyword if it is set as an exception', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'match_any', + value: ['word one'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('should filter 2 keyword if both are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'match_any', + value: ['word one', 'word six'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + }); + + it('should filter 3 keyword if all 3 are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'match_any', + value: ['word one', 'word five', 'word eight'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"is not one of" operator', () => { + it('will return 0 results if it cannot find what it is excluding', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'excluded', + type: 'match_any', + value: ['500', '600'], // both these values are not in the data set + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([]); + }); + + it('will return just the result we excluded', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'excluded', + type: 'match_any', + value: ['word one', 'word six'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + }); + + describe('"exists" operator', () => { + it('will return 1 results if matching against keyword for the empty array', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'included', + type: 'exists', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"does not exist" operator', () => { + it('will return 3 results if matching against keyword', async () => { + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + operator: 'excluded', + type: 'exists', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + }); + + describe('"is in list" operator', () => { + it('will return 4 results if we have two lists with an AND contradiction keyword === "word one" AND keyword === "word five"', async () => { + await importFile(supertest, 'keyword', ['word one'], 'list_items_1.txt'); + await importFile(supertest, 'keyword', ['word five'], 'list_items_2.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items_1.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + { + field: 'keyword', + list: { + id: 'list_items_2.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + + it('will return 3 results if we have two lists with an AND keyword === "word one" AND keyword === "word two" since we have an array', async () => { + await importFile(supertest, 'keyword', ['word one'], 'list_items_1.txt'); + await importFile(supertest, 'keyword', ['word two'], 'list_items_2.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items_1.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + { + field: 'keyword', + list: { + id: 'list_items_2.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('will return 3 results if we have a list that includes 1 keyword', async () => { + await importFile(supertest, 'keyword', ['word one'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('will return 2 results if we have a list that includes 2 keyword', async () => { + await importFile(supertest, 'keyword', ['word one', 'word six'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + }); + + it('will return only the empty array for results if we have a list that includes all keyword', async () => { + await importFile( + supertest, + 'keyword', + ['word one', 'word five', 'word eight'], + 'list_items.txt' + ); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"is not in list" operator', () => { + it('will return 1 result if we have a list that excludes 1 keyword', async () => { + await importFile(supertest, 'keyword', ['word one'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]); + }); + + it('will return 1 result if we have a list that excludes 1 keyword but repeat 2 elements from the array in the list', async () => { + await importFile(supertest, 'keyword', ['word one', 'word two'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]); + }); + + it('will return 2 results if we have a list that excludes 2 keyword', async () => { + await importFile(supertest, 'keyword', ['word one', 'word five'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + + it('will return 3 results if we have a list that excludes 3 items', async () => { + await importFile( + supertest, + 'keyword', + ['word one', 'word six', 'word ten'], + 'list_items.txt' + ); + const rule = getRuleForSignalTesting(['keyword_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'keyword', + list: { + id: 'list_items.txt', + type: 'keyword', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort(); + expect(hits).to.eql([ + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts index 5c110996c219..ee52c41bc78e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts @@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => { }); describe('"is in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a long against an index that has the longs stored as real longs. - describe.skip('working against long values in the data set', () => { + describe('working against long values in the data set', () => { it('will return 3 results if we have a list that includes 1 long', async () => { await importFile(supertest, 'long', ['1'], 'list_items.txt'); const rule = getRuleForSignalTesting(['long']); @@ -545,17 +543,16 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql([]); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 1 result if we have a list which contains the long range of 1-3', async () => { - await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt'); + it('will return 1 result if we have a list which contains the long range of 1-3', async () => { + await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt', ['1', '2', '3']); const rule = getRuleForSignalTesting(['long_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'long', list: { id: 'list_items.txt', - type: 'ip', + type: 'long_range', }, operator: 'included', type: 'list', @@ -565,16 +562,14 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess(supertest, id); await waitForSignalsToBePresent(supertest, 1, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.long).sort(); expect(hits).to.eql(['4']); }); }); }); describe('"is not in list" operator', () => { - // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent - // a long against an index that has the longs stored as real longs. - describe.skip('working against long values in the data set', () => { + describe('working against long values in the data set', () => { it('will return 1 result if we have a list that excludes 1 long', async () => { await importFile(supertest, 'long', ['1'], 'list_items.txt'); const rule = getRuleForSignalTesting(['long']); @@ -715,17 +710,16 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql(['1', '2', '3', '4']); }); - // TODO: Fix this bug and then unskip this test - it.skip('will return 3 results if we have a list which contains the long range of 1-3', async () => { - await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt'); + it('will return 3 results if we have a list which contains the long range of 1-3', async () => { + await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt', ['1', '2', '3']); const rule = getRuleForSignalTesting(['long_as_string']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ { - field: 'ip', + field: 'long', list: { id: 'list_items.txt', - type: 'ip', + type: 'long_range', }, operator: 'excluded', type: 'list', @@ -733,9 +727,9 @@ export default ({ getService }: FtrProviderContext) => { ], ]); await waitForRuleSuccess(supertest, id); - await waitForSignalsToBePresent(supertest, 1, [id]); + await waitForSignalsToBePresent(supertest, 3, [id]); const signalsOpen = await getSignalsById(supertest, id); - const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort(); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.long).sort(); expect(hits).to.eql(['1', '2', '3']); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts index d2066b1023d3..095d88514938 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts @@ -592,10 +592,37 @@ export default ({ getService }: FtrProviderContext) => { }); }); - // TODO: Unskip these once this is fixed - describe.skip('working against text values with spaces', () => { + describe('working against text values with spaces', () => { it('will return 3 results if we have a list that includes 1 text', async () => { - await importFile(supertest, 'text', ['one'], 'list_items.txt'); + await importTextFile(supertest, 'text', ['word one'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql(['word four', 'word three', 'word two']); + }); + + it('will return 3 results if we have a list that includes 1 text with additional wording', async () => { + await importTextFile( + supertest, + 'text', + ['word one additional wording'], + 'list_items.txt' + ); const rule = getRuleForSignalTesting(['text']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -618,7 +645,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('will return 2 results if we have a list that includes 2 text', async () => { - await importFile(supertest, 'text', ['one', 'three'], 'list_items.txt'); + await importFile(supertest, 'text', ['word one', 'word three'], 'list_items.txt'); const rule = getRuleForSignalTesting(['text']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -644,7 +671,7 @@ export default ({ getService }: FtrProviderContext) => { await importTextFile( supertest, 'text', - ['one', 'two', 'three', 'four'], + ['word one', 'word two', 'word three', 'word four'], 'list_items.txt' ); const rule = getRuleForSignalTesting(['text']); @@ -746,10 +773,37 @@ export default ({ getService }: FtrProviderContext) => { }); }); - // TODO: Unskip these once this is fixed - describe.skip('working against text values with spaces', () => { + describe('working against text values with spaces', () => { it('will return 1 result if we have a list that excludes 1 text', async () => { - await importTextFile(supertest, 'text', ['one'], 'list_items.txt'); + await importTextFile(supertest, 'text', ['word one'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql(['word one']); + }); + + it('will return 1 result if we have a list that excludes 1 text with additional wording', async () => { + await importTextFile( + supertest, + 'text', + ['word one additional wording'], + 'list_items.txt' + ); const rule = getRuleForSignalTesting(['text']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -772,7 +826,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('will return 2 results if we have a list that excludes 2 text', async () => { - await importTextFile(supertest, 'text', ['one', 'three'], 'list_items.txt'); + await importTextFile(supertest, 'text', ['word one', 'word three'], 'list_items.txt'); const rule = getRuleForSignalTesting(['text']); const { id } = await createRuleWithExceptionEntries(supertest, rule, [ [ @@ -798,7 +852,7 @@ export default ({ getService }: FtrProviderContext) => { await importTextFile( supertest, 'text', - ['one', 'two', 'three', 'four'], + ['word one', 'word two', 'word three', 'word four'], 'list_items.txt' ); const rule = getRuleForSignalTesting(['text']); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts new file mode 100644 index 000000000000..ed63f1a0db25 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts @@ -0,0 +1,619 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { + createListsIndex, + deleteAllExceptions, + deleteListsIndex, + importFile, +} from '../../../../lists_api_integration/utils'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { + createRule, + createRuleWithExceptionEntries, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getRuleForSignalTesting, + getSignalsById, + waitForRuleSuccess, + waitForSignalsToBePresent, +} from '../../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('Rule exception operators for data type text', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await createListsIndex(supertest); + await esArchiver.load('rule_exceptions/text_as_array'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await deleteAllExceptions(es); + await deleteListsIndex(supertest); + await esArchiver.unload('rule_exceptions/text_as_array'); + }); + + describe('"is" operator', () => { + it('should find all the text from the data set when no exceptions are set on the rule', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + + it('should filter 1 single text if it is set as an exception', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'match', + value: 'word one', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('should filter 2 text if both are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'match', + value: 'word one', + }, + ], + [ + { + field: 'text', + operator: 'included', + type: 'match', + value: 'word seven', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + }); + + it('should filter 3 text if all 3 are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'match', + value: 'word one', + }, + ], + [ + { + field: 'text', + operator: 'included', + type: 'match', + value: 'word six', + }, + ], + [ + { + field: 'text', + operator: 'included', + type: 'match', + value: 'word nine', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"is not" operator', () => { + it('will return 0 results if it cannot find what it is excluding', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'excluded', + type: 'match', + value: '500.0', // this value is not in the data set + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([]); + }); + + it('will return just 1 result we excluded', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'excluded', + type: 'match', + value: 'word one', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]); + }); + + it('will return 0 results if we exclude two text', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'excluded', + type: 'match', + value: 'word one', + }, + ], + [ + { + field: 'text', + operator: 'excluded', + type: 'match', + value: 'word five', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([]); + }); + }); + + describe('"is one of" operator', () => { + it('should filter 1 single text if it is set as an exception', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'match_any', + value: ['word one'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('should filter 2 text if both are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'match_any', + value: ['word one', 'word six'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + }); + + it('should filter 3 text if all 3 are set as exceptions', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'match_any', + value: ['word one', 'word five', 'word eight'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"is not one of" operator', () => { + it('will return 0 results if it cannot find what it is excluding', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'excluded', + type: 'match_any', + value: ['500', '600'], // both these values are not in the data set + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([]); + }); + + it('will return just the result we excluded', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'excluded', + type: 'match_any', + value: ['word one', 'word six'], + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + }); + + describe('"exists" operator', () => { + it('will return 1 results if matching against text for the empty array', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'included', + type: 'exists', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"does not exist" operator', () => { + it('will return 3 results if matching against text', async () => { + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + operator: 'excluded', + type: 'exists', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + }); + + describe('"is in list" operator', () => { + it('will return 4 results if we have two lists with an AND contradiction text === "word one" AND text === "word five"', async () => { + await importFile(supertest, 'text', ['word one'], 'list_items_1.txt'); + await importFile(supertest, 'text', ['word five'], 'list_items_2.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items_1.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + { + field: 'text', + list: { + id: 'list_items_2.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + + it('will return 3 results if we have two lists with an AND text === "word one" AND text === "word two" since we have an array', async () => { + await importFile(supertest, 'text', ['word one'], 'list_items_1.txt'); + await importFile(supertest, 'text', ['word two'], 'list_items_2.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items_1.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + { + field: 'text', + list: { + id: 'list_items_2.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('will return 3 results if we have a list that includes 1 text', async () => { + await importFile(supertest, 'text', ['word one'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + [], + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ]); + }); + + it('will return 2 results if we have a list that includes 2 text', async () => { + await importFile(supertest, 'text', ['word one', 'word six'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + }); + + it('will return only the empty array for results if we have a list that includes all text', async () => { + await importFile( + supertest, + 'text', + ['word one', 'word five', 'word eight'], + 'list_items.txt' + ); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'included', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([[]]); + }); + }); + + describe('"is not in list" operator', () => { + it('will return 1 result if we have a list that excludes 1 text', async () => { + await importFile(supertest, 'text', ['word one'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]); + }); + + it('will return 1 result if we have a list that excludes 1 text but repeat 2 elements from the array in the list', async () => { + await importFile(supertest, 'text', ['word one', 'word two'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]); + }); + + it('will return 2 results if we have a list that excludes 2 text', async () => { + await importFile(supertest, 'text', ['word one', 'word five'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + + it('will return 3 results if we have a list that excludes 3 items', async () => { + await importFile(supertest, 'text', ['word one', 'word six', 'word ten'], 'list_items.txt'); + const rule = getRuleForSignalTesting(['text_as_array']); + const { id } = await createRuleWithExceptionEntries(supertest, rule, [ + [ + { + field: 'text', + list: { + id: 'list_items.txt', + type: 'text', + }, + operator: 'excluded', + type: 'list', + }, + ], + ]); + await waitForRuleSuccess(supertest, id); + await waitForSignalsToBePresent(supertest, 3, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort(); + expect(hits).to.eql([ + ['word eight', 'word nine', 'word ten'], + ['word five', null, 'word six', 'word seven'], + ['word one', 'word two', 'word three', 'word four'], + ]); + }); + }); + }); +}; diff --git a/x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/data.json b/x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/data.json new file mode 100644 index 000000000000..4a4316a05b2d --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/data.json @@ -0,0 +1,51 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "ip_as_array", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "ip": [] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "ip_as_array", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "ip": ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "ip_as_array", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "ip": ["127.0.0.5", null, "127.0.0.6", "127.0.0.7"] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "ip_as_array", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "ip": ["127.0.0.8", "127.0.0.9", "127.0.0.10"] + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/mappings.json b/x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/mappings.json new file mode 100644 index 000000000000..c46b79ce1138 --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "ip_as_array", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "ip": { "type": "ip" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/data.json b/x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/data.json new file mode 100644 index 000000000000..2c51d4cbc63c --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/data.json @@ -0,0 +1,51 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "keyword_as_array", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "keyword": [] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "keyword_as_array", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "keyword": ["word one", "word two", "word three", "word four"] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "keyword_as_array", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "keyword": ["word five", null, "word six", "word seven"] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "keyword_as_array", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "keyword": ["word eight", "word nine", "word ten"] + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/mappings.json b/x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/mappings.json new file mode 100644 index 000000000000..df62e96aecfc --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "keyword_as_array", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "keyword": { "type": "keyword" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/text_as_array/data.json b/x-pack/test/functional/es_archives/rule_exceptions/text_as_array/data.json new file mode 100644 index 000000000000..228132cda90d --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/text_as_array/data.json @@ -0,0 +1,51 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "text_as_array", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "text": [] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "text_as_array", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "text": ["word one", "word two", "word three", "word four"] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "text_as_array", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "text": ["word five", null, "word six", "word seven"] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "text_as_array", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "text": ["word eight", "word nine", "word ten"] + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/rule_exceptions/text_as_array/mappings.json b/x-pack/test/functional/es_archives/rule_exceptions/text_as_array/mappings.json new file mode 100644 index 000000000000..b0a3885da991 --- /dev/null +++ b/x-pack/test/functional/es_archives/rule_exceptions/text_as_array/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "text_as_array", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "keyword": { "type": "text" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index 53472b459b8a..c008f4e1a4e5 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -175,12 +175,14 @@ export const deleteAllExceptions = async (es: Client): Promise => { * @param type The type to import as * @param contents The contents of the import * @param fileName filename to import as + * @param testValues Optional test values in case you're using CIDR or range based lists */ export const importFile = async ( supertest: SuperTest, type: Type, contents: string[], - fileName: string + fileName: string, + testValues?: string[] ): Promise => { await supertest .post(`${LIST_ITEM_URL}/_import?type=${type}`) @@ -191,7 +193,8 @@ export const importFile = async ( // although we have pushed the list and its items, it is async so we // have to wait for the contents before continuing - await waitForListItems(supertest, contents, fileName); + const testValuesOrContents = testValues ?? contents; + await waitForListItems(supertest, testValuesOrContents, fileName); }; /**