[Security Solution] Fixes CIDR, float, long, integer, array, and text based issues when using value lists in exceptions (#85191)

## Summary

Fixes different bugs/issues when using exceptions with value based lists for both the UI, the backend, and the large value based lists. See https://github.com/elastic/kibana/issues/79516, but this also fixes several other bugs found mentioned below.

For the front end UI:
* Adds the ability to specify value based lists that are IP Ranges when the source event is an IP. Before you could only match IP to IP and the IP Ranges lists could not be used. 
* Breaks down a few functions into smaller functions for unit test writing abilities.

You can now add ip ranges as list values for the UI when before it would not show up:
<img width="1035" alt="Screen Shot 2020-12-07 at 2 15 39 PM" src="https://user-images.githubusercontent.com/1151048/101406552-d6819b00-3896-11eb-9fb5-4c7c2ad93b2e.png">

For value based lists:
* Fixes text data type to use "and" between matching using `operator: 'and'` and changes it from a `terms query to a `match` query
* Adds new API for searching against types called `searchListItemByValues ` so that numeric, text, array based, and other non-stringable types can be sent and then the value based lists will push that to ElasticSearch. This shifts as many corner cases and string/numeric coercions to ElasticSearch rather than Kibana client side code.
* Adds ability to handle arrays within arrays through a `flatten` call.
* Utilizes the `named queries` from ElasticSearch for the new API so that clients can get which parts matched and then use that for their exception list logic rather than in-memory string to string checks. This fixes CIDR and ranges as well as works with arrays.

For the backend exception lists that used value based lists:
* Broke down the `filterEventsAgainstList` function into a folder called `filters` and the functions into other files for better unit based testing.
* Changed the calls from `getListItemByValues` to `searchListItemByValues` which can return exactly what it matched against and this will not break anyone using the existing REST API for `getListItemByValues` since that REST API and client side API stays the same.
* Cleaned up extra promises being used in a few spots that async/await automatically will create. 
* Removed the stringabilities and stringify in favor of just a simpler exact check using `JSON.stringify()`

For the tests:
* Adds unit tests to broken down functions
* Adds ip_array, keyword_array, text_array, FTR tests for the backend.
* Adds more CIDR and range based FTR tests for the backend.
* Unskips and fixes all the numeric tests and range tests that could not operate previously from bugs.

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

x-pack/plugins/lists/common/schemas/elastic_response/search_es_list_item_schema.mock.ts

@@ -70,6 +70,7 @@ export const getSearchListItemMock = (): SearchResponse<SearchEsListItemSchema>
         _score: 0,
         _source: getSearchEsListItemMock(),
         _type: '',
+        matched_queries: ['0.0'],
       },
     ],
     max_score: 0,

x-pack/plugins/lists/common/schemas/response/index.ts

@@ -15,3 +15,4 @@ export * from './found_list_schema';
 export * from './list_item_schema';
 export * from './list_schema';
 export * from './list_item_index_exist_schema';
+export * from './search_list_item_schema';

x-pack/plugins/lists/common/schemas/response/search_list_item_schema.mock.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SearchListItemSchema } from '../../../common/schemas';
+import { VALUE } from '../../../common/constants.mock';
+
+import { getListItemResponseMock } from './list_item_schema.mock';
+
+export const getSearchListItemResponseMock = (): SearchListItemSchema => ({
+  items: [getListItemResponseMock()],
+  value: VALUE,
+});

x-pack/plugins/lists/common/schemas/response/search_list_item_schema.test.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { left } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+
+import { exactCheck, foldLeftRight, getPaths } from '../../shared_imports';
+
+import { getSearchListItemResponseMock } from './search_list_item_schema.mock';
+import { SearchListItemSchema, searchListItemSchema } from './search_list_item_schema';
+
+describe('search_list_item_schema', () => {
+  test('it should validate a typical search list item response', () => {
+    const payload = getSearchListItemResponseMock();
+    const decoded = searchListItemSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should NOT validate with an "undefined" for "items"', () => {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { items, ...noItems } = getSearchListItemResponseMock();
+    const decoded = searchListItemSchema.decode(noItems);
+    const checked = exactCheck(noItems, decoded);
+    const message = pipe(checked, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([
+      'Invalid value "undefined" supplied to "items"',
+    ]);
+    expect(message.schema).toEqual({});
+  });
+
+  test('it should not allow an extra key to be sent in', () => {
+    const payload: SearchListItemSchema & { extraKey?: string } = getSearchListItemResponseMock();
+    payload.extraKey = 'some new value';
+    const decoded = searchListItemSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual(['invalid keys "extraKey"']);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/response/search_list_item_schema.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { listItemArraySchema } from './list_item_schema';
+
+/**
+ * NOTE: Although this is defined within "response" this does not expose a REST API
+ * endpoint right now for this particular response. Instead this is only used internally
+ * for the plugins at this moment. If this changes, please remove this message.
+ */
+export const searchListItemSchema = t.exact(
+  t.type({
+    items: listItemArraySchema,
+    value: t.unknown,
+  })
+);
+
+export type SearchListItemSchema = t.TypeOf<typeof searchListItemSchema>;
+
+export const searchListItemArraySchema = t.array(searchListItemSchema);
+export type SearchListItemArraySchema = t.TypeOf<typeof searchListItemArraySchema>;

x-pack/plugins/lists/server/services/items/delete_list_item_by_value.test.ts

@@ -47,7 +47,15 @@ describe('delete_list_item_by_value', () => {
       body: {
         query: {
           bool: {
-            filter: [{ term: { list_id: 'some-list-id' } }, { terms: { ip: ['127.0.0.1'] } }],
+            filter: [
+              { term: { list_id: 'some-list-id' } },
+              {
+                bool: {
+                  minimum_should_match: 1,
+                  should: [{ term: { ip: { _name: '0.0', value: '127.0.0.1' } } }],
+                },
+              },
+            ],
           },
         },
       },

x-pack/plugins/lists/server/services/items/index.ts

@@ -11,6 +11,7 @@ export * from './delete_list_item_by_value';
 export * from './delete_list_item';
 export * from './find_list_item';
 export * from './get_list_item_by_value';
+export * from './get_list_item_by_values';
 export * from './get_list_item';
 export * from './get_list_item_by_values';
 export * from './get_list_item_template';
@@ -18,3 +19,4 @@ export * from './get_list_item_index';
 export * from './update_list_item';
 export * from './write_lines_to_bulk_list_items';
 export * from './write_list_items_to_stream';
+export * from './search_list_item_by_values';

x-pack/plugins/lists/server/services/items/search_list_item_by_values.mock.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getCallClusterMock } from '../../../common/get_call_cluster.mock';
+import { SearchListItemByValuesOptions } from '../items';
+import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE, VALUE_2 } from '../../../common/constants.mock';
+
+export const searchListItemByValuesOptionsMocks = (): SearchListItemByValuesOptions => ({
+  callCluster: getCallClusterMock(),
+  listId: LIST_ID,
+  listItemIndex: LIST_ITEM_INDEX,
+  type: TYPE,
+  value: [VALUE, VALUE_2],
+});

x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SearchListItemArraySchema } from '../../../common/schemas';
+import { getSearchListItemMock } from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock';
+import { getCallClusterMock } from '../../../common/get_call_cluster.mock';
+import { LIST_ID, LIST_ITEM_INDEX, TYPE, VALUE, VALUE_2 } from '../../../common/constants.mock';
+
+import { searchListItemByValues } from './search_list_item_by_values';
+
+describe('search_list_item_by_values', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('Returns a an empty array of items if the value is empty', async () => {
+    const data = getSearchListItemMock();
+    data.hits.hits = [];
+    const callCluster = getCallClusterMock(data);
+    const listItem = await searchListItemByValues({
+      callCluster,
+      listId: LIST_ID,
+      listItemIndex: LIST_ITEM_INDEX,
+      type: TYPE,
+      value: [],
+    });
+
+    expect(listItem).toEqual([]);
+  });
+
+  test('Returns a an empty array of items if the ES query is also empty', async () => {
+    const data = getSearchListItemMock();
+    data.hits.hits = [];
+    const callCluster = getCallClusterMock(data);
+    const listItem = await searchListItemByValues({
+      callCluster,
+      listId: LIST_ID,
+      listItemIndex: LIST_ITEM_INDEX,
+      type: TYPE,
+      value: [VALUE, VALUE_2],
+    });
+
+    const expected: SearchListItemArraySchema = [
+      { items: [], value: VALUE },
+      { items: [], value: VALUE_2 },
+    ];
+    expect(listItem).toEqual(expected);
+  });
+
+  test('Returns transformed list item if the data exists within ES', async () => {
+    const data = getSearchListItemMock();
+    const callCluster = getCallClusterMock(data);
+    const listItem = await searchListItemByValues({
+      callCluster,
+      listId: LIST_ID,
+      listItemIndex: LIST_ITEM_INDEX,
+      type: TYPE,
+      value: [VALUE, VALUE_2],
+    });
+
+    const expected: SearchListItemArraySchema = [
+      {
+        items: [
+          {
+            _version: undefined,
+            created_at: '2020-04-20T15:25:31.830Z',
+            created_by: 'some user',
+            deserializer: undefined,
+            id: 'some-list-item-id',
+            list_id: 'some-list-id',
+            meta: {},
+            serializer: undefined,
+            tie_breaker_id: '6a76b69d-80df-4ab2-8c3e-85f466b06a0e',
+            type: 'ip',
+            updated_at: '2020-04-20T15:25:31.830Z',
+            updated_by: 'some user',
+            value: '127.0.0.1',
+          },
+        ],
+        value: '127.0.0.1',
+      },
+      {
+        items: [],
+        value: VALUE_2,
+      },
+    ];
+    expect(listItem).toEqual(expected);
+  });
+});

x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from 'kibana/server';
+
+import { SearchEsListItemSchema, SearchListItemArraySchema, Type } from '../../../common/schemas';
+import { getQueryFilterFromTypeValue, transformElasticNamedSearchToListItem } from '../utils';
+
+export interface SearchListItemByValuesOptions {
+  listId: string;
+  callCluster: LegacyAPICaller;
+  listItemIndex: string;
+  type: Type;
+  value: unknown[];
+}
+
+export const searchListItemByValues = async ({
+  listId,
+  callCluster,
+  listItemIndex,
+  type,
+  value,
+}: SearchListItemByValuesOptions): Promise<SearchListItemArraySchema> => {
+  const response = await callCluster<SearchEsListItemSchema>('search', {
+    body: {
+      query: {
+        bool: {
+          filter: getQueryFilterFromTypeValue({ listId, type, value }),
+        },
+      },
+    },
+    ignoreUnavailable: true,
+    index: listItemIndex,
+    size: 10000, // TODO: This has a limit on the number which is 10,000 the default of Elastic but we might want to provide a way to increase that number
+  });
+  return transformElasticNamedSearchToListItem({ response, type, value });
+};

x-pack/plugins/lists/server/services/lists/list_client.ts

@@ -12,6 +12,7 @@ import {
   ListItemArraySchema,
   ListItemSchema,
   ListSchema,
+  SearchListItemArraySchema,
 } from '../../../common/schemas';
 import { ConfigType } from '../../config';
 import {
@@ -35,6 +36,7 @@ import {
   getListItemIndex,
   getListItemTemplate,
   importListItemsToStream,
+  searchListItemByValues,
   updateListItem,
 } from '../../services/items';
 import {
@@ -67,6 +69,7 @@ import {
   GetListItemsByValueOptions,
   GetListOptions,
   ImportListItemsToStreamOptions,
+  SearchListItemByValuesOptions,
   UpdateListItemOptions,
   UpdateListOptions,
 } from './list_client_types';
@@ -472,6 +475,22 @@ export class ListClient {
     });
   };
 
+  public searchListItemByValues = async ({
+    type,
+    listId,
+    value,
+  }: SearchListItemByValuesOptions): Promise<SearchListItemArraySchema> => {
+    const { callCluster } = this;
+    const listItemIndex = this.getListItemIndex();
+    return searchListItemByValues({
+      callCluster,
+      listId,
+      listItemIndex,
+      type,
+      value,
+    });
+  };
+
   public findList = async ({
     filter,
     currentIndexPosition,

x-pack/plugins/lists/server/services/lists/list_client_types.ts

@@ -160,3 +160,9 @@ export interface FindListItemOptions {
   sortField: SortFieldOrUndefined;
   sortOrder: SortOrderOrUndefined;
 }
+
+export interface SearchListItemByValuesOptions {
+  type: Type;
+  listId: string;
+  value: unknown[];
+}

x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.test.ts

@@ -4,7 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { QueryFilterType, getQueryFilterFromTypeValue } from './get_query_filter_from_type_value';
+import {
+  QueryFilterType,
+  getEmptyQuery,
+  getQueryFilterFromTypeValue,
+  getShouldQuery,
+  getTermsQuery,
+  getTextQuery,
+} from './get_query_filter_from_type_value';
 
 describe('get_query_filter_from_type_value', () => {
   beforeEach(() => {
@@ -15,78 +22,813 @@ describe('get_query_filter_from_type_value', () => {
     jest.clearAllMocks();
   });
 
-  test('it returns an ip if given an ip', () => {
-    const queryFilter = getQueryFilterFromTypeValue({
-      listId: 'list-123',
-      type: 'ip',
-      value: ['127.0.0.1'],
+  describe('getQueryFilterFromTypeValue', () => {
+    test('it returns an ip if given an ip', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'ip',
+        value: ['127.0.0.1'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [{ term: { ip: { _name: '0.0', value: '127.0.0.1' } } }],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it returns two ip if given two ip', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'ip',
+        value: ['127.0.0.1', '127.0.0.2'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              { term: { ip: { _name: '0.0', value: '127.0.0.1' } } },
+              { term: { ip: { _name: '1.0', value: '127.0.0.2' } } },
+            ],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it returns a keyword if given a keyword', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'keyword',
+        value: ['host-name-1'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [{ term: { keyword: { _name: '0.0', value: 'host-name-1' } } }],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it returns two keywords if given two values', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'keyword',
+        value: ['host-name-1', 'host-name-2'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              { term: { keyword: { _name: '0.0', value: 'host-name-1' } } },
+              { term: { keyword: { _name: '1.0', value: 'host-name-2' } } },
+            ],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it returns an empty query given an empty value', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'keyword',
+        value: [],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_none: {
+                  _name: 'empty',
+                },
+              },
+            ],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it returns an empty query object given an empty array', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'ip',
+        value: [],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_none: {
+                  _name: 'empty',
+                },
+              },
+            ],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it returns an empty query object given an array with only null values', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'ip',
+        value: [null, null],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_none: {
+                  _name: 'empty',
+                },
+              },
+            ],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it filters out a null value if mixed with a string value for non-text based query', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'ip',
+        value: [null, 'host-name-1'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it filters out an object value if mixed with a string value for non-text based query', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'ip',
+        value: [{}, 'host-name-1'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it filters out a null value if mixed with a string value for text based query', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'text',
+        value: [null, 'host-name-1'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [{ match: { text: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
+    });
+
+    test('it filters out object values if mixed with a string value for text based query', () => {
+      const query = getQueryFilterFromTypeValue({
+        listId: 'list-123',
+        type: 'text',
+        value: [{}, 'host-name-1'],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [{ match: { text: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
     });
-    const expected: QueryFilterType = [
-      { term: { list_id: 'list-123' } },
-      { terms: { ip: ['127.0.0.1'] } },
-    ];
-    expect(queryFilter).toEqual(expected);
   });
 
-  test('it returns two ip if given two ip', () => {
-    const queryFilter = getQueryFilterFromTypeValue({
-      listId: 'list-123',
-      type: 'ip',
-      value: ['127.0.0.1', '127.0.0.2'],
+  describe('getEmptyQuery', () => {
+    test('it returns an empty query given a list_id', () => {
+      const emptyQuery = getEmptyQuery({ listId: 'list-123' });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        { bool: { minimum_should_match: 1, should: [{ match_none: { _name: 'empty' } }] } },
+      ];
+      expect(emptyQuery).toEqual(expected);
     });
-    const expected: QueryFilterType = [
-      { term: { list_id: 'list-123' } },
-      { terms: { ip: ['127.0.0.1', '127.0.0.2'] } },
-    ];
-    expect(queryFilter).toEqual(expected);
   });
 
-  test('it returns a keyword if given a keyword', () => {
-    const queryFilter = getQueryFilterFromTypeValue({
-      listId: 'list-123',
-      type: 'keyword',
-      value: ['host-name-1'],
+  describe('getTermsQuery', () => {
+    describe('scalar values', () => {
+      test('it returns a expected terms query give a single string value, listId, and type', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: ['127.0.0.1'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ term: { ip: { _name: '0.0', value: '127.0.0.1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected terms query given two string values, listId, and type', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: ['127.0.0.1', '127.0.0.2'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { term: { ip: { _name: '0.0', value: '127.0.0.1' } } },
+                { term: { ip: { _name: '1.0', value: '127.0.0.2' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected numeric terms without converting them into strings', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [5, 3],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { term: { ip: { _name: '0.0', value: 5 } } },
+                { term: { ip: { _name: '1.0', value: 3 } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns a string and a numeric without converting them into a homogenous type', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [5, '3'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { term: { ip: { _name: '0.0', value: 5 } } },
+                { term: { ip: { _name: '1.0', value: '3' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out a null value if mixed with a string value', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [null, 'host-name-1'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out an object value if mixed with a string value', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [{}, 'host-name-1'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ term: { ip: { _name: '1.0', value: 'host-name-1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+    });
+
+    describe('array values', () => {
+      test('it returns a expected terms query give a single string value, listId, and type', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [['127.0.0.1']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ terms: { _name: '0.0', ip: ['127.0.0.1'] } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected terms query given two string values, listId, and type', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [['127.0.0.1'], ['127.0.0.2']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { terms: { _name: '0.0', ip: ['127.0.0.1'] } },
+                { terms: { _name: '1.0', ip: ['127.0.0.2'] } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected numeric terms without converting them into strings', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[5], [3]],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ terms: { _name: '0.0', ip: [5] } }, { terms: { _name: '1.0', ip: [3] } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns a string and a numeric without converting them into a homogenous type', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[5], ['3']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { terms: { _name: '0.0', ip: [5] } },
+                { terms: { _name: '1.0', ip: ['3'] } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out a null value if mixed with a string value', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[null], ['host-name-1']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ terms: { _name: '1.0', ip: ['host-name-1'] } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out an object value if mixed with a string value', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[{}], ['host-name-1']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ terms: { _name: '1.0', ip: ['host-name-1'] } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it flattens and removes null values correctly in a deeply nested set of arrays', () => {
+        const query = getTermsQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [
+            [null],
+            [
+              'host-name-1',
+              ['host-name-2', [null], ['host-name-3'], ['host-name-4', null, 'host-name-5']],
+            ],
+            ['host-name-6'],
+          ],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                {
+                  terms: {
+                    _name: '1.0',
+                    ip: ['host-name-1', 'host-name-2', 'host-name-3', 'host-name-4', 'host-name-5'],
+                  },
+                },
+                { terms: { _name: '2.0', ip: ['host-name-6'] } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
     });
-    const expected: QueryFilterType = [
-      { term: { list_id: 'list-123' } },
-      { terms: { keyword: ['host-name-1'] } },
-    ];
-    expect(queryFilter).toEqual(expected);
   });
 
-  test('it returns two keywords if given two values', () => {
-    const queryFilter = getQueryFilterFromTypeValue({
-      listId: 'list-123',
-      type: 'keyword',
-      value: ['host-name-1', 'host-name-2'],
+  describe('getTextQuery', () => {
+    describe('scalar values', () => {
+      test('it returns a expected terms query give a single string value, listId, and type', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: ['127.0.0.1'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected terms query given two string values, listId, and type', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: ['127.0.0.1', '127.0.0.2'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } },
+                { match: { ip: { _name: '1.0', operator: 'and', query: '127.0.0.2' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected numeric terms without converting them into strings', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [5, 3],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } },
+                { match: { ip: { _name: '1.0', operator: 'and', query: 3 } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns a string and a numeric without converting them into a homogenous type', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [5, '3'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } },
+                { match: { ip: { _name: '1.0', operator: 'and', query: '3' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out a null value if mixed with a string value', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [null, 'host-name-1'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out an object value if mixed with a string value', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [{}, 'host-name-1'],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+    });
+
+    describe('array values', () => {
+      test('it returns a expected terms query give a single string value, listId, and type', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [['127.0.0.1']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected terms query given two string values, listId, and type', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [['127.0.0.1'], ['127.0.0.2']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '0.0', operator: 'and', query: '127.0.0.1' } } },
+                { match: { ip: { _name: '1.0', operator: 'and', query: '127.0.0.2' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns two expected numeric terms without converting them into strings', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[5], [3]],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } },
+                { match: { ip: { _name: '1.0', operator: 'and', query: 3 } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it returns a string and a numeric without converting them into a homogenous type', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[5], ['3']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '0.0', operator: 'and', query: 5 } } },
+                { match: { ip: { _name: '1.0', operator: 'and', query: '3' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out a null value if mixed with a string value', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[null], ['host-name-1']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it filters out a object value if mixed with a string value', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [[{}], ['host-name-1']],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } }],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
+
+      test('it flattens and removes null values correctly in a deeply nested set of arrays', () => {
+        const query = getTextQuery({
+          listId: 'list-123',
+          type: 'ip',
+          value: [
+            [null],
+            [
+              'host-name-1',
+              ['host-name-2', [null], ['host-name-3'], ['host-name-4', null, 'host-name-5']],
+            ],
+            ['host-name-6'],
+          ],
+        });
+        const expected: QueryFilterType = [
+          { term: { list_id: 'list-123' } },
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [
+                { match: { ip: { _name: '1.0', operator: 'and', query: 'host-name-1' } } },
+                { match: { ip: { _name: '1.1', operator: 'and', query: 'host-name-2' } } },
+                { match: { ip: { _name: '1.2', operator: 'and', query: 'host-name-3' } } },
+                { match: { ip: { _name: '1.3', operator: 'and', query: 'host-name-4' } } },
+                { match: { ip: { _name: '1.4', operator: 'and', query: 'host-name-5' } } },
+                { match: { ip: { _name: '2.0', operator: 'and', query: 'host-name-6' } } },
+              ],
+            },
+          },
+        ];
+        expect(query).toEqual(expected);
+      });
     });
-    const expected: QueryFilterType = [
-      { term: { list_id: 'list-123' } },
-      { terms: { keyword: ['host-name-1', 'host-name-2'] } },
-    ];
-    expect(queryFilter).toEqual(expected);
   });
 
-  test('it returns an empty keyword given an empty value', () => {
-    const queryFilter = getQueryFilterFromTypeValue({
-      listId: 'list-123',
-      type: 'keyword',
-      value: [],
+  describe('getShouldQuery', () => {
+    test('it returns a should as-is when passed one', () => {
+      const query = getShouldQuery({
+        listId: 'list-123',
+        should: [
+          {
+            bool: {
+              minimum_should_match: 1,
+              should: [{ terms: { _name: '0.0', ip: ['127.0.0.1'] } }],
+            },
+          },
+        ],
+      });
+      const expected: QueryFilterType = [
+        { term: { list_id: 'list-123' } },
+        {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                bool: {
+                  minimum_should_match: 1,
+                  should: [{ terms: { _name: '0.0', ip: ['127.0.0.1'] } }],
+                },
+              },
+            ],
+          },
+        },
+      ];
+      expect(query).toEqual(expected);
     });
-    const expected: QueryFilterType = [
-      { term: { list_id: 'list-123' } },
-      { terms: { keyword: [] } },
-    ];
-    expect(queryFilter).toEqual(expected);
-  });
-
-  test('it returns an empty ip given an empty value', () => {
-    const queryFilter = getQueryFilterFromTypeValue({
-      listId: 'list-123',
-      type: 'ip',
-      value: [],
-    });
-    const expected: QueryFilterType = [{ term: { list_id: 'list-123' } }, { terms: { ip: [] } }];
-    expect(queryFilter).toEqual(expected);
   });
 });

x-pack/plugins/lists/server/services/utils/get_query_filter_from_type_value.ts

@@ -4,19 +4,170 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { isEmpty, isObject } from 'lodash/fp';
+
 import { Type } from '../../../common/schemas';
 
 export type QueryFilterType = [
-  { term: Record<string, string> },
-  { terms: Record<string, string[]> }
+  { term: Record<string, unknown> },
+  { terms: Record<string, unknown[]> } | { bool: {} }
 ];
 
+/**
+ * Given a type, value, and listId, this will return a valid query. If the type is
+ * "text" it will return a "text" match, otherwise it returns a terms query. If an
+ * array or array of arrays is passed, this will flatten, remove any "null" values,
+ * and then the result.
+ * @param type The type of list
+ * @param value The unknown value
+ * @param listId The list id
+ */
 export const getQueryFilterFromTypeValue = ({
   type,
   value,
   listId,
 }: {
   type: Type;
-  value: string[];
+  value: unknown[];
   listId: string;
-}): QueryFilterType => [{ term: { list_id: listId } }, { terms: { [type]: value } }];
+}): QueryFilterType => {
+  const valueFlattened = value
+    .flat(Infinity)
+    .filter((singleValue) => singleValue != null && !isObject(singleValue));
+  if (isEmpty(valueFlattened)) {
+    return getEmptyQuery({ listId });
+  } else if (type === 'text') {
+    return getTextQuery({ listId, type, value });
+  } else {
+    return getTermsQuery({ listId, type, value });
+  }
+};
+
+/**
+ * Returns an empty named query that should not match anything
+ * @param listId The list id to associate with the empty query
+ */
+export const getEmptyQuery = ({ listId }: { listId: string }): QueryFilterType => [
+  { term: { list_id: listId } },
+  {
+    bool: {
+      minimum_should_match: 1,
+      should: [
+        {
+          match_none: {
+            _name: 'empty',
+          },
+        },
+      ],
+    },
+  },
+];
+
+/**
+ * Returns a terms query against a large value based list. If it detects that an array or item has a "null"
+ * value it will filter that value out. If it has arrays within arrays it will flatten those out as well.
+ * @param value The value which can be unknown
+ * @param type The list type type
+ * @param listId The list id
+ */
+export const getTermsQuery = ({
+  value,
+  type,
+  listId,
+}: {
+  value: unknown[];
+  type: Type;
+  listId: string;
+}): QueryFilterType => {
+  const should = value.reduce<unknown[]>((accum, item, index) => {
+    if (Array.isArray(item)) {
+      const itemFlattened = item
+        .flat(Infinity)
+        .filter((singleValue) => singleValue != null && !isObject(singleValue));
+      if (itemFlattened.length === 0) {
+        return accum;
+      } else {
+        return [...accum, { terms: { _name: `${index}.0`, [type]: itemFlattened } }];
+      }
+    } else {
+      if (item == null || isObject(item)) {
+        return accum;
+      } else {
+        return [...accum, { term: { [type]: { _name: `${index}.0`, value: item } } }];
+      }
+    }
+  }, []);
+  return getShouldQuery({ listId, should });
+};
+
+/**
+ * Returns a text query against a large value based list. If it detects that an array or item has a "null"
+ * value it will filter that value out. If it has arrays within arrays it will flatten those out as well.
+ * @param value The value which can be unknown
+ * @param type The list type type
+ * @param listId The list id
+ */
+export const getTextQuery = ({
+  value,
+  type,
+  listId,
+}: {
+  value: unknown[];
+  type: Type;
+  listId: string;
+}): QueryFilterType => {
+  const should = value.reduce<unknown[]>((accum, item, index) => {
+    if (Array.isArray(item)) {
+      const itemFlattened = item
+        .flat(Infinity)
+        .filter((singleValue) => singleValue != null && !isObject(singleValue));
+      if (itemFlattened.length === 0) {
+        return accum;
+      } else {
+        return [
+          ...accum,
+          ...itemFlattened.map((flatItem, secondIndex) => ({
+            match: {
+              [type]: { _name: `${index}.${secondIndex}`, operator: 'and', query: flatItem },
+            },
+          })),
+        ];
+      }
+    } else {
+      if (item == null || isObject(item)) {
+        return accum;
+      } else {
+        return [
+          ...accum,
+          { match: { [type]: { _name: `${index}.0`, operator: 'and', query: item } } },
+        ];
+      }
+    }
+  }, []);
+
+  return getShouldQuery({ listId, should });
+};
+
+/**
+ * Given an unknown should this constructs a simple bool and terms with the should
+ * clause/query.
+ * @param listId The list id to query against
+ * @param should The unknown should to construct the query against
+ */
+export const getShouldQuery = ({
+  listId,
+  should,
+}: {
+  listId: string;
+  should: unknown;
+}): QueryFilterType => {
+  return [
+    { term: { list_id: listId } },
+    {
+      bool: {
+        minimum_should_match: 1,
+        should,
+      },
+    },
+  ];
+};

x-pack/plugins/lists/server/services/utils/index.ts

@@ -15,6 +15,7 @@ export * from './get_search_after_with_tie_breaker';
 export * from './get_sort_with_tie_breaker';
 export * from './get_source_with_tie_breaker';
 export * from './scroll_to_start_page';
+export * from './transform_elastic_named_search_to_list_item';
 export * from './transform_elastic_to_list';
 export * from './transform_elastic_to_list_item';
 export * from './transform_list_item_to_elastic_query';

x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.test.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getSearchListItemResponseMock } from '../../../common/schemas/response/search_list_item_schema.mock';
+import { LIST_INDEX, LIST_ITEM_ID, TYPE, VALUE } from '../../../common/constants.mock';
+import {
+  getSearchEsListItemMock,
+  getSearchListItemMock,
+} from '../../../common/schemas/elastic_response/search_es_list_item_schema.mock';
+import { SearchListItemArraySchema } from '../../../common/schemas';
+
+import { transformElasticNamedSearchToListItem } from './transform_elastic_named_search_to_list_item';
+
+describe('transform_elastic_named_search_to_list_item', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('if given an empty array for values, it returns an empty array', () => {
+    const response = getSearchListItemMock();
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [],
+    });
+    const expected: SearchListItemArraySchema = [];
+    expect(queryFilter).toEqual(expected);
+  });
+
+  test('if given an empty array for hits, it returns an empty match', () => {
+    const response = getSearchListItemMock();
+    response.hits.hits = [];
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [VALUE],
+    });
+    const expected: SearchListItemArraySchema = [{ items: [], value: VALUE }];
+    expect(queryFilter).toEqual(expected);
+  });
+
+  test('it transforms a single elastic type to a search list item type', () => {
+    const response = getSearchListItemMock();
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [VALUE],
+    });
+    const expected: SearchListItemArraySchema = [getSearchListItemResponseMock()];
+    expect(queryFilter).toEqual(expected);
+  });
+
+  test('it transforms two elastic types to a search list item type', () => {
+    const response = getSearchListItemMock();
+    response.hits.hits = [
+      ...response.hits.hits,
+      {
+        _id: LIST_ITEM_ID,
+        _index: LIST_INDEX,
+        _score: 0,
+        _source: getSearchEsListItemMock(),
+        _type: '',
+        matched_queries: ['1.0'],
+      },
+    ];
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [VALUE, VALUE],
+    });
+    const expected: SearchListItemArraySchema = [
+      getSearchListItemResponseMock(),
+      getSearchListItemResponseMock(),
+    ];
+    expect(queryFilter).toEqual(expected);
+  });
+
+  test('it transforms only 1 elastic type to a search list item type if only 1 is found as a value', () => {
+    const response = getSearchListItemMock();
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [VALUE, '127.0.0.2'],
+    });
+    const expected: SearchListItemArraySchema = [
+      getSearchListItemResponseMock(),
+      { items: [], value: '127.0.0.2' },
+    ];
+    expect(queryFilter).toEqual(expected);
+  });
+
+  test('it attaches two found results if the value is found in two hits from Elastic Search', () => {
+    const response = getSearchListItemMock();
+    response.hits.hits = [
+      ...response.hits.hits,
+      {
+        _id: LIST_ITEM_ID,
+        _index: LIST_INDEX,
+        _score: 0,
+        _source: getSearchEsListItemMock(),
+        _type: '',
+        matched_queries: ['0.0'],
+      },
+    ];
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [VALUE],
+    });
+    const {
+      items: [firstItem],
+      value,
+    } = getSearchListItemResponseMock();
+    const expected: SearchListItemArraySchema = [
+      {
+        items: [firstItem, firstItem],
+        value,
+      },
+    ];
+    expect(queryFilter).toEqual(expected);
+  });
+
+  test('it will return an empty array if no values are passed in', () => {
+    const response = getSearchListItemMock();
+    response.hits.hits = [
+      ...response.hits.hits,
+      {
+        _id: LIST_ITEM_ID,
+        _index: LIST_INDEX,
+        _score: 0,
+        _source: getSearchEsListItemMock(),
+        _type: '',
+        matched_queries: ['1.0'],
+      },
+    ];
+    const queryFilter = transformElasticNamedSearchToListItem({
+      response,
+      type: TYPE,
+      value: [],
+    });
+    expect(queryFilter).toEqual([]);
+  });
+});

x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SearchResponse } from 'elasticsearch';
+
+import { SearchEsListItemSchema, SearchListItemArraySchema, Type } from '../../../common/schemas';
+
+import { transformElasticHitsToListItem } from './transform_elastic_to_list_item';
+
+export interface TransformElasticMSearchToListItemOptions {
+  response: SearchResponse<SearchEsListItemSchema>;
+  type: Type;
+  value: unknown[];
+}
+
+/**
+ * Given an Elasticsearch response this will look to see if the named query matches the
+ * index found. The named query will have to be in the format of, "1.0", "1.1", "2.0" where the
+ * major number "1,2,n" will match with the index.
+ * Ref: https://www.elastic.co/guide/en/elasticsearch//reference/7.9/query-dsl-bool-query.html#named-queries
+ * @param response The elastic response
+ * @param type The list type
+ * @param value The values to check against the named queries.
+ */
+export const transformElasticNamedSearchToListItem = ({
+  response,
+  type,
+  value,
+}: TransformElasticMSearchToListItemOptions): SearchListItemArraySchema => {
+  return value.map((singleValue, index) => {
+    const matchingHits = response.hits.hits.filter((hit) => {
+      if (hit.matched_queries != null) {
+        return hit.matched_queries.some((matchedQuery) => {
+          const [matchedQueryIndex] = matchedQuery.split('.');
+          return matchedQueryIndex === `${index}`;
+        });
+      } else {
+        return false;
+      }
+    });
+    const items = transformElasticHitsToListItem({ hits: matchingHits, type });
+    return {
+      items,
+      value: singleValue,
+    };
+  });
+};

x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.test.ts

@@ -8,7 +8,10 @@ import { getSearchListItemMock } from '../../../common/schemas/elastic_response/
 import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock';
 import { ListItemArraySchema } from '../../../common/schemas';
 
-import { transformElasticToListItem } from './transform_elastic_to_list_item';
+import {
+  transformElasticHitsToListItem,
+  transformElasticToListItem,
+} from './transform_elastic_to_list_item';
 
 describe('transform_elastic_to_list_item', () => {
   beforeEach(() => {
@@ -19,28 +22,61 @@ describe('transform_elastic_to_list_item', () => {
     jest.clearAllMocks();
   });
 
-  test('it transforms an elastic type to a list item type', () => {
-    const response = getSearchListItemMock();
-    const queryFilter = transformElasticToListItem({
-      response,
-      type: 'ip',
+  describe('transformElasticToListItem', () => {
+    test('it transforms an elastic type to a list item type', () => {
+      const response = getSearchListItemMock();
+      const queryFilter = transformElasticToListItem({
+        response,
+        type: 'ip',
+      });
+      const expected: ListItemArraySchema = [getListItemResponseMock()];
+      expect(queryFilter).toEqual(expected);
+    });
+
+    test('it transforms an elastic keyword type to a list item type', () => {
+      const response = getSearchListItemMock();
+      response.hits.hits[0]._source.ip = undefined;
+      response.hits.hits[0]._source.keyword = 'host-name-example';
+      const queryFilter = transformElasticToListItem({
+        response,
+        type: 'keyword',
+      });
+      const listItemResponse = getListItemResponseMock();
+      listItemResponse.type = 'keyword';
+      listItemResponse.value = 'host-name-example';
+      const expected: ListItemArraySchema = [listItemResponse];
+      expect(queryFilter).toEqual(expected);
     });
-    const expected: ListItemArraySchema = [getListItemResponseMock()];
-    expect(queryFilter).toEqual(expected);
   });
 
-  test('it transforms an elastic keyword type to a list item type', () => {
-    const response = getSearchListItemMock();
-    response.hits.hits[0]._source.ip = undefined;
-    response.hits.hits[0]._source.keyword = 'host-name-example';
-    const queryFilter = transformElasticToListItem({
-      response,
-      type: 'keyword',
+  describe('transformElasticHitsToListItem', () => {
+    test('it transforms an elastic type to a list item type', () => {
+      const {
+        hits: { hits },
+      } = getSearchListItemMock();
+      const queryFilter = transformElasticHitsToListItem({
+        hits,
+        type: 'ip',
+      });
+      const expected: ListItemArraySchema = [getListItemResponseMock()];
+      expect(queryFilter).toEqual(expected);
+    });
+
+    test('it transforms an elastic keyword type to a list item type', () => {
+      const {
+        hits: { hits },
+      } = getSearchListItemMock();
+      hits[0]._source.ip = undefined;
+      hits[0]._source.keyword = 'host-name-example';
+      const queryFilter = transformElasticHitsToListItem({
+        hits,
+        type: 'keyword',
+      });
+      const listItemResponse = getListItemResponseMock();
+      listItemResponse.type = 'keyword';
+      listItemResponse.value = 'host-name-example';
+      const expected: ListItemArraySchema = [listItemResponse];
+      expect(queryFilter).toEqual(expected);
     });
-    const listItemResponse = getListItemResponseMock();
-    listItemResponse.type = 'keyword';
-    listItemResponse.value = 'host-name-example';
-    const expected: ListItemArraySchema = [listItemResponse];
-    expect(queryFilter).toEqual(expected);
   });
 });

x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts

@@ -17,11 +17,23 @@ export interface TransformElasticToListItemOptions {
   type: Type;
 }
 
+export interface TransformElasticHitToListItemOptions {
+  hits: SearchResponse<SearchEsListItemSchema>['hits']['hits'];
+  type: Type;
+}
+
 export const transformElasticToListItem = ({
   response,
   type,
 }: TransformElasticToListItemOptions): ListItemArraySchema => {
-  return response.hits.hits.map((hit) => {
+  return transformElasticHitsToListItem({ hits: response.hits.hits, type });
+};
+
+export const transformElasticHitsToListItem = ({
+  hits,
+  type,
+}: TransformElasticHitToListItemOptions): ListItemArraySchema => {
+  return hits.map((hit) => {
     const {
       _id,
       _source: {

x-pack/plugins/security_solution/public/common/components/autocomplete/field_value_lists.tsx

@@ -9,7 +9,7 @@ import { EuiFormRow, EuiComboBoxOptionOption, EuiComboBox } from '@elastic/eui';
 import { IFieldType } from '../../../../../../../src/plugins/data/common';
 import { useFindLists, ListSchema } from '../../../lists_plugin_deps';
 import { useKibana } from '../../../common/lib/kibana';
-import { getGenericComboBoxProps } from './helpers';
+import { filterFieldToList, getGenericComboBoxProps } from './helpers';
 import * as i18n from './translations';
 
 interface AutocompleteFieldListsProps {
@@ -41,17 +41,10 @@ export const AutocompleteFieldListsComponent: React.FC<AutocompleteFieldListsPro
   const { loading, result, start } = useFindLists();
   const getLabel = useCallback(({ name }) => name, []);
 
-  const optionsMemo = useMemo(() => {
-    if (
-      selectedField != null &&
-      selectedField.esTypes != null &&
-      selectedField.esTypes.length > 0
-    ) {
-      return lists.filter(({ type }) => selectedField.esTypes?.includes(type));
-    } else {
-      return [];
-    }
-  }, [lists, selectedField]);
+  const optionsMemo = useMemo(() => filterFieldToList(lists, selectedField), [
+    lists,
+    selectedField,
+  ]);
   const selectedOptionsMemo = useMemo(() => {
     if (selectedValue != null) {
       const list = lists.filter(({ id }) => id === selectedValue);

x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.test.ts

@@ -6,6 +6,7 @@
 import moment from 'moment';
 import '../../../common/mock/match_media';
 import { getField } from '../../../../../../../src/plugins/data/common/index_patterns/fields/fields.mocks';
+import { IFieldType } from '../../../../../../../src/plugins/data/common';
 
 import * as i18n from './translations';
 import {
@@ -15,7 +16,16 @@ import {
   existsOperator,
   doesNotExistOperator,
 } from './operators';
-import { getOperators, checkEmptyValue, paramIsValid, getGenericComboBoxProps } from './helpers';
+import {
+  getOperators,
+  checkEmptyValue,
+  paramIsValid,
+  getGenericComboBoxProps,
+  typeMatch,
+  filterFieldToList,
+} from './helpers';
+import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
+import { ListSchema } from '../../../../../lists/common';
 
 describe('helpers', () => {
   // @ts-ignore
@@ -260,4 +270,117 @@ describe('helpers', () => {
       });
     });
   });
+
+  describe('#typeMatch', () => {
+    test('ip -> ip is true', () => {
+      expect(typeMatch('ip', 'ip')).toEqual(true);
+    });
+
+    test('keyword -> keyword is true', () => {
+      expect(typeMatch('keyword', 'keyword')).toEqual(true);
+    });
+
+    test('text -> text is true', () => {
+      expect(typeMatch('text', 'text')).toEqual(true);
+    });
+
+    test('ip_range -> ip is true', () => {
+      expect(typeMatch('ip_range', 'ip')).toEqual(true);
+    });
+
+    test('date_range -> date is true', () => {
+      expect(typeMatch('date_range', 'date')).toEqual(true);
+    });
+
+    test('double_range -> double is true', () => {
+      expect(typeMatch('double_range', 'double')).toEqual(true);
+    });
+
+    test('float_range -> float is true', () => {
+      expect(typeMatch('float_range', 'float')).toEqual(true);
+    });
+
+    test('integer_range -> integer is true', () => {
+      expect(typeMatch('integer_range', 'integer')).toEqual(true);
+    });
+
+    test('long_range -> long is true', () => {
+      expect(typeMatch('long_range', 'long')).toEqual(true);
+    });
+
+    test('ip -> date is false', () => {
+      expect(typeMatch('ip', 'date')).toEqual(false);
+    });
+
+    test('long -> float is false', () => {
+      expect(typeMatch('long', 'float')).toEqual(false);
+    });
+
+    test('integer -> long is false', () => {
+      expect(typeMatch('integer', 'long')).toEqual(false);
+    });
+  });
+
+  describe('#filterFieldToList', () => {
+    test('it returns empty array if given a undefined for field', () => {
+      const filter = filterFieldToList([], undefined);
+      expect(filter).toEqual([]);
+    });
+
+    test('it returns empty array if filed does not contain esTypes', () => {
+      const field: IFieldType = { name: 'some-name', type: 'some-type' };
+      const filter = filterFieldToList([], field);
+      expect(filter).toEqual([]);
+    });
+
+    test('it returns single filtered list of ip_range -> ip', () => {
+      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
+      const listItem: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
+      const filter = filterFieldToList([listItem], field);
+      const expected: ListSchema[] = [listItem];
+      expect(filter).toEqual(expected);
+    });
+
+    test('it returns single filtered list of ip -> ip', () => {
+      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
+      const listItem: ListSchema = { ...getListResponseMock(), type: 'ip' };
+      const filter = filterFieldToList([listItem], field);
+      const expected: ListSchema[] = [listItem];
+      expect(filter).toEqual(expected);
+    });
+
+    test('it returns single filtered list of keyword -> keyword', () => {
+      const field: IFieldType = { name: 'some-name', type: 'keyword', esTypes: ['keyword'] };
+      const listItem: ListSchema = { ...getListResponseMock(), type: 'keyword' };
+      const filter = filterFieldToList([listItem], field);
+      const expected: ListSchema[] = [listItem];
+      expect(filter).toEqual(expected);
+    });
+
+    test('it returns single filtered list of text -> text', () => {
+      const field: IFieldType = { name: 'some-name', type: 'text', esTypes: ['text'] };
+      const listItem: ListSchema = { ...getListResponseMock(), type: 'text' };
+      const filter = filterFieldToList([listItem], field);
+      const expected: ListSchema[] = [listItem];
+      expect(filter).toEqual(expected);
+    });
+
+    test('it returns 2 filtered lists of ip_range -> ip', () => {
+      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
+      const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
+      const listItem2: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
+      const filter = filterFieldToList([listItem1, listItem2], field);
+      const expected: ListSchema[] = [listItem1, listItem2];
+      expect(filter).toEqual(expected);
+    });
+
+    test('it returns 1 filtered lists of ip_range -> ip if the 2nd is not compatible type', () => {
+      const field: IFieldType = { name: 'some-name', type: 'ip', esTypes: ['ip'] };
+      const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
+      const listItem2: ListSchema = { ...getListResponseMock(), type: 'text' };
+      const filter = filterFieldToList([listItem1, listItem2], field);
+      const expected: ListSchema[] = [listItem1];
+      expect(filter).toEqual(expected);
+    });
+  });
 });

x-pack/plugins/security_solution/public/common/components/autocomplete/helpers.ts

@@ -18,6 +18,7 @@ import {
 } from './operators';
 import { GetGenericComboBoxPropsReturn, OperatorOption } from './types';
 import * as i18n from './translations';
+import { ListSchema, Type } from '../../../lists_plugin_deps';
 
 /**
  * Returns the appropriate operators given a field type
@@ -138,3 +139,36 @@ export function getGenericComboBoxProps<T>({
     selectedComboOptions: newSelectedComboOptions,
   };
 }
+
+/**
+ * Given an array of lists and optionally a field this will return all
+ * the lists that match against the field based on the types from the field
+ * @param lists The lists to match against the field
+ * @param field The field to check against the list to see if they are compatible
+ */
+export const filterFieldToList = (lists: ListSchema[], field?: IFieldType): ListSchema[] => {
+  if (field != null) {
+    const { esTypes = [] } = field;
+    return lists.filter(({ type }) => esTypes.some((esType) => typeMatch(type, esType)));
+  } else {
+    return [];
+  }
+};
+
+/**
+ * Given an input list type and a string based ES type this will match
+ * if they're exact or if they are compatible with a range
+ * @param type The type to match against the esType
+ * @param esType The ES type to match with
+ */
+export const typeMatch = (type: Type, esType: string): boolean => {
+  return (
+    type === esType ||
+    (type === 'ip_range' && esType === 'ip') ||
+    (type === 'date_range' && esType === 'date') ||
+    (type === 'double_range' && esType === 'double') ||
+    (type === 'float_range' && esType === 'float') ||
+    (type === 'integer_range' && esType === 'integer') ||
+    (type === 'long_range' && esType === 'long')
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts

@@ -1,216 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-import { get } from 'lodash/fp';
-import { Logger } from 'src/core/server';
-
-import { ListClient } from '../../../../../lists/server';
-import { BuildRuleMessage } from './rule_messages';
-import {
-  EntryList,
-  ExceptionListItemSchema,
-  entriesList,
-  Type,
-} from '../../../../../lists/common/schemas';
-import { hasLargeValueList } from '../../../../common/detection_engine/utils';
-import { SearchTypes } from '../../../../common/detection_engine/types';
-import { SearchResponse } from '../../types';
-
-// narrow unioned type to be single
-const isStringableType = (val: SearchTypes): val is string | number | boolean =>
-  ['string', 'number', 'boolean'].includes(typeof val);
-
-const isStringableArray = (val: SearchTypes): val is Array<string | number | boolean> => {
-  if (!Array.isArray(val)) {
-    return false;
-  }
-  // TS does not allow .every to be called on val as-is, even though every type in the union
-  // is an array. https://github.com/microsoft/TypeScript/issues/36390
-  // @ts-expect-error
-  return val.every((subVal) => isStringableType(subVal));
-};
-
-export const createSetToFilterAgainst = async <T>({
-  events,
-  field,
-  listId,
-  listType,
-  listClient,
-  logger,
-  buildRuleMessage,
-}: {
-  events: SearchResponse<T>['hits']['hits'];
-  field: string;
-  listId: string;
-  listType: Type;
-  listClient: ListClient;
-  logger: Logger;
-  buildRuleMessage: BuildRuleMessage;
-}): Promise<Set<SearchTypes>> => {
-  const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => {
-    const valueField = get(field, searchResultItem._source);
-    if (valueField != null) {
-      if (isStringableType(valueField)) {
-        acc.add(valueField.toString());
-      } else if (isStringableArray(valueField)) {
-        valueField.forEach((subVal) => acc.add(subVal.toString()));
-      }
-    }
-    return acc;
-  }, new Set<string>());
-  logger.debug(
-    `number of distinct values from ${field}: ${[...valuesFromSearchResultField].length}`
-  );
-
-  // matched will contain any list items that matched with the
-  // values passed in from the Set.
-  const matchedListItems = await listClient.getListItemByValues({
-    listId,
-    type: listType,
-    value: [...valuesFromSearchResultField],
-  });
-
-  logger.debug(`number of matched items from list with id ${listId}: ${matchedListItems.length}`);
-  // create a set of list values that were a hit - easier to work with
-  const matchedListItemsSet = new Set<SearchTypes>(matchedListItems.map((item) => item.value));
-  return matchedListItemsSet;
-};
-
-export const filterEventsAgainstList = async <T>({
-  listClient,
-  exceptionsList,
-  logger,
-  eventSearchResult,
-  buildRuleMessage,
-}: {
-  listClient: ListClient;
-  exceptionsList: ExceptionListItemSchema[];
-  logger: Logger;
-  eventSearchResult: SearchResponse<T>;
-  buildRuleMessage: BuildRuleMessage;
-}): Promise<SearchResponse<T>> => {
-  try {
-    if (exceptionsList == null || exceptionsList.length === 0) {
-      logger.debug(buildRuleMessage('about to return original search result'));
-      return eventSearchResult;
-    }
-
-    const exceptionItemsWithLargeValueLists = exceptionsList.reduce<ExceptionListItemSchema[]>(
-      (acc, exception) => {
-        const { entries } = exception;
-        if (hasLargeValueList(entries)) {
-          return [...acc, exception];
-        }
-
-        return acc;
-      },
-      []
-    );
-
-    if (exceptionItemsWithLargeValueLists.length === 0) {
-      logger.debug(
-        buildRuleMessage('no exception items of type list found - returning original search result')
-      );
-      return eventSearchResult;
-    }
-
-    const valueListExceptionItems = exceptionsList.filter((listItem: ExceptionListItemSchema) => {
-      return listItem.entries.every((entry) => entriesList.is(entry));
-    });
-
-    // now that we have all the exception items which are value lists (whether single entry or have multiple entries)
-    const res = await valueListExceptionItems.reduce<Promise<SearchResponse<T>['hits']['hits']>>(
-      async (
-        filteredAccum: Promise<SearchResponse<T>['hits']['hits']>,
-        exceptionItem: ExceptionListItemSchema
-      ) => {
-        // 1. acquire the values from the specified fields to check
-        // e.g. if the value list is checking against source.ip, gather
-        // all the values for source.ip from the search response events.
-
-        // 2. search against the value list with the values found in the search result
-        // and see if there are any matches. For every match, add that value to a set
-        // that represents the "matched" values
-
-        // 3. filter the search result against the set from step 2 using the
-        // given operator (included vs excluded).
-        // acquire the list values we are checking for in the field.
-        const filtered = await filteredAccum;
-        const typedEntries = exceptionItem.entries.filter((entry): entry is EntryList =>
-          entriesList.is(entry)
-        );
-        const fieldAndSetTuples = await Promise.all(
-          typedEntries.map(async (entry) => {
-            const { list, field, operator } = entry;
-            const { id, type } = list;
-            const matchedSet = await createSetToFilterAgainst({
-              events: filtered,
-              field,
-              listId: id,
-              listType: type,
-              listClient,
-              logger,
-              buildRuleMessage,
-            });
-
-            return Promise.resolve({ field, operator, matchedSet });
-          })
-        );
-
-        // check if for each tuple, the entry is not in both for when two value list entries exist.
-        // need to re-write this as a reduce.
-        const filteredEvents = filtered.filter((item) => {
-          const vals = fieldAndSetTuples.map((tuple) => {
-            const eventItem = get(tuple.field, item._source);
-            if (tuple.operator === 'included') {
-              // only create a signal if the field value is not in the value list
-              if (eventItem != null) {
-                if (isStringableType(eventItem)) {
-                  return !tuple.matchedSet.has(eventItem);
-                } else if (isStringableArray(eventItem)) {
-                  return !eventItem.some((val) => tuple.matchedSet.has(val));
-                }
-              }
-              return true;
-            } else if (tuple.operator === 'excluded') {
-              // only create a signal if the field value is in the value list
-              if (eventItem != null) {
-                if (isStringableType(eventItem)) {
-                  return tuple.matchedSet.has(eventItem);
-                } else if (isStringableArray(eventItem)) {
-                  return eventItem.some((val) => tuple.matchedSet.has(val));
-                }
-              }
-              return true;
-            }
-            return false;
-          });
-          return vals.some((value) => value);
-        });
-        const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
-        logger.debug(
-          buildRuleMessage(`Exception with id ${exceptionItem.id} filtered out ${diff} events`)
-        );
-        const toReturn = filteredEvents;
-        return toReturn;
-      },
-      Promise.resolve<SearchResponse<T>['hits']['hits']>(eventSearchResult.hits.hits)
-    );
-
-    const toReturn: SearchResponse<T> = {
-      took: eventSearchResult.took,
-      timed_out: eventSearchResult.timed_out,
-      _shards: eventSearchResult._shards,
-      hits: {
-        total: res.length,
-        max_score: eventSearchResult.hits.max_score,
-        hits: res,
-      },
-    };
-    return toReturn;
-  } catch (exc) {
-    throw new Error(`Failed to query lists index. Reason: ${exc.message}`);
-  }
-};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts

@@ -0,0 +1,287 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { createFieldAndSetTuples } from './create_field_and_set_tuples';
+import { mockLogger, sampleDocWithSortId } from '../__mocks__/es_results';
+
+import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
+import { listMock } from '../../../../../../lists/server/mocks';
+import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock';
+import { EntryList } from '../../../../../../lists/common';
+import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock';
+
+describe('filterEventsAgainstList', () => {
+  let listClient = listMock.getListClient();
+  let exceptionItem = getExceptionListItemSchemaMock();
+  let events = [sampleDocWithSortId('123', '1.1.1.1')];
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    listClient = listMock.getListClient();
+    listClient.searchListItemByValues = jest.fn(({ value }) =>
+      Promise.resolve(
+        value.map((item) => ({
+          ...getSearchListItemResponseMock(),
+          value: item,
+        }))
+      )
+    );
+    exceptionItem = {
+      ...getExceptionListItemSchemaMock(),
+      entries: [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys.txt',
+            type: 'ip',
+          },
+        },
+      ],
+    };
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('it returns an empty array if exceptionItem entries are empty', async () => {
+    exceptionItem.entries = [];
+    const field = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(field).toEqual([]);
+  });
+
+  test('it returns a single field and set tuple if entries has a single item', async () => {
+    const field = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(field.length).toEqual(1);
+  });
+
+  test('it returns "included" if the operator is "included"', async () => {
+    (exceptionItem.entries[0] as EntryList).operator = 'included';
+    const [{ operator }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(operator).toEqual('included');
+  });
+
+  test('it returns "excluded" if the operator is "excluded"', async () => {
+    (exceptionItem.entries[0] as EntryList).operator = 'excluded';
+    const [{ operator }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(operator).toEqual('excluded');
+  });
+
+  test('it returns "field" if the "field is "source.ip"', async () => {
+    (exceptionItem.entries[0] as EntryList).field = 'source.ip';
+    const [{ field }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(field).toEqual('source.ip');
+  });
+
+  test('it returns a single matched set as a JSON.stringify() set from the "events"', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+    (exceptionItem.entries[0] as EntryList).field = 'source.ip';
+    const [{ matchedSet }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1')]);
+  });
+
+  test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
+    (exceptionItem.entries[0] as EntryList).field = 'source.ip';
+    const [{ matchedSet }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]);
+  });
+
+  test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => {
+    events = [sampleDocWithSortId('123', ['1.1.1.1', '2.2.2.2'])];
+    (exceptionItem.entries[0] as EntryList).field = 'source.ip';
+    const [{ matchedSet }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1', '2.2.2.2'])]);
+  });
+
+  test('it returns 2 fields when given two exception list items', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
+    exceptionItem.entries = [
+      {
+        field: 'source.ip',
+        operator: 'included',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+      {
+        field: 'destination.ip',
+        operator: 'excluded',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+    ];
+    const fields = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(fields.length).toEqual(2);
+  });
+
+  test('it returns two matched sets from two different events, one excluded, and one included', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
+    exceptionItem.entries = [
+      {
+        field: 'source.ip',
+        operator: 'included',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+      {
+        field: 'destination.ip',
+        operator: 'excluded',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+    ];
+    const [{ operator: operator1 }, { operator: operator2 }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(operator1).toEqual('included');
+    expect(operator2).toEqual('excluded');
+  });
+
+  test('it returns two fields from two different events', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
+    exceptionItem.entries = [
+      {
+        field: 'source.ip',
+        operator: 'included',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+      {
+        field: 'destination.ip',
+        operator: 'excluded',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+    ];
+    const [{ field: field1 }, { field: field2 }] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect(field1).toEqual('source.ip');
+    expect(field2).toEqual('destination.ip');
+  });
+
+  test('it returns two matches from two different events', async () => {
+    events = [
+      sampleDocWithSortId('123', '1.1.1.1', '3.3.3.3'),
+      sampleDocWithSortId('456', '2.2.2.2', '5.5.5.5'),
+    ];
+    exceptionItem.entries = [
+      {
+        field: 'source.ip',
+        operator: 'included',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+      {
+        field: 'destination.ip',
+        operator: 'excluded',
+        type: 'list',
+        list: {
+          id: 'ci-badguys.txt',
+          type: 'ip',
+        },
+      },
+    ];
+    const [
+      { matchedSet: matchedSet1 },
+      { matchedSet: matchedSet2 },
+    ] = await createFieldAndSetTuples({
+      listClient,
+      logger: mockLogger,
+      events,
+      exceptionItem,
+      buildRuleMessage,
+    });
+    expect([...matchedSet1]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]);
+    expect([...matchedSet2]).toEqual([JSON.stringify('3.3.3.3'), JSON.stringify('5.5.5.5')]);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { EntryList, entriesList } from '../../../../../../lists/common';
+import { createSetToFilterAgainst } from './create_set_to_filter_against';
+import { CreateFieldAndSetTuplesOptions, FieldSet } from './types';
+
+export const createFieldAndSetTuples = async <T>({
+  events,
+  exceptionItem,
+  listClient,
+  logger,
+  buildRuleMessage,
+}: CreateFieldAndSetTuplesOptions<T>): Promise<FieldSet[]> => {
+  const typedEntries = exceptionItem.entries.filter((entry): entry is EntryList =>
+    entriesList.is(entry)
+  );
+  const fieldAndSetTuples = await Promise.all(
+    typedEntries.map(async (entry) => {
+      const { list, field, operator } = entry;
+      const { id, type } = list;
+      const matchedSet = await createSetToFilterAgainst({
+        events,
+        field,
+        listId: id,
+        listType: type,
+        listClient,
+        logger,
+        buildRuleMessage,
+      });
+
+      return { field, operator, matchedSet };
+    })
+  );
+  return fieldAndSetTuples;
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { mockLogger, sampleDocWithSortId } from '../__mocks__/es_results';
+
+import { listMock } from '../../../../../../lists/server/mocks';
+import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock';
+import { createSetToFilterAgainst } from './create_set_to_filter_against';
+import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock';
+
+describe('createSetToFilterAgainst', () => {
+  let listClient = listMock.getListClient();
+  let events = [sampleDocWithSortId('123', '1.1.1.1')];
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    listClient = listMock.getListClient();
+    listClient.searchListItemByValues = jest.fn(({ value }) =>
+      Promise.resolve(
+        value.map((item) => ({
+          ...getSearchListItemResponseMock(),
+          value: item,
+        }))
+      )
+    );
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('it returns an empty array if list return is empty', async () => {
+    listClient.searchListItemByValues = jest.fn().mockResolvedValue([]);
+    const field = await createSetToFilterAgainst({
+      events,
+      field: 'source.ip',
+      listId: 'list-123',
+      listType: 'ip',
+      listClient,
+      logger: mockLogger,
+      buildRuleMessage,
+    });
+    expect([...field]).toEqual([]);
+  });
+
+  test('it returns 1 field if the list returns a single item', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+    const field = await createSetToFilterAgainst({
+      events,
+      field: 'source.ip',
+      listId: 'list-123',
+      listType: 'ip',
+      listClient,
+      logger: mockLogger,
+      buildRuleMessage,
+    });
+    expect(listClient.searchListItemByValues).toHaveBeenCalledWith({
+      listId: 'list-123',
+      type: 'ip',
+      value: ['1.1.1.1'],
+    });
+    expect([...field]).toEqual([JSON.stringify('1.1.1.1')]);
+  });
+
+  test('it returns 2 fields if the list returns 2 items', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
+    const field = await createSetToFilterAgainst({
+      events,
+      field: 'source.ip',
+      listId: 'list-123',
+      listType: 'ip',
+      listClient,
+      logger: mockLogger,
+      buildRuleMessage,
+    });
+    expect(listClient.searchListItemByValues).toHaveBeenCalledWith({
+      listId: 'list-123',
+      type: 'ip',
+      value: ['1.1.1.1', '2.2.2.2'],
+    });
+    expect([...field]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]);
+  });
+
+  test('it returns 0 fields if the field does not match up to a valid field within the event', async () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
+    const field = await createSetToFilterAgainst({
+      events,
+      field: 'nonexistent.field', // field does not exist
+      listId: 'list-123',
+      listType: 'ip',
+      listClient,
+      logger: mockLogger,
+      buildRuleMessage,
+    });
+    expect(listClient.searchListItemByValues).toHaveBeenCalledWith({
+      listId: 'list-123',
+      type: 'ip',
+      value: [],
+    });
+    expect([...field]).toEqual([]);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { get } from 'lodash/fp';
+import { CreateSetToFilterAgainstOptions } from './types';
+
+/**
+ * Creates a field set to filter against using the stringed version of the
+ * data type for compare. Creates a set of list values that are stringified that
+ * are easier to work with as well as ensures that deep values can work since it turns
+ * things into a string using JSON.stringify().
+ *
+ * @param events The events to filter against
+ * @param field The field checking against the list
+ * @param listId The list id for the list function call
+ * @param listType The type of list for the list function call
+ * @param listClient The list client API
+ * @param logger logger for errors, debug, etc...
+ */
+export const createSetToFilterAgainst = async <T>({
+  events,
+  field,
+  listId,
+  listType,
+  listClient,
+  logger,
+  buildRuleMessage,
+}: CreateSetToFilterAgainstOptions<T>): Promise<Set<unknown>> => {
+  const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => {
+    const valueField = get(field, searchResultItem._source);
+    if (valueField != null) {
+      acc.add(valueField);
+    }
+    return acc;
+  }, new Set<unknown>());
+
+  logger.debug(
+    buildRuleMessage(
+      `number of distinct values from ${field}: ${[...valuesFromSearchResultField].length}`
+    )
+  );
+
+  const matchedListItems = await listClient.searchListItemByValues({
+    listId,
+    type: listType,
+    value: [...valuesFromSearchResultField],
+  });
+
+  logger.debug(
+    buildRuleMessage(
+      `number of matched items from list with id ${listId}: ${matchedListItems.length}`
+    )
+  );
+
+  return new Set<unknown>(
+    matchedListItems
+      .filter((item) => item.items.length !== 0)
+      .map((item) => JSON.stringify(item.value))
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { sampleDocWithSortId } from '../__mocks__/es_results';
+
+import { listMock } from '../../../../../../lists/server/mocks';
+import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock';
+import { filterEvents } from './filter_events';
+import { FieldSet } from './types';
+
+describe('filterEvents', () => {
+  let listClient = listMock.getListClient();
+  let events = [sampleDocWithSortId('123', '1.1.1.1')];
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    listClient = listMock.getListClient();
+    listClient.searchListItemByValues = jest.fn(({ value }) =>
+      Promise.resolve(
+        value.map((item) => ({
+          ...getSearchListItemResponseMock(),
+          value: item,
+        }))
+      )
+    );
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('it filters out the event if it is "included"', () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+    const fieldAndSetTuples: FieldSet[] = [
+      {
+        field: 'source.ip',
+        operator: 'included',
+        matchedSet: new Set([JSON.stringify('1.1.1.1')]),
+      },
+    ];
+    const field = filterEvents({
+      events,
+      fieldAndSetTuples,
+    });
+    expect([...field]).toEqual([]);
+  });
+
+  test('it does not filter out the event if it is "excluded"', () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+    const fieldAndSetTuples: FieldSet[] = [
+      {
+        field: 'source.ip',
+        operator: 'excluded',
+        matchedSet: new Set([JSON.stringify('1.1.1.1')]),
+      },
+    ];
+    const field = filterEvents({
+      events,
+      fieldAndSetTuples,
+    });
+    expect([...field]).toEqual(events);
+  });
+
+  test('it does NOT filter out the event if the field is not found', () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1')];
+    const fieldAndSetTuples: FieldSet[] = [
+      {
+        field: 'madeup.nonexistent', // field does not exist
+        operator: 'included',
+        matchedSet: new Set([JSON.stringify('1.1.1.1')]),
+      },
+    ];
+    const field = filterEvents({
+      events,
+      fieldAndSetTuples,
+    });
+    expect([...field]).toEqual(events);
+  });
+
+  test('it does NOT filter out the event if it is in both an inclusion and exclusion list', () => {
+    events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
+    const fieldAndSetTuples: FieldSet[] = [
+      {
+        field: 'source.ip',
+        operator: 'included',
+        matchedSet: new Set([JSON.stringify('1.1.1.1')]),
+      },
+      {
+        field: 'source.ip',
+        operator: 'excluded',
+        matchedSet: new Set([JSON.stringify('1.1.1.1')]),
+      },
+    ];
+
+    const field = filterEvents({
+      events,
+      fieldAndSetTuples,
+    });
+    expect([...field]).toEqual(events);
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { get } from 'lodash/fp';
+import { SearchResponse } from '../../../types';
+import { FilterEventsOptions } from './types';
+
+/**
+ * Check if for each tuple, the entry is not in both for when two or more value list entries exist.
+ * If the entry is in both an inclusion and exclusion list it will not be filtered out.
+ * @param events The events to check against
+ * @param fieldAndSetTuples The field and set tuples
+ */
+export const filterEvents = <T>({
+  events,
+  fieldAndSetTuples,
+}: FilterEventsOptions<T>): SearchResponse<T>['hits']['hits'] => {
+  return events.filter((item) => {
+    return fieldAndSetTuples
+      .map((tuple) => {
+        const eventItem = get(tuple.field, item._source);
+        if (eventItem == null) {
+          return true;
+        } else if (tuple.operator === 'included') {
+          // only create a signal if the event is not in the value list
+          return !tuple.matchedSet.has(JSON.stringify(eventItem));
+        } else if (tuple.operator === 'excluded') {
+          // only create a signal if the event is in the value list
+          return tuple.matchedSet.has(JSON.stringify(eventItem));
+        } else {
+          return false;
+        }
+      })
+      .some((value) => value);
+  });
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.test.ts

@@ -5,27 +5,27 @@
  */
 
 import uuid from 'uuid';
-import { filterEventsAgainstList } from './filter_events_with_list';
-import { buildRuleMessageFactory } from './rule_messages';
-import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results';
+import { filterEventsAgainstList } from './filter_events_against_list';
+import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock';
+import { mockLogger, repeatedSearchResultsWithSortId } from '../__mocks__/es_results';
 
-import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
-import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock';
-import { listMock } from '../../../../../lists/server/mocks';
+import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
+import { listMock } from '../../../../../../lists/server/mocks';
+import { getSearchListItemResponseMock } from '../../../../../../lists/common/schemas/response/search_list_item_schema.mock';
 
 const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4());
-const buildRuleMessage = buildRuleMessageFactory({
-  id: 'fake id',
-  ruleId: 'fake rule id',
-  index: 'fakeindex',
-  name: 'fake name',
-});
+
 describe('filterEventsAgainstList', () => {
   let listClient = listMock.getListClient();
+
   beforeEach(() => {
     jest.clearAllMocks();
     listClient = listMock.getListClient();
-    listClient.getListItemByValues = jest.fn().mockResolvedValue([]);
+    listClient.searchListItemByValues = jest.fn().mockResolvedValue([]);
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
   });
 
   it('should respond with eventSearchResult if exceptionList is empty array', async () => {
@@ -87,6 +87,7 @@ describe('filterEventsAgainstList', () => {
       });
       expect(res.hits.hits.length).toEqual(4);
     });
+
     it('should respond with less items in the list if some values match', async () => {
       const exceptionItem = getExceptionListItemSchemaMock();
       exceptionItem.entries = [
@@ -100,10 +101,10 @@ describe('filterEventsAgainstList', () => {
           },
         },
       ];
-      listClient.getListItemByValues = jest.fn(({ value }) =>
+      listClient.searchListItemByValues = jest.fn(({ value }) =>
         Promise.resolve(
           value.slice(0, 2).map((item) => ({
-            ...getListItemResponseMock(),
+            ...getSearchListItemResponseMock(),
             value: item,
           }))
         )
@@ -120,8 +121,8 @@ describe('filterEventsAgainstList', () => {
         ]),
         buildRuleMessage,
       });
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip');
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual(
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip');
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual(
         'ci-badguys.txt'
       );
       expect(res.hits.hits.length).toEqual(2);
@@ -159,13 +160,13 @@ describe('filterEventsAgainstList', () => {
       ];
 
       // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '2.2.2.2' },
-        { ...getListItemResponseMock(), value: '4.4.4.4' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
+        { ...getSearchListItemResponseMock(), value: '4.4.4.4' },
       ]);
       // this call represents an exception list with a value list containing ['6.6.6.6']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '6.6.6.6' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: '6.6.6.6' },
       ]);
 
       const res = await filterEventsAgainstList({
@@ -185,7 +186,7 @@ describe('filterEventsAgainstList', () => {
         ]),
         buildRuleMessage,
       });
-      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
       expect(res.hits.hits.length).toEqual(6);
 
       // @ts-expect-error
@@ -221,12 +222,12 @@ describe('filterEventsAgainstList', () => {
       ];
 
       // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
       ]);
       // this call represents an exception list with a value list containing ['6.6.6.6']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '6.6.6.6' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: '6.6.6.6' },
       ]);
 
       const res = await filterEventsAgainstList({
@@ -246,7 +247,7 @@ describe('filterEventsAgainstList', () => {
         ]),
         buildRuleMessage,
       });
-      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
       // @ts-expect-error
       const ipVals = res.hits.hits.map((item) => item._source.source.ip);
       expect(res.hits.hits.length).toEqual(7);
@@ -280,12 +281,12 @@ describe('filterEventsAgainstList', () => {
       ];
 
       // this call represents an exception list with a value list containing ['2.2.2.2']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
       ]);
       // this call represents an exception list with a value list containing ['4.4.4.4']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '4.4.4.4' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: '4.4.4.4' },
       ]);
 
       const res = await filterEventsAgainstList({
@@ -321,7 +322,7 @@ describe('filterEventsAgainstList', () => {
         ),
         buildRuleMessage,
       });
-      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
       expect(res.hits.hits.length).toEqual(8);
 
       // @ts-expect-error
@@ -362,8 +363,8 @@ describe('filterEventsAgainstList', () => {
       ];
 
       // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValue([
-        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValue([
+        { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
       ]);
 
       const res = await filterEventsAgainstList({
@@ -383,7 +384,7 @@ describe('filterEventsAgainstList', () => {
         ]),
         buildRuleMessage,
       });
-      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
       expect(res.hits.hits.length).toEqual(9);
 
       // @ts-expect-error
@@ -401,7 +402,7 @@ describe('filterEventsAgainstList', () => {
       ]).toEqual(ipVals);
     });
 
-    it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => {
+    it('should respond with same items in the list given one exception item with two entries of type list and array of values in document', async () => {
       const exceptionItem = getExceptionListItemSchemaMock();
       exceptionItem.entries = [
         {
@@ -425,12 +426,12 @@ describe('filterEventsAgainstList', () => {
       ];
 
       // this call represents an exception list with a value list containing ['2.2.2.2']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: ['2.2.2.2', '3.3.3.3'] },
       ]);
       // this call represents an exception list with a value list containing ['4.4.4.4']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '4.4.4.4' },
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: ['3.3.3.3', '4.4.4.4'] },
       ]);
 
       const res = await filterEventsAgainstList({
@@ -454,17 +455,16 @@ describe('filterEventsAgainstList', () => {
         ),
         buildRuleMessage,
       });
-      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
-        '1.1.1.1',
-        '2.2.2.2',
-        '3.3.3.3',
+      expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
+        ['1.1.1.1', '1.1.1.1'],
+        ['1.1.1.1', '2.2.2.2'],
+        ['2.2.2.2', '3.3.3.3'],
       ]);
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
-        '1.1.1.1',
-        '2.2.2.2',
-        '3.3.3.3',
-        '4.4.4.4',
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
+        ['1.1.1.1', '2.2.2.2'],
+        ['2.2.2.2', '3.3.3.3'],
+        ['3.3.3.3', '4.4.4.4'],
       ]);
       expect(res.hits.hits.length).toEqual(2);
 
@@ -505,6 +505,7 @@ describe('filterEventsAgainstList', () => {
       });
       expect(res.hits.hits.length).toEqual(0);
     });
+
     it('should respond with less items in the list if some values match', async () => {
       const exceptionItem = getExceptionListItemSchemaMock();
       exceptionItem.entries = [
@@ -518,10 +519,10 @@ describe('filterEventsAgainstList', () => {
           },
         },
       ];
-      listClient.getListItemByValues = jest.fn(({ value }) =>
+      listClient.searchListItemByValues = jest.fn(({ value }) =>
         Promise.resolve(
           value.slice(0, 2).map((item) => ({
-            ...getListItemResponseMock(),
+            ...getSearchListItemResponseMock(),
             value: item,
           }))
         )
@@ -538,14 +539,14 @@ describe('filterEventsAgainstList', () => {
         ]),
         buildRuleMessage,
       });
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip');
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual(
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip');
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual(
         'ci-badguys.txt'
       );
       expect(res.hits.hits.length).toEqual(2);
     });
 
-    it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => {
+    it('should respond with the same items in the list given one exception item with two entries of type list and array of values in document', async () => {
       const exceptionItem = getExceptionListItemSchemaMock();
       exceptionItem.entries = [
         {
@@ -568,13 +569,16 @@ describe('filterEventsAgainstList', () => {
         },
       ];
 
-      // this call represents an exception list with a value list containing ['2.2.2.2']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      // this call represents an exception list with a value list containing ['2.2.2.2', '3.3.3.3']
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        {
+          ...getSearchListItemResponseMock(),
+          value: ['1.1.1.1', '2.2.2.2'],
+        },
       ]);
-      // this call represents an exception list with a value list containing ['4.4.4.4']
-      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
-        { ...getListItemResponseMock(), value: '4.4.4.4' },
+      // this call represents an exception list with a value list containing ['3.3.3.3', '4.4.4.4']
+      (listClient.searchListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getSearchListItemResponseMock(), value: ['3.3.3.3', '4.4.4.4'] },
       ]);
 
       const res = await filterEventsAgainstList({
@@ -598,17 +602,16 @@ describe('filterEventsAgainstList', () => {
         ),
         buildRuleMessage,
       });
-      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
-        '1.1.1.1',
-        '2.2.2.2',
-        '3.3.3.3',
+      expect(listClient.searchListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
+        ['1.1.1.1', '1.1.1.1'],
+        ['1.1.1.1', '2.2.2.2'],
+        ['2.2.2.2', '3.3.3.3'],
       ]);
-      expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
-        '1.1.1.1',
-        '2.2.2.2',
-        '3.3.3.3',
-        '4.4.4.4',
+      expect((listClient.searchListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
+        ['1.1.1.1', '2.2.2.2'],
+        ['2.2.2.2', '3.3.3.3'],
+        ['3.3.3.3', '4.4.4.4'],
       ]);
       expect(res.hits.hits.length).toEqual(2);
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events_against_list.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ExceptionListItemSchema, entriesList } from '../../../../../../lists/common/schemas';
+import { hasLargeValueList } from '../../../../../common/detection_engine/utils';
+import { FilterEventsAgainstListOptions } from './types';
+import { filterEvents } from './filter_events';
+import { createFieldAndSetTuples } from './create_field_and_set_tuples';
+import { SearchResponse } from '../../../types';
+
+/**
+ * Filters events against a large value based list. It does this through these
+ * steps below.
+ *
+ * 1. acquire the values from the specified fields to check
+ * e.g. if the value list is checking against source.ip, gather
+ * all the values for source.ip from the search response events.
+ *
+ * 2. search against the value list with the values found in the search result
+ * and see if there are any matches. For every match, add that value to a set
+ * that represents the "matched" values
+ *
+ * 3. filter the search result against the set from step 2 using the
+ * given operator (included vs excluded).
+ * acquire the list values we are checking for in the field.
+ *
+ * @param listClient The list client to use for queries
+ * @param exceptionsList The exception list
+ * @param logger Logger for messages
+ * @param eventSearchResult The current events from the search
+ */
+export const filterEventsAgainstList = async <T>({
+  listClient,
+  exceptionsList,
+  logger,
+  eventSearchResult,
+  buildRuleMessage,
+}: FilterEventsAgainstListOptions<T>): Promise<SearchResponse<T>> => {
+  try {
+    const atLeastOneLargeValueList = exceptionsList.some(({ entries }) =>
+      hasLargeValueList(entries)
+    );
+
+    if (!atLeastOneLargeValueList) {
+      logger.debug(
+        buildRuleMessage('no exception items of type list found - returning original search result')
+      );
+      return eventSearchResult;
+    }
+
+    const valueListExceptionItems = exceptionsList.filter((listItem: ExceptionListItemSchema) => {
+      return listItem.entries.every((entry) => entriesList.is(entry));
+    });
+
+    const res = await valueListExceptionItems.reduce<Promise<SearchResponse<T>['hits']['hits']>>(
+      async (
+        filteredAccum: Promise<SearchResponse<T>['hits']['hits']>,
+        exceptionItem: ExceptionListItemSchema
+      ) => {
+        const events = await filteredAccum;
+        const fieldAndSetTuples = await createFieldAndSetTuples({
+          events,
+          exceptionItem,
+          listClient,
+          logger,
+          buildRuleMessage,
+        });
+        const filteredEvents = filterEvents({ events, fieldAndSetTuples });
+        const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
+        logger.debug(
+          buildRuleMessage(`Exception with id ${exceptionItem.id} filtered out ${diff} events`)
+        );
+        return filteredEvents;
+      },
+      Promise.resolve<SearchResponse<T>['hits']['hits']>(eventSearchResult.hits.hits)
+    );
+
+    return {
+      took: eventSearchResult.took,
+      timed_out: eventSearchResult.timed_out,
+      _shards: eventSearchResult._shards,
+      hits: {
+        total: res.length,
+        max_score: eventSearchResult.hits.max_score,
+        hits: res,
+      },
+    };
+  } catch (exc) {
+    throw new Error(`Failed to query large value based lists index. Reason: ${exc.message}`);
+  }
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/types.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Logger } from 'src/core/server';
+
+import { ListClient } from '../../../../../../lists/server';
+import { BuildRuleMessage } from '../rule_messages';
+import { ExceptionListItemSchema, Type } from '../../../../../../lists/common/schemas';
+import { SearchResponse } from '../../../types';
+
+export interface FilterEventsAgainstListOptions<T> {
+  listClient: ListClient;
+  exceptionsList: ExceptionListItemSchema[];
+  logger: Logger;
+  eventSearchResult: SearchResponse<T>;
+  buildRuleMessage: BuildRuleMessage;
+}
+
+export interface CreateSetToFilterAgainstOptions<T> {
+  events: SearchResponse<T>['hits']['hits'];
+  field: string;
+  listId: string;
+  listType: Type;
+  listClient: ListClient;
+  logger: Logger;
+  buildRuleMessage: BuildRuleMessage;
+}
+
+export interface FilterEventsOptions<T> {
+  events: SearchResponse<T>['hits']['hits'];
+  fieldAndSetTuples: FieldSet[];
+}
+
+export interface CreateFieldAndSetTuplesOptions<T> {
+  events: SearchResponse<T>['hits']['hits'];
+  exceptionItem: ExceptionListItemSchema;
+  listClient: ListClient;
+  logger: Logger;
+  buildRuleMessage: BuildRuleMessage;
+}
+
+export interface FieldSet {
+  field: string;
+  operator: 'excluded' | 'included';
+  matchedSet: Set<unknown>;
+}

x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_messages.mock.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { buildRuleMessageFactory } from './rule_messages';
+
+export const buildRuleMessageMock = buildRuleMessageFactory({
+  id: 'fake id',
+  ruleId: 'fake rule id',
+  index: 'fakeindex',
+  name: 'fake name',
+});

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -19,10 +19,11 @@ import { buildRuleMessageFactory } from './rule_messages';
 import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants';
 import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
 import uuid from 'uuid';
-import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock';
 import { listMock } from '../../../../../lists/server/mocks';
 import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
 import { BulkResponse } from './types';
+import { SearchListItemArraySchema } from '../../../../../lists/common/schemas';
+import { getSearchListItemResponseMock } from '../../../../../lists/common/schemas/response/search_list_item_schema.mock';
 
 const buildRuleMessage = buildRuleMessageFactory({
   id: 'fake id',
@@ -39,7 +40,7 @@ describe('searchAfterAndBulkCreate', () => {
   beforeEach(() => {
     jest.clearAllMocks();
     listClient = listMock.getListClient();
-    listClient.getListItemByValues = jest.fn().mockResolvedValue([]);
+    listClient.searchListItemByValues = jest.fn().mockResolvedValue([]);
     inputIndexPattern = ['auditbeat-*'];
     mockService = alertsMock.createAlertServices();
   });
@@ -362,9 +363,12 @@ describe('searchAfterAndBulkCreate', () => {
   });
 
   test('should return success when all search results are in the allowlist and with sortId present', async () => {
-    listClient.getListItemByValues = jest
-      .fn()
-      .mockResolvedValue([{ value: '1.1.1.1' }, { value: '2.2.2.2' }, { value: '3.3.3.3' }]);
+    const searchListItems: SearchListItemArraySchema = [
+      { ...getSearchListItemResponseMock(), value: '1.1.1.1' },
+      { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
+      { ...getSearchListItemResponseMock(), value: '3.3.3.3' },
+    ];
+    listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems);
     const sampleParams = sampleRuleAlertParams(30);
     mockService.callCluster
       .mockResolvedValueOnce(
@@ -423,9 +427,14 @@ describe('searchAfterAndBulkCreate', () => {
   });
 
   test('should return success when all search results are in the allowlist and no sortId present', async () => {
-    listClient.getListItemByValues = jest
-      .fn()
-      .mockResolvedValue([{ value: '1.1.1.1' }, { value: '2.2.2.2' }, { value: '3.3.3.3' }]);
+    const searchListItems: SearchListItemArraySchema = [
+      { ...getSearchListItemResponseMock(), value: '1.1.1.1' },
+      { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
+      { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
+      { ...getSearchListItemResponseMock(), value: '2.2.2.2' },
+    ];
+
+    listClient.searchListItemByValues = jest.fn().mockResolvedValue(searchListItems);
     const sampleParams = sampleRuleAlertParams(30);
     mockService.callCluster.mockResolvedValueOnce(
       repeatedSearchResultsWithNoSortId(4, 4, someGuids.slice(0, 3), [
@@ -605,10 +614,10 @@ describe('searchAfterAndBulkCreate', () => {
       })
       .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits());
 
-    listClient.getListItemByValues = jest.fn(({ value }) =>
+    listClient.searchListItemByValues = jest.fn(({ value }) =>
       Promise.resolve(
         value.slice(0, 2).map((item) => ({
-          ...getListItemResponseMock(),
+          ...getSearchListItemResponseMock(),
           value: item,
         }))
       )
@@ -711,10 +720,10 @@ describe('searchAfterAndBulkCreate', () => {
     ];
     const sampleParams = sampleRuleAlertParams(30);
     mockService.callCluster.mockResolvedValueOnce(sampleEmptyDocSearchResults());
-    listClient.getListItemByValues = jest.fn(({ value }) =>
+    listClient.searchListItemByValues = jest.fn(({ value }) =>
       Promise.resolve(
         value.slice(0, 2).map((item) => ({
-          ...getListItemResponseMock(),
+          ...getSearchListItemResponseMock(),
           value: item,
         }))
       )
@@ -771,10 +780,10 @@ describe('searchAfterAndBulkCreate', () => {
       .mockImplementation(() => {
         throw Error('Fake Error'); // throws the exception we are testing
       });
-    listClient.getListItemByValues = jest.fn(({ value }) =>
+    listClient.searchListItemByValues = jest.fn(({ value }) =>
       Promise.resolve(
         value.slice(0, 2).map((item) => ({
-          ...getListItemResponseMock(),
+          ...getSearchListItemResponseMock(),
           value: item,
         }))
       )

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts

@@ -6,7 +6,7 @@
 
 import { singleSearchAfter } from './single_search_after';
 import { singleBulkCreate } from './single_bulk_create';
-import { filterEventsAgainstList } from './filter_events_with_list';
+import { filterEventsAgainstList } from './filters/filter_events_against_list';
 import { sendAlertTelemetryEvents } from './send_telemetry_events';
 import {
   createSearchAfterReturnType,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -66,7 +66,7 @@ import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk
 import { createThreatSignals } from './threat_mapping/create_threat_signals';
 import { getIndexVersion } from '../routes/index/get_index_version';
 import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template';
-import { filterEventsAgainstList } from './filter_events_with_list';
+import { filterEventsAgainstList } from './filters/filter_events_against_list';
 import { isOutdated } from '../migrations/helpers';
 
 export const signalRulesAlertType = ({

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts

@@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"is in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a double against an index that has the doubles stored as real doubles.
-      describe.skip('working against double values in the data set', () => {
+      describe('working against double values in the data set', () => {
         it('will return 3 results if we have a list that includes 1 double', async () => {
           await importFile(supertest, 'double', ['1.0'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['double']);
@@ -545,17 +543,19 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql([]);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 1 result if we have a list which contains the double range of 1.0-1.2', async () => {
-          await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt');
+        it('will return 1 result if we have a list which contains the double range of 1.0-1.2', async () => {
+          await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt', [
+            '1.0',
+            '1.2',
+          ]);
           const rule = getRuleForSignalTesting(['double_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'double',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'double_range',
                 },
                 operator: 'included',
                 type: 'list',
@@ -565,16 +565,14 @@ export default ({ getService }: FtrProviderContext) => {
           await waitForRuleSuccess(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.double).sort();
           expect(hits).to.eql(['1.3']);
         });
       });
     });
 
     describe('"is not in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a double against an index that has the doubles stored as real doubles.
-      describe.skip('working against double values in the data set', () => {
+      describe('working against double values in the data set', () => {
         it('will return 1 result if we have a list that excludes 1 double', async () => {
           await importFile(supertest, 'double', ['1.0'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['double']);
@@ -715,17 +713,19 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql(['1.0', '1.1', '1.2', '1.3']);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 3 results if we have a list which contains the double range of 1.0-1.2', async () => {
-          await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt');
+        it('will return 3 results if we have a list which contains the double range of 1.0-1.2', async () => {
+          await importFile(supertest, 'double_range', ['1.0-1.2'], 'list_items.txt', [
+            '1.0',
+            '1.2',
+          ]);
           const rule = getRuleForSignalTesting(['double_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'double',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'double_range',
                 },
                 operator: 'excluded',
                 type: 'list',
@@ -733,9 +733,9 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccess(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.double).sort();
           expect(hits).to.eql(['1.0', '1.1', '1.2']);
         });
       });

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts

@@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"is in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a float against an index that has the floats stored as real floats.
-      describe.skip('working against float values in the data set', () => {
+      describe('working against float values in the data set', () => {
         it('will return 3 results if we have a list that includes 1 float', async () => {
           await importFile(supertest, 'float', ['1.0'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['float']);
@@ -545,17 +543,16 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql([]);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 1 result if we have a list which contains the float range of 1.0-1.2', async () => {
-          await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt');
+        it('will return 1 result if we have a list which contains the float range of 1.0-1.2', async () => {
+          await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt', ['1.0', '1.2']);
           const rule = getRuleForSignalTesting(['float_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'float',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'float_range',
                 },
                 operator: 'included',
                 type: 'list',
@@ -565,16 +562,14 @@ export default ({ getService }: FtrProviderContext) => {
           await waitForRuleSuccess(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.float).sort();
           expect(hits).to.eql(['1.3']);
         });
       });
     });
 
     describe('"is not in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a float against an index that has the floats stored as real floats.
-      describe.skip('working against float values in the data set', () => {
+      describe('working against float values in the data set', () => {
         it('will return 1 result if we have a list that excludes 1 float', async () => {
           await importFile(supertest, 'float', ['1.0'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['float']);
@@ -715,17 +710,16 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql(['1.0', '1.1', '1.2', '1.3']);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 3 results if we have a list which contains the float range of 1.0-1.2', async () => {
-          await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt');
+        it('will return 3 results if we have a list which contains the float range of 1.0-1.2', async () => {
+          await importFile(supertest, 'float_range', ['1.0-1.2'], 'list_items.txt', ['1.0', '1.2']);
           const rule = getRuleForSignalTesting(['float_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'float',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'float_range',
                 },
                 operator: 'excluded',
                 type: 'list',
@@ -733,9 +727,9 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccess(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.float).sort();
           expect(hits).to.eql(['1.0', '1.1', '1.2']);
         });
       });

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts

@@ -16,8 +16,11 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
     loadTestFile(require.resolve('./float'));
     loadTestFile(require.resolve('./integer'));
     loadTestFile(require.resolve('./ip'));
+    loadTestFile(require.resolve('./ip_array'));
     loadTestFile(require.resolve('./keyword'));
+    loadTestFile(require.resolve('./keyword_array'));
     loadTestFile(require.resolve('./long'));
     loadTestFile(require.resolve('./text'));
+    loadTestFile(require.resolve('./text_array'));
   });
 };

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts

@@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"is in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a integer against an index that has the integers stored as real integers.
-      describe.skip('working against integer values in the data set', () => {
+      describe('working against integer values in the data set', () => {
         it('will return 3 results if we have a list that includes 1 integer', async () => {
           await importFile(supertest, 'integer', ['1'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['integer']);
@@ -545,17 +543,16 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql([]);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 1 result if we have a list which contains the integer range of 1-3', async () => {
-          await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt');
+        it('will return 1 result if we have a list which contains the integer range of 1-3', async () => {
+          await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt', ['1', '2']);
           const rule = getRuleForSignalTesting(['integer_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'integer',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'integer_range',
                 },
                 operator: 'included',
                 type: 'list',
@@ -565,16 +562,14 @@ export default ({ getService }: FtrProviderContext) => {
           await waitForRuleSuccess(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.integer).sort();
           expect(hits).to.eql(['4']);
         });
       });
     });
 
     describe('"is not in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a integer against an index that has the integers stored as real integers.
-      describe.skip('working against integer values in the data set', () => {
+      describe('working against integer values in the data set', () => {
         it('will return 1 result if we have a list that excludes 1 integer', async () => {
           await importFile(supertest, 'integer', ['1'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['integer']);
@@ -715,17 +710,16 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql(['1', '2', '3', '4']);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 3 results if we have a list which contains the integer range of 1-3', async () => {
-          await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt');
+        it('will return 3 results if we have a list which contains the integer range of 1-3', async () => {
+          await importFile(supertest, 'integer_range', ['1-3'], 'list_items.txt', ['1', '2', '3']);
           const rule = getRuleForSignalTesting(['integer_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'integer',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'integer_range',
                 },
                 operator: 'excluded',
                 type: 'list',
@@ -733,9 +727,9 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccess(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.integer).sort();
           expect(hits).to.eql(['1', '2', '3']);
         });
       });

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts

@@ -180,7 +180,7 @@ export default ({ getService }: FtrProviderContext) => {
         expect(ips).to.eql([]);
       });
 
-      it('should filter a CIDR range of 127.0.0.1/30', async () => {
+      it('should filter a CIDR range of "127.0.0.1/30"', async () => {
         const rule = getRuleForSignalTesting(['ip']);
         const { id } = await createRuleWithExceptionEntries(supertest, rule, [
           [
@@ -494,9 +494,12 @@ export default ({ getService }: FtrProviderContext) => {
         expect(ips).to.eql([]);
       });
 
-      // TODO: Fix this bug and then unskip this test
-      it.skip('will return 1 result if we have a list which contains the CIDR range of 127.0.0.1/30', async () => {
-        await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt');
+      it('will return 1 result if we have a list which contains the CIDR range of "127.0.0.1/30"', async () => {
+        await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt', [
+          '127.0.0.1',
+          '127.0.0.2',
+          '127.0.0.3',
+        ]);
         const rule = getRuleForSignalTesting(['ip']);
         const { id } = await createRuleWithExceptionEntries(supertest, rule, [
           [
@@ -504,7 +507,63 @@ export default ({ getService }: FtrProviderContext) => {
               field: 'ip',
               list: {
                 id: 'list_items.txt',
-                type: 'ip',
+                type: 'ip_range',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql(['127.0.0.4']);
+      });
+
+      it('will return 1 result if we have a list which contains the range syntax of "127.0.0.1-127.0.0.3"', async () => {
+        await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.3'], 'list_items.txt', [
+          '127.0.0.1',
+          '127.0.0.2',
+          '127.0.0.3',
+        ]);
+        const rule = getRuleForSignalTesting(['ip']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql(['127.0.0.4']);
+      });
+
+      it('will return 1 result if we have a list which contains the range mixed syntax of "127.0.0.1/32,127.0.0.2-127.0.0.3"', async () => {
+        await importFile(
+          supertest,
+          'ip_range',
+          ['127.0.0.1/32', '127.0.0.2-127.0.0.3'],
+          'list_items.txt',
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3']
+        );
+        const rule = getRuleForSignalTesting(['ip']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
               },
               operator: 'included',
               type: 'list',
@@ -594,9 +653,12 @@ export default ({ getService }: FtrProviderContext) => {
         expect(ips).to.eql(['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']);
       });
 
-      // TODO: Fix this bug and then unskip this test
-      it.skip('will return 3 results if we have a list which contains the CIDR range of 127.0.0.1/30', async () => {
-        await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt');
+      it('will return 3 results if we have a list which contains the CIDR range of "127.0.0.1/30"', async () => {
+        await importFile(supertest, 'ip_range', ['127.0.0.1/30'], 'list_items.txt', [
+          '127.0.0.1',
+          '127.0.0.2',
+          '127.0.0.3',
+        ]);
         const rule = getRuleForSignalTesting(['ip']);
         const { id } = await createRuleWithExceptionEntries(supertest, rule, [
           [
@@ -604,9 +666,36 @@ export default ({ getService }: FtrProviderContext) => {
               field: 'ip',
               list: {
                 id: 'list_items.txt',
-                type: 'ip',
+                type: 'ip_range',
               },
-              operator: 'included',
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql(['127.0.0.1', '127.0.0.2', '127.0.0.3']);
+      });
+
+      it('will return 3 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.3"', async () => {
+        await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.3'], 'list_items.txt', [
+          '127.0.0.1',
+          '127.0.0.2',
+          '127.0.0.3',
+        ]);
+        const rule = getRuleForSignalTesting(['ip']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
+              },
+              operator: 'excluded',
               type: 'list',
             },
           ],

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts

@@ -0,0 +1,735 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+import {
+  createListsIndex,
+  deleteAllExceptions,
+  deleteListsIndex,
+  importFile,
+} from '../../../../lists_api_integration/utils';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import {
+  createRule,
+  createRuleWithExceptionEntries,
+  createSignalsIndex,
+  deleteAllAlerts,
+  deleteSignalsIndex,
+  getRuleForSignalTesting,
+  getSignalsById,
+  waitForRuleSuccess,
+  waitForSignalsToBePresent,
+} from '../../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext) => {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+  const es = getService('es');
+
+  describe('Rule exception operators for data type ip', () => {
+    beforeEach(async () => {
+      await createSignalsIndex(supertest);
+      await createListsIndex(supertest);
+      await esArchiver.load('rule_exceptions/ip_as_array');
+    });
+
+    afterEach(async () => {
+      await deleteSignalsIndex(supertest);
+      await deleteAllAlerts(supertest);
+      await deleteAllExceptions(es);
+      await deleteListsIndex(supertest);
+      await esArchiver.unload('rule_exceptions/ip_as_array');
+    });
+
+    describe('"is" operator', () => {
+      it('should find all the ips from the data set when no exceptions are set on the rule', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRule(supertest, rule);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 4, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          [],
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+
+      it('should filter 1 single ip if it is set as an exception', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.1',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          [],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+
+      it('should filter 2 ips if both are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.1',
+            },
+          ],
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.5',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
+      });
+
+      it('should filter 3 ips if all 3 are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.1',
+            },
+          ],
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.5',
+            },
+          ],
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.8',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[]]);
+      });
+
+      it('should filter a CIDR range of "127.0.0.1/30"', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.1/30', // CIDR IP Range is 127.0.0.0 - 127.0.0.3
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          [],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+
+      it('should filter a CIDR range of "127.0.0.4/31"', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match',
+              value: '127.0.0.4/31', // CIDR IP Range is 127.0.0.4 - 127.0.0.5
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
+      });
+    });
+
+    describe('"is not" operator', () => {
+      it('will return 0 results if it cannot find what it is excluding', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match',
+              value: '192.168.0.1', // this value does not exist
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([]);
+      });
+
+      it('will return just 1 result we excluded', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match',
+              value: '127.0.0.1',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']]);
+      });
+
+      it('will return just 1 result we excluded 2 from the same array elements', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match',
+              value: '127.0.0.1',
+            },
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match',
+              value: '127.0.0.2',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']]);
+      });
+
+      it('will return 0 results if we exclude two ips', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match',
+              value: '127.0.0.1',
+            },
+          ],
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match',
+              value: '127.0.0.5',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([]);
+      });
+    });
+
+    describe('"is one of" operator', () => {
+      it('should filter 1 single ip if it is set as an exception', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match_any',
+              value: ['127.0.0.1'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          [],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+
+      it('should filter 2 ips if both are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match_any',
+              value: ['127.0.0.1', '127.0.0.5'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
+      });
+
+      it('should filter 3 ips if all 3 are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'match_any',
+              value: ['127.0.0.1', '127.0.0.5', '127.0.0.8'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[]]);
+      });
+    });
+
+    describe('"is not one of" operator', () => {
+      it('will return 0 results if it cannot find what it is excluding', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match_any',
+              value: ['192.168.0.1', '192.168.0.2'], // These values do not exist
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([]);
+      });
+
+      it('will return just the result we excluded', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'match_any',
+              value: ['127.0.0.1', '127.0.0.5'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+        ]);
+      });
+    });
+
+    describe('"exists" operator', () => {
+      it('will return 1 empty result if matching against ip', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'included',
+              type: 'exists',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[]]);
+      });
+    });
+
+    describe('"does not exist" operator', () => {
+      it('will return 3 results if matching against ip', async () => {
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              operator: 'excluded',
+              type: 'exists',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+    });
+
+    describe('"is in list" operator', () => {
+      it('will return 3 results if we have a list that includes 1 ip', async () => {
+        await importFile(supertest, 'ip', ['127.0.0.1'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          [],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+
+      it('will return 2 results if we have a list that includes 2 ips', async () => {
+        await importFile(supertest, 'ip', ['127.0.0.1', '127.0.0.5'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
+      });
+
+      it('will return 1 result if we have a list that includes all ips', async () => {
+        await importFile(
+          supertest,
+          'ip',
+          ['127.0.0.1', '127.0.0.5', '127.0.0.8'],
+          'list_items.txt'
+        );
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[]]);
+      });
+
+      it('will return 2 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => {
+        await importFile(
+          supertest,
+          'ip_range',
+          ['127.0.0.1/32', '127.0.0.2/31', '127.0.0.4/30'],
+          'list_items.txt',
+          [
+            '127.0.0.1',
+            '127.0.0.2',
+            '127.0.0.3',
+            '127.0.0.4',
+            '127.0.0.5',
+            '127.0.0.6',
+            '127.0.0.7',
+          ]
+        );
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
+      });
+
+      it('will return 2 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.7', async () => {
+        await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.7'], 'list_items.txt', [
+          '127.0.0.1',
+          '127.0.0.2',
+          '127.0.0.3',
+          '127.0.0.4',
+          '127.0.0.5',
+          '127.0.0.6',
+          '127.0.0.7',
+        ]);
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
+      });
+    });
+
+    describe('"is not in list" operator', () => {
+      it('will return 1 result if we have a list that excludes 1 ip', async () => {
+        await importFile(supertest, 'ip', ['127.0.0.1'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4']]);
+      });
+
+      it('will return 2 results if we have a list that excludes 2 ips', async () => {
+        await importFile(supertest, 'ip', ['127.0.0.1', '127.0.0.5'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list that excludes all ips', async () => {
+        await importFile(
+          supertest,
+          'ip',
+          ['127.0.0.1', '127.0.0.5', '127.0.0.8'],
+          'list_items.txt'
+        );
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+          ['127.0.0.8', '127.0.0.9', '127.0.0.10'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => {
+        await importFile(
+          supertest,
+          'ip_range',
+          ['127.0.0.1/32', '127.0.0.2/31', '127.0.0.4/30'],
+          'list_items.txt',
+          [
+            '127.0.0.1',
+            '127.0.0.2',
+            '127.0.0.3',
+            '127.0.0.4',
+            '127.0.0.5',
+            '127.0.0.6',
+            '127.0.0.7',
+          ]
+        );
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.7"', async () => {
+        await importFile(supertest, 'ip_range', ['127.0.0.1-127.0.0.7'], 'list_items.txt', [
+          '127.0.0.1',
+          '127.0.0.2',
+          '127.0.0.3',
+          '127.0.0.4',
+          '127.0.0.5',
+          '127.0.0.6',
+          '127.0.0.7',
+        ]);
+        const rule = getRuleForSignalTesting(['ip_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'ip',
+              list: {
+                id: 'list_items.txt',
+                type: 'ip_range',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const ips = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+        expect(ips).to.eql([
+          ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'],
+          ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'],
+        ]);
+      });
+    });
+  });
+};

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts

@@ -402,6 +402,39 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"is in list" operator', () => {
+      it('will return 4 results if we have two lists with an AND contradiction keyword === "word one" AND keyword === "word two"', async () => {
+        await importFile(supertest, 'keyword', ['word one'], 'list_items_1.txt');
+        await importFile(supertest, 'keyword', ['word two'], 'list_items_2.txt');
+        const rule = getRuleForSignalTesting(['keyword']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items_1.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items_2.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 4, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql(['word four', 'word one', 'word three', 'word two']);
+      });
+
       it('will return 3 results if we have a list that includes 1 keyword', async () => {
         await importFile(supertest, 'keyword', ['word one'], 'list_items.txt');
         const rule = getRuleForSignalTesting(['keyword']);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts

@@ -0,0 +1,624 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+import {
+  createListsIndex,
+  deleteAllExceptions,
+  deleteListsIndex,
+  importFile,
+} from '../../../../lists_api_integration/utils';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import {
+  createRule,
+  createRuleWithExceptionEntries,
+  createSignalsIndex,
+  deleteAllAlerts,
+  deleteSignalsIndex,
+  getRuleForSignalTesting,
+  getSignalsById,
+  waitForRuleSuccess,
+  waitForSignalsToBePresent,
+} from '../../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext) => {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+  const es = getService('es');
+
+  describe('Rule exception operators for data type keyword', () => {
+    beforeEach(async () => {
+      await createSignalsIndex(supertest);
+      await createListsIndex(supertest);
+      await esArchiver.load('rule_exceptions/keyword_as_array');
+    });
+
+    afterEach(async () => {
+      await deleteSignalsIndex(supertest);
+      await deleteAllAlerts(supertest);
+      await deleteAllExceptions(es);
+      await deleteListsIndex(supertest);
+      await esArchiver.unload('rule_exceptions/keyword_as_array');
+    });
+
+    describe('"is" operator', () => {
+      it('should find all the keyword from the data set when no exceptions are set on the rule', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRule(supertest, rule);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+
+      it('should filter 1 single keyword if it is set as an exception', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('should filter 2 keyword if both are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match',
+              value: 'word seven',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
+      });
+
+      it('should filter 3 keyword if all 3 are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match',
+              value: 'word six',
+            },
+          ],
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match',
+              value: 'word nine',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"is not" operator', () => {
+      it('will return 0 results if it cannot find what it is excluding', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'match',
+              value: '500.0', // this value is not in the data set
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([]);
+      });
+
+      it('will return just 1 result we excluded', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]);
+      });
+
+      it('will return 0 results if we exclude two keyword', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'match',
+              value: 'word five',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([]);
+      });
+    });
+
+    describe('"is one of" operator', () => {
+      it('should filter 1 single keyword if it is set as an exception', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match_any',
+              value: ['word one'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('should filter 2 keyword if both are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match_any',
+              value: ['word one', 'word six'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
+      });
+
+      it('should filter 3 keyword if all 3 are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'match_any',
+              value: ['word one', 'word five', 'word eight'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"is not one of" operator', () => {
+      it('will return 0 results if it cannot find what it is excluding', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'match_any',
+              value: ['500', '600'], // both these values are not in the data set
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([]);
+      });
+
+      it('will return just the result we excluded', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'match_any',
+              value: ['word one', 'word six'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+    });
+
+    describe('"exists" operator', () => {
+      it('will return 1 results if matching against keyword for the empty array', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'included',
+              type: 'exists',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"does not exist" operator', () => {
+      it('will return 3 results if matching against keyword', async () => {
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              operator: 'excluded',
+              type: 'exists',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+    });
+
+    describe('"is in list" operator', () => {
+      it('will return 4 results if we have two lists with an AND contradiction keyword === "word one" AND keyword === "word five"', async () => {
+        await importFile(supertest, 'keyword', ['word one'], 'list_items_1.txt');
+        await importFile(supertest, 'keyword', ['word five'], 'list_items_2.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items_1.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items_2.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+
+      it('will return 3 results if we have two lists with an AND keyword === "word one" AND keyword === "word two" since we have an array', async () => {
+        await importFile(supertest, 'keyword', ['word one'], 'list_items_1.txt');
+        await importFile(supertest, 'keyword', ['word two'], 'list_items_2.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items_1.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items_2.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list that includes 1 keyword', async () => {
+        await importFile(supertest, 'keyword', ['word one'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('will return 2 results if we have a list that includes 2 keyword', async () => {
+        await importFile(supertest, 'keyword', ['word one', 'word six'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
+      });
+
+      it('will return only the empty array for results if we have a list that includes all keyword', async () => {
+        await importFile(
+          supertest,
+          'keyword',
+          ['word one', 'word five', 'word eight'],
+          'list_items.txt'
+        );
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"is not in list" operator', () => {
+      it('will return 1 result if we have a list that excludes 1 keyword', async () => {
+        await importFile(supertest, 'keyword', ['word one'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]);
+      });
+
+      it('will return 1 result if we have a list that excludes 1 keyword but repeat 2 elements from the array in the list', async () => {
+        await importFile(supertest, 'keyword', ['word one', 'word two'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]);
+      });
+
+      it('will return 2 results if we have a list that excludes 2 keyword', async () => {
+        await importFile(supertest, 'keyword', ['word one', 'word five'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list that excludes 3 items', async () => {
+        await importFile(
+          supertest,
+          'keyword',
+          ['word one', 'word six', 'word ten'],
+          'list_items.txt'
+        );
+        const rule = getRuleForSignalTesting(['keyword_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'keyword',
+              list: {
+                id: 'list_items.txt',
+                type: 'keyword',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.keyword).sort();
+        expect(hits).to.eql([
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+    });
+  });
+};

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts

@@ -404,9 +404,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"is in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a long against an index that has the longs stored as real longs.
-      describe.skip('working against long values in the data set', () => {
+      describe('working against long values in the data set', () => {
         it('will return 3 results if we have a list that includes 1 long', async () => {
           await importFile(supertest, 'long', ['1'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['long']);
@@ -545,17 +543,16 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql([]);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 1 result if we have a list which contains the long range of 1-3', async () => {
-          await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt');
+        it('will return 1 result if we have a list which contains the long range of 1-3', async () => {
+          await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt', ['1', '2', '3']);
           const rule = getRuleForSignalTesting(['long_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'long',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'long_range',
                 },
                 operator: 'included',
                 type: 'list',
@@ -565,16 +562,14 @@ export default ({ getService }: FtrProviderContext) => {
           await waitForRuleSuccess(supertest, id);
           await waitForSignalsToBePresent(supertest, 1, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.long).sort();
           expect(hits).to.eql(['4']);
         });
       });
     });
 
     describe('"is not in list" operator', () => {
-      // TODO: Enable this test once the bugs are fixed, we cannot use a list of strings that represent
-      // a long against an index that has the longs stored as real longs.
-      describe.skip('working against long values in the data set', () => {
+      describe('working against long values in the data set', () => {
         it('will return 1 result if we have a list that excludes 1 long', async () => {
           await importFile(supertest, 'long', ['1'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['long']);
@@ -715,17 +710,16 @@ export default ({ getService }: FtrProviderContext) => {
           expect(hits).to.eql(['1', '2', '3', '4']);
         });
 
-        // TODO: Fix this bug and then unskip this test
-        it.skip('will return 3 results if we have a list which contains the long range of 1-3', async () => {
-          await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt');
+        it('will return 3 results if we have a list which contains the long range of 1-3', async () => {
+          await importFile(supertest, 'long_range', ['1-3'], 'list_items.txt', ['1', '2', '3']);
           const rule = getRuleForSignalTesting(['long_as_string']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
               {
-                field: 'ip',
+                field: 'long',
                 list: {
                   id: 'list_items.txt',
-                  type: 'ip',
+                  type: 'long_range',
                 },
                 operator: 'excluded',
                 type: 'list',
@@ -733,9 +727,9 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccess(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source.ip).sort();
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.long).sort();
           expect(hits).to.eql(['1', '2', '3']);
         });
       });

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts

@@ -592,10 +592,37 @@ export default ({ getService }: FtrProviderContext) => {
         });
       });
 
-      // TODO: Unskip these once this is fixed
-      describe.skip('working against text values with spaces', () => {
+      describe('working against text values with spaces', () => {
         it('will return 3 results if we have a list that includes 1 text', async () => {
-          await importFile(supertest, 'text', ['one'], 'list_items.txt');
+          await importTextFile(supertest, 'text', ['word one'], 'list_items.txt');
+          const rule = getRuleForSignalTesting(['text']);
+          const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+            [
+              {
+                field: 'text',
+                list: {
+                  id: 'list_items.txt',
+                  type: 'text',
+                },
+                operator: 'included',
+                type: 'list',
+              },
+            ],
+          ]);
+          await waitForRuleSuccess(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsOpen = await getSignalsById(supertest, id);
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+          expect(hits).to.eql(['word four', 'word three', 'word two']);
+        });
+
+        it('will return 3 results if we have a list that includes 1 text with additional wording', async () => {
+          await importTextFile(
+            supertest,
+            'text',
+            ['word one additional wording'],
+            'list_items.txt'
+          );
           const rule = getRuleForSignalTesting(['text']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
@@ -618,7 +645,7 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('will return 2 results if we have a list that includes 2 text', async () => {
-          await importFile(supertest, 'text', ['one', 'three'], 'list_items.txt');
+          await importFile(supertest, 'text', ['word one', 'word three'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['text']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
@@ -644,7 +671,7 @@ export default ({ getService }: FtrProviderContext) => {
           await importTextFile(
             supertest,
             'text',
-            ['one', 'two', 'three', 'four'],
+            ['word one', 'word two', 'word three', 'word four'],
             'list_items.txt'
           );
           const rule = getRuleForSignalTesting(['text']);
@@ -746,10 +773,37 @@ export default ({ getService }: FtrProviderContext) => {
         });
       });
 
-      // TODO: Unskip these once this is fixed
-      describe.skip('working against text values with spaces', () => {
+      describe('working against text values with spaces', () => {
         it('will return 1 result if we have a list that excludes 1 text', async () => {
-          await importTextFile(supertest, 'text', ['one'], 'list_items.txt');
+          await importTextFile(supertest, 'text', ['word one'], 'list_items.txt');
+          const rule = getRuleForSignalTesting(['text']);
+          const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+            [
+              {
+                field: 'text',
+                list: {
+                  id: 'list_items.txt',
+                  type: 'text',
+                },
+                operator: 'excluded',
+                type: 'list',
+              },
+            ],
+          ]);
+          await waitForRuleSuccess(supertest, id);
+          await waitForSignalsToBePresent(supertest, 1, [id]);
+          const signalsOpen = await getSignalsById(supertest, id);
+          const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+          expect(hits).to.eql(['word one']);
+        });
+
+        it('will return 1 result if we have a list that excludes 1 text with additional wording', async () => {
+          await importTextFile(
+            supertest,
+            'text',
+            ['word one additional wording'],
+            'list_items.txt'
+          );
           const rule = getRuleForSignalTesting(['text']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
@@ -772,7 +826,7 @@ export default ({ getService }: FtrProviderContext) => {
         });
 
         it('will return 2 results if we have a list that excludes 2 text', async () => {
-          await importTextFile(supertest, 'text', ['one', 'three'], 'list_items.txt');
+          await importTextFile(supertest, 'text', ['word one', 'word three'], 'list_items.txt');
           const rule = getRuleForSignalTesting(['text']);
           const { id } = await createRuleWithExceptionEntries(supertest, rule, [
             [
@@ -798,7 +852,7 @@ export default ({ getService }: FtrProviderContext) => {
           await importTextFile(
             supertest,
             'text',
-            ['one', 'two', 'three', 'four'],
+            ['word one', 'word two', 'word three', 'word four'],
             'list_items.txt'
           );
           const rule = getRuleForSignalTesting(['text']);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts

@@ -0,0 +1,619 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+import {
+  createListsIndex,
+  deleteAllExceptions,
+  deleteListsIndex,
+  importFile,
+} from '../../../../lists_api_integration/utils';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import {
+  createRule,
+  createRuleWithExceptionEntries,
+  createSignalsIndex,
+  deleteAllAlerts,
+  deleteSignalsIndex,
+  getRuleForSignalTesting,
+  getSignalsById,
+  waitForRuleSuccess,
+  waitForSignalsToBePresent,
+} from '../../../utils';
+
+// eslint-disable-next-line import/no-default-export
+export default ({ getService }: FtrProviderContext) => {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+  const es = getService('es');
+
+  describe('Rule exception operators for data type text', () => {
+    beforeEach(async () => {
+      await createSignalsIndex(supertest);
+      await createListsIndex(supertest);
+      await esArchiver.load('rule_exceptions/text_as_array');
+    });
+
+    afterEach(async () => {
+      await deleteSignalsIndex(supertest);
+      await deleteAllAlerts(supertest);
+      await deleteAllExceptions(es);
+      await deleteListsIndex(supertest);
+      await esArchiver.unload('rule_exceptions/text_as_array');
+    });
+
+    describe('"is" operator', () => {
+      it('should find all the text from the data set when no exceptions are set on the rule', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRule(supertest, rule);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+
+      it('should filter 1 single text if it is set as an exception', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('should filter 2 text if both are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match',
+              value: 'word seven',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
+      });
+
+      it('should filter 3 text if all 3 are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match',
+              value: 'word six',
+            },
+          ],
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match',
+              value: 'word nine',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"is not" operator', () => {
+      it('will return 0 results if it cannot find what it is excluding', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'match',
+              value: '500.0', // this value is not in the data set
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([]);
+      });
+
+      it('will return just 1 result we excluded', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]);
+      });
+
+      it('will return 0 results if we exclude two text', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'match',
+              value: 'word one',
+            },
+          ],
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'match',
+              value: 'word five',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([]);
+      });
+    });
+
+    describe('"is one of" operator', () => {
+      it('should filter 1 single text if it is set as an exception', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match_any',
+              value: ['word one'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('should filter 2 text if both are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match_any',
+              value: ['word one', 'word six'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
+      });
+
+      it('should filter 3 text if all 3 are set as exceptions', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'match_any',
+              value: ['word one', 'word five', 'word eight'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"is not one of" operator', () => {
+      it('will return 0 results if it cannot find what it is excluding', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'match_any',
+              value: ['500', '600'], // both these values are not in the data set
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([]);
+      });
+
+      it('will return just the result we excluded', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'match_any',
+              value: ['word one', 'word six'],
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+    });
+
+    describe('"exists" operator', () => {
+      it('will return 1 results if matching against text for the empty array', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'included',
+              type: 'exists',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"does not exist" operator', () => {
+      it('will return 3 results if matching against text', async () => {
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              operator: 'excluded',
+              type: 'exists',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+    });
+
+    describe('"is in list" operator', () => {
+      it('will return 4 results if we have two lists with an AND contradiction text === "word one" AND text === "word five"', async () => {
+        await importFile(supertest, 'text', ['word one'], 'list_items_1.txt');
+        await importFile(supertest, 'text', ['word five'], 'list_items_2.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items_1.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+            {
+              field: 'text',
+              list: {
+                id: 'list_items_2.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+
+      it('will return 3 results if we have two lists with an AND text === "word one" AND text === "word two" since we have an array', async () => {
+        await importFile(supertest, 'text', ['word one'], 'list_items_1.txt');
+        await importFile(supertest, 'text', ['word two'], 'list_items_2.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items_1.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+            {
+              field: 'text',
+              list: {
+                id: 'list_items_2.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list that includes 1 text', async () => {
+        await importFile(supertest, 'text', ['word one'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          [],
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+        ]);
+      });
+
+      it('will return 2 results if we have a list that includes 2 text', async () => {
+        await importFile(supertest, 'text', ['word one', 'word six'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
+      });
+
+      it('will return only the empty array for results if we have a list that includes all text', async () => {
+        await importFile(
+          supertest,
+          'text',
+          ['word one', 'word five', 'word eight'],
+          'list_items.txt'
+        );
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'included',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([[]]);
+      });
+    });
+
+    describe('"is not in list" operator', () => {
+      it('will return 1 result if we have a list that excludes 1 text', async () => {
+        await importFile(supertest, 'text', ['word one'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]);
+      });
+
+      it('will return 1 result if we have a list that excludes 1 text but repeat 2 elements from the array in the list', async () => {
+        await importFile(supertest, 'text', ['word one', 'word two'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([['word one', 'word two', 'word three', 'word four']]);
+      });
+
+      it('will return 2 results if we have a list that excludes 2 text', async () => {
+        await importFile(supertest, 'text', ['word one', 'word five'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+
+      it('will return 3 results if we have a list that excludes 3 items', async () => {
+        await importFile(supertest, 'text', ['word one', 'word six', 'word ten'], 'list_items.txt');
+        const rule = getRuleForSignalTesting(['text_as_array']);
+        const { id } = await createRuleWithExceptionEntries(supertest, rule, [
+          [
+            {
+              field: 'text',
+              list: {
+                id: 'list_items.txt',
+                type: 'text',
+              },
+              operator: 'excluded',
+              type: 'list',
+            },
+          ],
+        ]);
+        await waitForRuleSuccess(supertest, id);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
+        const signalsOpen = await getSignalsById(supertest, id);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.text).sort();
+        expect(hits).to.eql([
+          ['word eight', 'word nine', 'word ten'],
+          ['word five', null, 'word six', 'word seven'],
+          ['word one', 'word two', 'word three', 'word four'],
+        ]);
+      });
+    });
+  });
+};

x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/data.json

@@ -0,0 +1,51 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "ip_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:00:53.000Z",
+      "ip": []
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "ip_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:01:53.000Z",
+      "ip": ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"]
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "ip_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:02:53.000Z",
+      "ip": ["127.0.0.5", null, "127.0.0.6", "127.0.0.7"]
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "4",
+    "index": "ip_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:03:53.000Z",
+      "ip": ["127.0.0.8", "127.0.0.9", "127.0.0.10"]
+    },
+    "type": "_doc"
+  }
+}

x-pack/test/functional/es_archives/rule_exceptions/ip_as_array/mappings.json

@@ -0,0 +1,20 @@
+{
+  "type": "index",
+  "value": {
+    "index": "ip_as_array",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "ip": { "type": "ip" }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/data.json

@@ -0,0 +1,51 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "keyword_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:00:53.000Z",
+      "keyword": []
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "keyword_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:01:53.000Z",
+      "keyword": ["word one", "word two", "word three", "word four"]
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "keyword_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:02:53.000Z",
+      "keyword": ["word five", null, "word six", "word seven"]
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "4",
+    "index": "keyword_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:03:53.000Z",
+      "keyword": ["word eight", "word nine", "word ten"]
+    },
+    "type": "_doc"
+  }
+}

x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array/mappings.json

@@ -0,0 +1,20 @@
+{
+  "type": "index",
+  "value": {
+    "index": "keyword_as_array",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "keyword": { "type": "keyword" }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

x-pack/test/functional/es_archives/rule_exceptions/text_as_array/data.json

@@ -0,0 +1,51 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "text_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:00:53.000Z",
+      "text": []
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "text_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:01:53.000Z",
+      "text": ["word one", "word two", "word three", "word four"]
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "text_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:02:53.000Z",
+      "text": ["word five", null, "word six", "word seven"]
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "4",
+    "index": "text_as_array",
+    "source": {
+      "@timestamp": "2020-10-28T05:03:53.000Z",
+      "text": ["word eight", "word nine", "word ten"]
+    },
+    "type": "_doc"
+  }
+}

x-pack/test/functional/es_archives/rule_exceptions/text_as_array/mappings.json

@@ -0,0 +1,20 @@
+{
+  "type": "index",
+  "value": {
+    "index": "text_as_array",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "keyword": { "type": "text" }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

x-pack/test/lists_api_integration/utils.ts

@@ -175,12 +175,14 @@ export const deleteAllExceptions = async (es: Client): Promise<void> => {
  * @param type The type to import as
  * @param contents The contents of the import
  * @param fileName filename to import as
+ * @param testValues Optional test values in case you're using CIDR or range based lists
  */
 export const importFile = async (
   supertest: SuperTest<supertestAsPromised.Test>,
   type: Type,
   contents: string[],
-  fileName: string
+  fileName: string,
+  testValues?: string[]
 ): Promise<void> => {
   await supertest
     .post(`${LIST_ITEM_URL}/_import?type=${type}`)
@@ -191,7 +193,8 @@ export const importFile = async (
 
   // although we have pushed the list and its items, it is async so we
   // have to wait for the contents before continuing
-  await waitForListItems(supertest, contents, fileName);
+  const testValuesOrContents = testValues ?? contents;
+  await waitForListItems(supertest, testValuesOrContents, fileName);
 };
 
 /**