maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Security Solution] [Detections] Combine multiple timestamp searches into single request (#96078)

Browse source 
* merge multiple timestamp queries into one single search

* fix types and unit tests

* remove unused code for sending secondary search

* removes unused excludeDocsWithTimestampOverride

* adds integration tests to cover cases that should / should not generate signals when timestamp override is present in rule

* adds integration test to ensure unmapped sort fields do not break search after functionality of detection rules

* Need to figure out why moving the tests around fixed them...

* updates tests with new es archive data and fixes bug where exclusion filter was hardcoded to event.ingested :yikes:

* remove dead commented out code

* fixes typo in test file, removes redundant delete signals call in integration test, fixes logic for possibility of receving a null value in sort ids, removes unused utility function for checking valid sort ids

* a unit test for checking if an empty string of a sort id is present was failing because we moved the logic for checking that out of the build search query function and up into the big loop. So I moved that unit test into the search after bulk create test file.

* fix types

* removes isEmpty since it doesn't check for empty strings
This commit is contained in:
Devin W. Hurley 2021-04-20 15:16:01 -04:00 committed by GitHub
parent f37492069a
commit 4d2414e7f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 654 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file

@ -115,6 +115,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
export const sampleDocWithSortId = (
someUuid: string = sampleIdGuid,
sortIds: string[] = ['1234567891111', '2233447556677'],
ip?: string | string[],
destIp?: string | string[]
): SignalSourceHit => ({
@ -139,7 +140,7 @@ export const sampleDocWithSortId = (
'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'],
'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'],
},
sort: ['1234567891111'],
sort: sortIds,
});
export const sampleDocNoSortId = (
@ -630,7 +631,8 @@ export const repeatedSearchResultsWithSortId = (
pageSize: number,
guids: string[],
ips?: Array<string | string[]>,
destIps?: Array<string | string[]>
destIps?: Array<string | string[]>,
sortIds?: string[]
): SignalSearchResponse => ({
took: 10,
timed_out: false,
@ -646,6 +648,7 @@ export const repeatedSearchResultsWithSortId = (
hits: Array.from({ length: pageSize }).map((x, index) => ({
...sampleDocWithSortId(
guids[index],
sortIds,
ips ? ips[index] : '127.0.0.1',
destIps ? destIps[index] : '127.0.0.1'
),

176
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts
View file

@ -15,9 +15,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
@ -39,12 +38,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
@ -73,16 +79,16 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is an empty string it should not be included', () => {
test('it builds a now-5m up to today filter with timestampOverride', () => {
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: '',
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
searchAfterSortIds: undefined,
timestampOverride: 'event.ingested',
});
expect(query).toEqual({
allow_no_indices: true,
@ -91,6 +97,10 @@ describe('create_signals', () => {
ignore_unavailable: true,
body: {
docvalue_fields: [
{
field: 'event.ingested',
format: 'strict_date_optional_time',
},
{
field: '@timestamp',
format: 'strict_date_optional_time',
@ -104,12 +114,43 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
should: [
{
range: {
'event.ingested': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
bool: {
must_not: {
exists: {
field: 'event.ingested',
},
},
},
},
],
},
},
],
minimum_should_match: 1,
},
},
],
@ -128,6 +169,12 @@ describe('create_signals', () => {
},
],
sort: [
{
'event.ingested': {
order: 'asc',
unmapped_type: 'date',
},
},
{
'@timestamp': {
order: 'asc',
@ -138,7 +185,8 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is a valid sortId string', () => {
test('if searchAfterSortIds is a valid sortId string', () => {
const fakeSortId = '123456789012';
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
@ -146,9 +194,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: fakeSortId,
searchAfterSortIds: [fakeSortId],
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
@ -170,12 +217,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
@ -205,7 +259,7 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is a valid sortId number', () => {
test('if searchAfterSortIds is a valid sortId number', () => {
const fakeSortIdNumber = 123456789012;
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
@ -213,9 +267,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: fakeSortIdNumber,
searchAfterSortIds: [fakeSortIdNumber],
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
@ -237,12 +290,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
@ -279,9 +339,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
@ -303,12 +362,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
@ -352,9 +418,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
@ -371,12 +436,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],

125
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import type { estypes } from '@elastic/elasticsearch';
import { SortResults } from '@elastic/elasticsearch/api/types';
import { isEmpty } from 'lodash';
import {
SortOrderOrUndefined,
TimestampOverrideOrUndefined,
@ -18,9 +20,8 @@ interface BuildEventsSearchQuery {
filter?: estypes.QueryContainer;
size: number;
sortOrder?: SortOrderOrUndefined;
searchAfterSortId: string | number | undefined;
searchAfterSortIds: SortResults | undefined;
timestampOverride: TimestampOverrideOrUndefined;
excludeDocsWithTimestampOverride: boolean;
}
export const buildEventsSearchQuery = ({
@ -30,10 +31,9 @@ export const buildEventsSearchQuery = ({
to,
filter,
size,
searchAfterSortId,
searchAfterSortIds,
sortOrder,
timestampOverride,
excludeDocsWithTimestampOverride,
}: BuildEventsSearchQuery) => {
const defaultTimeFields = ['@timestamp'];
const timestamps =
@ -43,36 +43,62 @@ export const buildEventsSearchQuery = ({
format: 'strict_date_optional_time',
}));
const sortField =
timestampOverride != null && !excludeDocsWithTimestampOverride
? timestampOverride
: '@timestamp';
const rangeFilter: estypes.QueryContainer[] = [
{
range: {
[sortField]: {
lte: to,
gte: from,
format: 'strict_date_optional_time',
},
},
},
];
if (excludeDocsWithTimestampOverride) {
rangeFilter.push({
bool: {
must_not: {
exists: {
field: timestampOverride,
const rangeFilter: estypes.QueryContainer[] =
timestampOverride != null
? [
{
range: {
[timestampOverride]: {
lte: to,
gte: from,
format: 'strict_date_optional_time',
},
},
},
},
},
});
}
// @ts-expect-error undefined in not assignable to QueryContainer
// but tests contain undefined, so I suppose it's desired behaviour
const filterWithTime: estypes.QueryContainer[] = [filter, { bool: { filter: rangeFilter } }];
{
bool: {
filter: [
{
range: {
'@timestamp': {
lte: to,
gte: from,
// @ts-expect-error
format: 'strict_date_optional_time',
},
},
},
{
bool: {
must_not: {
exists: {
field: timestampOverride,
},
},
},
},
],
},
},
]
: [
{
range: {
'@timestamp': {
lte: to,
gte: from,
format: 'strict_date_optional_time',
},
},
},
];
const filterWithTime: estypes.QueryContainer[] = [
// but tests contain undefined, so I suppose it's desired behaviour
// @ts-expect-error undefined in not assignable to QueryContainer
filter,
{ bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } },
];
const searchQuery = {
allow_no_indices: true,
@ -99,22 +125,39 @@ export const buildEventsSearchQuery = ({
],
...(aggregations ? { aggregations } : {}),
sort: [
{
[sortField]: {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
},
...(timestampOverride != null
? [
{
[timestampOverride]: {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
},
{
'@timestamp': {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
},
]
: [
{
'@timestamp': {
order: sortOrder ?? 'asc',
unmapped_type: 'date',
},
},
]),
],
},
};
if (searchAfterSortId) {
if (searchAfterSortIds != null && !isEmpty(searchAfterSortIds)) {
return {
...searchQuery,
body: {
...searchQuery.body,
search_after: [searchAfterSortId],
search_after: searchAfterSortIds,
},
};
}

32
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts
View file

@ -17,7 +17,7 @@ import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'
describe('filterEventsAgainstList', () => {
let listClient = listMock.getListClient();
let exceptionItem = getExceptionListItemSchemaMock();
let events = [sampleDocWithSortId('123', '1.1.1.1')];
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
beforeEach(() => {
jest.clearAllMocks();
@ -44,7 +44,7 @@ describe('filterEventsAgainstList', () => {
},
],
};
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
});
afterEach(() => {
@ -111,7 +111,7 @@ describe('filterEventsAgainstList', () => {
});
test('it returns a single matched set as a JSON.stringify() set from the "events"', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
(exceptionItem.entries[0] as EntryList).field = 'source.ip';
const [{ matchedSet }] = await createFieldAndSetTuples({
listClient,
@ -124,7 +124,10 @@ describe('filterEventsAgainstList', () => {
});
test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('456', undefined, '2.2.2.2'),
];
(exceptionItem.entries[0] as EntryList).field = 'source.ip';
const [{ matchedSet }] = await createFieldAndSetTuples({
listClient,
@ -137,7 +140,7 @@ describe('filterEventsAgainstList', () => {
});
test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => {
events = [sampleDocWithSortId('123', ['1.1.1.1', '2.2.2.2'])];
events = [sampleDocWithSortId('123', undefined, ['1.1.1.1', '2.2.2.2'])];
(exceptionItem.entries[0] as EntryList).field = 'source.ip';
const [{ matchedSet }] = await createFieldAndSetTuples({
listClient,
@ -150,7 +153,10 @@ describe('filterEventsAgainstList', () => {
});
test('it returns 2 fields when given two exception list items', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('456', undefined, '2.2.2.2'),
];
exceptionItem.entries = [
{
field: 'source.ip',
@ -182,7 +188,10 @@ describe('filterEventsAgainstList', () => {
});
test('it returns two matched sets from two different events, one excluded, and one included', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('456', undefined, '2.2.2.2'),
];
exceptionItem.entries = [
{
field: 'source.ip',
@ -215,7 +224,10 @@ describe('filterEventsAgainstList', () => {
});
test('it returns two fields from two different events', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('456', undefined, '2.2.2.2'),
];
exceptionItem.entries = [
{
field: 'source.ip',
@ -249,8 +261,8 @@ describe('filterEventsAgainstList', () => {
test('it returns two matches from two different events', async () => {
events = [
sampleDocWithSortId('123', '1.1.1.1', '3.3.3.3'),
sampleDocWithSortId('456', '2.2.2.2', '5.5.5.5'),
sampleDocWithSortId('123', undefined, '1.1.1.1', '3.3.3.3'),
sampleDocWithSortId('456', undefined, '2.2.2.2', '5.5.5.5'),
];
exceptionItem.entries = [
{

16
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts
View file

@ -14,7 +14,7 @@ import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'
describe('createSetToFilterAgainst', () => {
let listClient = listMock.getListClient();
let events = [sampleDocWithSortId('123', '1.1.1.1')];
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
beforeEach(() => {
jest.clearAllMocks();
@ -27,7 +27,7 @@ describe('createSetToFilterAgainst', () => {
}))
)
);
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
});
afterEach(() => {
@ -49,7 +49,7 @@ describe('createSetToFilterAgainst', () => {
});
test('it returns 1 field if the list returns a single item', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
const field = await createSetToFilterAgainst({
events,
field: 'source.ip',
@ -68,7 +68,10 @@ describe('createSetToFilterAgainst', () => {
});
test('it returns 2 fields if the list returns 2 items', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('123', undefined, '2.2.2.2'),
];
const field = await createSetToFilterAgainst({
events,
field: 'source.ip',
@ -87,7 +90,10 @@ describe('createSetToFilterAgainst', () => {
});
test('it returns 0 fields if the field does not match up to a valid field within the event', async () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('123', undefined, '2.2.2.2'),
];
const field = await createSetToFilterAgainst({
events,
field: 'nonexistent.field', // field does not exist

15
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts
View file

@ -14,7 +14,7 @@ import { FieldSet } from './types';
describe('filterEvents', () => {
let listClient = listMock.getListClient();
let events = [sampleDocWithSortId('123', '1.1.1.1')];
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
beforeEach(() => {
jest.clearAllMocks();
@ -27,7 +27,7 @@ describe('filterEvents', () => {
}))
)
);
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
});
afterEach(() => {
@ -35,7 +35,7 @@ describe('filterEvents', () => {
});
test('it filters out the event if it is "included"', () => {
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
const fieldAndSetTuples: FieldSet[] = [
{
field: 'source.ip',
@ -51,7 +51,7 @@ describe('filterEvents', () => {
});
test('it does not filter out the event if it is "excluded"', () => {
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
const fieldAndSetTuples: FieldSet[] = [
{
field: 'source.ip',
@ -67,7 +67,7 @@ describe('filterEvents', () => {
});
test('it does NOT filter out the event if the field is not found', () => {
events = [sampleDocWithSortId('123', '1.1.1.1')];
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
const fieldAndSetTuples: FieldSet[] = [
{
field: 'madeup.nonexistent', // field does not exist
@ -83,7 +83,10 @@ describe('filterEvents', () => {
});
test('it does NOT filter out the event if it is in both an inclusion and exclusion list', () => {
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
events = [
sampleDocWithSortId('123', undefined, '1.1.1.1'),
sampleDocWithSortId('123', undefined, '2.2.2.2'),
];
const fieldAndSetTuples: FieldSet[] = [
{
field: 'source.ip',

78
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file

@ -426,6 +426,84 @@ describe('searchAfterAndBulkCreate', () => {
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
});
test('should return success when empty string sortId present', async () => {
mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce(
elasticsearchClientMock.createSuccessTransportRequestPromise({
took: 100,
errors: false,
items: [
{
create: {
_id: someGuids[0],
_index: 'myfakeindex',
status: 201,
},
},
{
create: {
_id: someGuids[1],
_index: 'myfakeindex',
status: 201,
},
},
{
create: {
_id: someGuids[2],
_index: 'myfakeindex',
status: 201,
},
},
{
create: {
_id: someGuids[3],
_index: 'myfakeindex',
status: 201,
},
},
],
})
);
mockService.scopedClusterClient.asCurrentUser.search
.mockResolvedValueOnce(
elasticsearchClientMock.createSuccessTransportRequestPromise(
repeatedSearchResultsWithSortId(
4,
4,
someGuids.slice(0, 3),
['1.1.1.1', '2.2.2.2', '2.2.2.2', '2.2.2.2'],
// this is the case we are testing, if we receive an empty string for one of the sort ids.
['', '2222222222222']
)
)
)
.mockResolvedValueOnce(
elasticsearchClientMock.createSuccessTransportRequestPromise(
sampleDocSearchResultsNoSortIdNoHits()
)
);
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
ruleSO,
tuples,
listClient,
exceptionsList: [],
services: mockService,
logger: mockLogger,
eventsTelemetry: undefined,
id: sampleRuleGuid,
inputIndexPattern,
signalsIndex: DEFAULT_SIGNALS_INDEX,
pageSize: 1,
filter: undefined,
refresh: false,
buildRuleMessage,
});
expect(success).toEqual(true);
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
expect(createdSignalsCount).toEqual(4);
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
});
test('should return success when all search results are in the allowlist and no sortId present', async () => {
const searchListItems: SearchListItemArraySchema = [
{ ...getSearchListItemResponseMock(), value: ['1.1.1.1'] },

72
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts
View file

@ -5,9 +5,8 @@
* 2.0.
*/
/* eslint-disable complexity */
import { identity } from 'lodash';
import { SortResults } from '@elastic/elasticsearch/api/types';
import { singleSearchAfter } from './single_search_after';
import { singleBulkCreate } from './single_bulk_create';
import { filterEventsAgainstList } from './filters/filter_events_against_list';
@ -19,6 +18,7 @@ import {
createTotalHitsFromSearchResult,
mergeReturns,
mergeSearchResults,
getSafeSortIds,
} from './utils';
import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types';
@ -44,10 +44,8 @@ export const searchAfterAndBulkCreate = async ({
let toReturn = createSearchAfterReturnType();
// sortId tells us where to start our next consecutive search_after query
let sortId: string | undefined;
let sortIds: SortResults | undefined;
let hasSortId = true; // default to true so we execute the search on initial run
let backupSortId: string | undefined;
let hasBackupSortId = ruleParams.timestampOverride ? true : false;
// signalsCreatedCount keeps track of how many signals we have created,
// to ensure we don't exceed maxSignals
@ -69,60 +67,12 @@ export const searchAfterAndBulkCreate = async ({
while (signalsCreatedCount < tuple.maxSignals) {
try {
let mergedSearchResults = createSearchResultReturnType();
logger.debug(buildRuleMessage(`sortIds: ${sortId}`));
// if there is a timestampOverride param we always want to do a secondary search against @timestamp
if (ruleParams.timestampOverride != null && hasBackupSortId) {
// only execute search if we have something to sort on or if it is the first search
const {
searchResult: searchResultB,
searchDuration: searchDurationB,
searchErrors: searchErrorsB,
} = await singleSearchAfter({
buildRuleMessage,
searchAfterSortId: backupSortId,
index: inputIndexPattern,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
services,
logger,
// @ts-expect-error please, declare a type explicitly instead of unknown
filter,
pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
timestampOverride: ruleParams.timestampOverride,
excludeDocsWithTimestampOverride: true,
});
// call this function setSortIdOrExit()
const lastSortId = searchResultB?.hits?.hits[searchResultB.hits.hits.length - 1]?.sort;
if (lastSortId != null && lastSortId.length !== 0) {
// @ts-expect-error @elastic/elasticsearch SortResults contains null not assignable to backupSortId
backupSortId = lastSortId[0];
hasBackupSortId = true;
} else {
logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB'));
hasBackupSortId = false;
}
mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]);
toReturn = mergeReturns([
toReturn,
createSearchAfterReturnTypeFromResponse({
searchResult: mergedSearchResults,
timestampOverride: undefined,
}),
createSearchAfterReturnType({
searchAfterTimes: [searchDurationB],
errors: searchErrorsB,
}),
]);
}
logger.debug(buildRuleMessage(`sortIds: ${sortIds}`));
if (hasSortId) {
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
buildRuleMessage,
searchAfterSortId: sortId,
searchAfterSortIds: sortIds,
index: inputIndexPattern,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
@ -132,7 +82,6 @@ export const searchAfterAndBulkCreate = async ({
filter,
pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
timestampOverride: ruleParams.timestampOverride,
excludeDocsWithTimestampOverride: false,
});
mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]);
toReturn = mergeReturns([
@ -147,10 +96,11 @@ export const searchAfterAndBulkCreate = async ({
}),
]);
const lastSortId = searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort;
if (lastSortId != null && lastSortId.length !== 0) {
// @ts-expect-error @elastic/elasticsearch SortResults contains null not assignable to sortId
sortId = lastSortId[0];
const lastSortIds = getSafeSortIds(
searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort
);
if (lastSortIds != null && lastSortIds.length !== 0) {
sortIds = lastSortIds;
hasSortId = true;
} else {
hasSortId = false;
@ -236,7 +186,7 @@ export const searchAfterAndBulkCreate = async ({
sendAlertTelemetryEvents(logger, eventsTelemetry, filteredEvents, buildRuleMessage);
}
if (!hasSortId && !hasBackupSortId) {
if (!hasSortId) {
logger.debug(buildRuleMessage('ran out of sort ids to sort on'));
break;
}

19
x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts
View file

@ -34,7 +34,7 @@ describe('singleSearchAfter', () => {
elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId())
);
const { searchResult } = await singleSearchAfter({
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
index: [],
from: 'now-360s',
to: 'now',
@ -44,7 +44,6 @@ describe('singleSearchAfter', () => {
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
});
expect(searchResult).toEqual(sampleDocSearchResultsNoSortId());
});
@ -53,7 +52,7 @@ describe('singleSearchAfter', () => {
elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId())
);
const { searchErrors } = await singleSearchAfter({
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
index: [],
from: 'now-360s',
to: 'now',
@ -63,7 +62,6 @@ describe('singleSearchAfter', () => {
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
});
expect(searchErrors).toEqual([]);
});
@ -104,7 +102,7 @@ describe('singleSearchAfter', () => {
})
);
const { searchErrors } = await singleSearchAfter({
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
index: [],
from: 'now-360s',
to: 'now',
@ -114,21 +112,20 @@ describe('singleSearchAfter', () => {
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
});
expect(searchErrors).toEqual([
'index: "index-123" reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"',
]);
});
test('if singleSearchAfter works with a given sort id', async () => {
const searchAfterSortId = '1234567891111';
const searchAfterSortIds = ['1234567891111'];
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
elasticsearchClientMock.createSuccessTransportRequestPromise(
sampleDocSearchResultsWithSortId()
)
);
const { searchResult } = await singleSearchAfter({
searchAfterSortId,
searchAfterSortIds,
index: [],
from: 'now-360s',
to: 'now',
@ -138,18 +135,17 @@ describe('singleSearchAfter', () => {
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
});
expect(searchResult).toEqual(sampleDocSearchResultsWithSortId());
});
test('if singleSearchAfter throws error', async () => {
const searchAfterSortId = '1234567891111';
const searchAfterSortIds = ['1234567891111'];
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
elasticsearchClientMock.createErrorTransportRequestPromise(new Error('Fake Error'))
);
await expect(
singleSearchAfter({
searchAfterSortId,
searchAfterSortIds,
index: [],
from: 'now-360s',
to: 'now',
@ -159,7 +155,6 @@ describe('singleSearchAfter', () => {
filter: undefined,
timestampOverride: undefined,
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
})
).rejects.toThrow('Fake Error');
});

14
x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts
View file

@ -6,6 +6,7 @@
*/
import type { estypes } from '@elastic/elasticsearch';
import { performance } from 'perf_hooks';
import { SearchRequest, SortResults } from '@elastic/elasticsearch/api/types';
import {
AlertInstanceContext,
AlertInstanceState,
@ -23,7 +24,7 @@ import {
interface SingleSearchAfterParams {
aggregations?: Record<string, estypes.AggregationContainer>;
searchAfterSortId: string | undefined;
searchAfterSortIds: SortResults | undefined;
index: string[];
from: string;
to: string;
@ -34,13 +35,12 @@ interface SingleSearchAfterParams {
filter?: estypes.QueryContainer;
timestampOverride: TimestampOverrideOrUndefined;
buildRuleMessage: BuildRuleMessage;
excludeDocsWithTimestampOverride: boolean;
}
// utilize search_after for paging results into bulk.
export const singleSearchAfter = async ({
aggregations,
searchAfterSortId,
searchAfterSortIds,
index,
from,
to,
@ -51,7 +51,6 @@ export const singleSearchAfter = async ({
sortOrder,
timestampOverride,
buildRuleMessage,
excludeDocsWithTimestampOverride,
}: SingleSearchAfterParams): Promise<{
searchResult: SignalSearchResponse;
searchDuration: string;
@ -66,15 +65,16 @@ export const singleSearchAfter = async ({
filter,
size: pageSize,
sortOrder,
searchAfterSortId,
searchAfterSortIds,
timestampOverride,
excludeDocsWithTimestampOverride,
});
const start = performance.now();
const {
body: nextSearchAfterResult,
} = await services.scopedClusterClient.asCurrentUser.search<SignalSource>(searchAfterQuery);
} = await services.scopedClusterClient.asCurrentUser.search<SignalSource>(
searchAfterQuery as SearchRequest
);
const end = performance.now();
const searchErrors = createErrorsFromShard({
errors: nextSearchAfterResult._shards.failures ?? [],

3
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_previous_threshold_signals.ts
View file

@ -71,7 +71,7 @@ export const findPreviousThresholdSignals = async ({
};
return singleSearchAfter({
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride,
index: indexPattern,
from,
@ -81,6 +81,5 @@ export const findPreviousThresholdSignals = async ({
filter,
pageSize: 10000, // TODO: multiple pages?
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
});
};

1
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts
View file

@ -141,6 +141,5 @@ export const findThresholdSignals = async ({
pageSize: 1,
sortOrder: 'desc',
buildRuleMessage,
excludeDocsWithTimestampOverride: false,
});
};

23
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
View file

@ -13,6 +13,7 @@ import type { estypes } from '@elastic/elasticsearch';
import { isEmpty, partition } from 'lodash';
import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport';
import { SortResults } from '@elastic/elasticsearch/api/types';
import {
TimestampOverrideOrUndefined,
Privilege,
@ -846,3 +847,25 @@ export const isThreatParams = (params: RuleParams): params is ThreatRuleParams =
params.type === 'threat_match';
export const isMachineLearningParams = (params: RuleParams): params is MachineLearningRuleParams =>
params.type === 'machine_learning';
/**
* Prevent javascript from returning Number.MAX_SAFE_INTEGER when Elasticsearch expects
* Java's Long.MAX_VALUE. This happens when sorting fields by date which are
* unmapped in the provided index
*
* Ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620
*
* return stringified Long.MAX_VALUE if we receive Number.MAX_SAFE_INTEGER
* @param sortIds SortResults | undefined
* @returns SortResults
*/
export const getSafeSortIds = (sortIds: SortResults | undefined) => {
return sortIds?.map((sortId) => {
// haven't determined when we would receive a null value for a sort id
// but in case we do, default to sending the stringified Java max_int
if (sortId == null || sortId === '' || sortId >= Number.MAX_SAFE_INTEGER) {
return '9223372036854775807';
}
return sortId;
});
};

113
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts
View file

@ -1322,5 +1322,118 @@ export default ({ getService }: FtrProviderContext) => {
});
});
});
describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => {
beforeEach(async () => {
await createSignalsIndex(supertest);
await esArchiver.load('auditbeat/hosts');
});
afterEach(async () => {
await deleteSignalsIndex(supertest);
await deleteAllAlerts(supertest);
await esArchiver.unload('auditbeat/hosts');
});
/**
* This represents our worst case scenario where this field is not mapped on any index
* We want to check that our logic continues to function within the constraints of search after
* Elasticsearch returns java's long.MAX_VALUE for unmapped date fields
* Javascript does not support numbers this large, but without passing in a number of this size
* The search_after will continue to return the same results and not iterate to the next set
* So to circumvent this limitation of javascript we return the stringified version of Java's
* Long.MAX_VALUE so that search_after does not enter into an infinite loop.
*
* ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620
*/
it('should generate 200 signals when timestamp override does not exist', async () => {
const rule: QueryCreateSchema = {
...getRuleForSignalTesting(['auditbeat-*']),
timestamp_override: 'event.fakeingested',
max_signals: 200,
};
const { id } = await createRule(supertest, rule);
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
await waitForSignalsToBePresent(supertest, 200, [id]);
const signalsResponse = await getSignalsByIds(supertest, [id], 200);
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
expect(signals.length).equal(200);
});
});
/**
* Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override,
* then the documents will be queried and sorted using the timestamp override field.
* If no timestamp override field exists in the indices but one was provided to the rule,
* the rule's query will additionally search for events using the `@timestamp` field
*/
describe('Signals generated from events with timestamp override field', async () => {
beforeEach(async () => {
await deleteSignalsIndex(supertest);
await createSignalsIndex(supertest);
await esArchiver.load('security_solution/timestamp_override_1');
await esArchiver.load('security_solution/timestamp_override_2');
await esArchiver.load('security_solution/timestamp_override_3');
await esArchiver.load('security_solution/timestamp_override_4');
});
afterEach(async () => {
await deleteSignalsIndex(supertest);
await deleteAllAlerts(supertest);
await esArchiver.unload('security_solution/timestamp_override_1');
await esArchiver.unload('security_solution/timestamp_override_2');
await esArchiver.unload('security_solution/timestamp_override_3');
await esArchiver.unload('security_solution/timestamp_override_4');
});
it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => {
const rule: QueryCreateSchema = {
...getRuleForSignalTesting(['myfa*']),
timestamp_override: 'event.ingested',
};
const { id } = await createRule(supertest, rule);
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
await waitForSignalsToBePresent(supertest, 3, [id]);
const signalsResponse = await getSignalsByIds(supertest, [id], 3);
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
expect(signalsOrderedByEventId.length).equal(3);
});
it('should generate 2 signals with @timestamp', async () => {
const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']);
const { id } = await createRule(supertest, rule);
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsResponse = await getSignalsByIds(supertest, [id]);
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
expect(signalsOrderedByEventId.length).equal(2);
});
it('should generate 2 signals when timestamp override does not exist', async () => {
const rule: QueryCreateSchema = {
...getRuleForSignalTesting(['myfa*']),
timestamp_override: 'event.fakeingestfield',
};
const { id } = await createRule(supertest, rule);
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsResponse = await getSignalsByIds(supertest, [id, id]);
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
expect(signalsOrderedByEventId.length).equal(2);
});
});
});
};

BIN
x-pack/test/functional/es_archives/security_solution/timestamp_override/data.json.gz
View file

Binary file not shown.

30
x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json
View file

@ -1,19 +1,19 @@
{
"type": "index",
"value": {
"index": "myfakeindex-1",
"mappings" : {
"properties" : {
"message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
"type": "index",
"value": {
"index": "myfakeindex-1",
"mappings": {
"properties": {
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}

10
x-pack/test/functional/es_archives/security_solution/timestamp_override_1/data.json Normal file
View file

@ -0,0 +1,10 @@
{
"type": "doc",
"value": {
"index": "myfakeindex-1",
"source": {
"message": "hello world 1"
},
"type": "_doc"
}
}

19
x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json Normal file
View file

@ -0,0 +1,19 @@
{
"type": "index",
"value": {
"index": "myfakeindex-1",
"mappings": {
"properties": {
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}

13
x-pack/test/functional/es_archives/security_solution/timestamp_override_2/data.json Normal file
View file

@ -0,0 +1,13 @@
{
"type": "doc",
"value": {
"index": "myfakeindex-2",
"source": {
"message": "hello world 2",
"event": {
"ingested": "2020-12-16T15:16:18.570Z"
}
},
"type": "_doc"
}
}

26
x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json Normal file
View file

@ -0,0 +1,26 @@
{
"type": "index",
"value": {
"index": "myfakeindex-2",
"mappings": {
"properties": {
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event": {
"properties": {
"ingested": {
"type": "date"
}
}
}
}
}
}
}

11
x-pack/test/functional/es_archives/security_solution/timestamp_override_3/data.json Normal file
View file

@ -0,0 +1,11 @@
{
"type": "doc",
"value": {
"index": "myfakeindex-3",
"source": {
"message": "hello world 3",
"@timestamp": "2020-12-16T15:16:18.570Z"
},
"type": "_doc"
}
}

22
x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json Normal file
View file

@ -0,0 +1,22 @@
{
"type": "index",
"value": {
"index": "myfakeindex-3",
"mappings": {
"properties": {
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"@timestamp": {
"type": "date"
}
}
}
}
}

14
x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json Normal file
View file

@ -0,0 +1,14 @@
{
"type": "doc",
"value": {
"index": "myfakeindex-4",
"source": {
"message": "hello world 4",
"@timestamp": "2020-12-16T15:16:18.570Z",
"event": {
"ingested": "2020-12-16T15:16:18.570Z"
}
},
"type": "_doc"
}
}

29
x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json Normal file
View file

@ -0,0 +1,29 @@
{
"type": "index",
"value": {
"index": "myfakeindex-4",
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"message": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"event": {
"properties": {
"ingested": {
"type": "date"
}
}
}
}
}
}
}