[Security Solution] [Detections] Combine multiple timestamp searches into single request (#96078)
* merge multiple timestamp queries into one single search * fix types and unit tests * remove unused code for sending secondary search * removes unused excludeDocsWithTimestampOverride * adds integration tests to cover cases that should / should not generate signals when timestamp override is present in rule * adds integration test to ensure unmapped sort fields do not break search after functionality of detection rules * Need to figure out why moving the tests around fixed them... * updates tests with new es archive data and fixes bug where exclusion filter was hardcoded to event.ingested :yikes: * remove dead commented out code * fixes typo in test file, removes redundant delete signals call in integration test, fixes logic for possibility of receving a null value in sort ids, removes unused utility function for checking valid sort ids * a unit test for checking if an empty string of a sort id is present was failing because we moved the logic for checking that out of the build search query function and up into the big loop. So I moved that unit test into the search after bulk create test file. * fix types * removes isEmpty since it doesn't check for empty strings
This commit is contained in:
parent
f37492069a
commit
4d2414e7f5
|
@ -115,6 +115,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
|
|||
|
||||
export const sampleDocWithSortId = (
|
||||
someUuid: string = sampleIdGuid,
|
||||
sortIds: string[] = ['1234567891111', '2233447556677'],
|
||||
ip?: string | string[],
|
||||
destIp?: string | string[]
|
||||
): SignalSourceHit => ({
|
||||
|
@ -139,7 +140,7 @@ export const sampleDocWithSortId = (
|
|||
'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'],
|
||||
'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'],
|
||||
},
|
||||
sort: ['1234567891111'],
|
||||
sort: sortIds,
|
||||
});
|
||||
|
||||
export const sampleDocNoSortId = (
|
||||
|
@ -630,7 +631,8 @@ export const repeatedSearchResultsWithSortId = (
|
|||
pageSize: number,
|
||||
guids: string[],
|
||||
ips?: Array<string | string[]>,
|
||||
destIps?: Array<string | string[]>
|
||||
destIps?: Array<string | string[]>,
|
||||
sortIds?: string[]
|
||||
): SignalSearchResponse => ({
|
||||
took: 10,
|
||||
timed_out: false,
|
||||
|
@ -646,6 +648,7 @@ export const repeatedSearchResultsWithSortId = (
|
|||
hits: Array.from({ length: pageSize }).map((x, index) => ({
|
||||
...sampleDocWithSortId(
|
||||
guids[index],
|
||||
sortIds,
|
||||
ips ? ips[index] : '127.0.0.1',
|
||||
destIps ? destIps[index] : '127.0.0.1'
|
||||
),
|
||||
|
|
|
@ -15,9 +15,8 @@ describe('create_signals', () => {
|
|||
to: 'today',
|
||||
filter: {},
|
||||
size: 100,
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
timestampOverride: undefined,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
|
@ -39,12 +38,19 @@ describe('create_signals', () => {
|
|||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
|
@ -73,16 +79,16 @@ describe('create_signals', () => {
|
|||
},
|
||||
});
|
||||
});
|
||||
test('if searchAfterSortId is an empty string it should not be included', () => {
|
||||
|
||||
test('it builds a now-5m up to today filter with timestampOverride', () => {
|
||||
const query = buildEventsSearchQuery({
|
||||
index: ['auditbeat-*'],
|
||||
from: 'now-5m',
|
||||
to: 'today',
|
||||
filter: {},
|
||||
size: 100,
|
||||
searchAfterSortId: '',
|
||||
timestampOverride: undefined,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
searchAfterSortIds: undefined,
|
||||
timestampOverride: 'event.ingested',
|
||||
});
|
||||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
|
@ -91,6 +97,10 @@ describe('create_signals', () => {
|
|||
ignore_unavailable: true,
|
||||
body: {
|
||||
docvalue_fields: [
|
||||
{
|
||||
field: 'event.ingested',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
{
|
||||
field: '@timestamp',
|
||||
format: 'strict_date_optional_time',
|
||||
|
@ -104,12 +114,43 @@ describe('create_signals', () => {
|
|||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'event.ingested': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
bool: {
|
||||
must_not: {
|
||||
exists: {
|
||||
field: 'event.ingested',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
|
@ -128,6 +169,12 @@ describe('create_signals', () => {
|
|||
},
|
||||
],
|
||||
sort: [
|
||||
{
|
||||
'event.ingested': {
|
||||
order: 'asc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
{
|
||||
'@timestamp': {
|
||||
order: 'asc',
|
||||
|
@ -138,7 +185,8 @@ describe('create_signals', () => {
|
|||
},
|
||||
});
|
||||
});
|
||||
test('if searchAfterSortId is a valid sortId string', () => {
|
||||
|
||||
test('if searchAfterSortIds is a valid sortId string', () => {
|
||||
const fakeSortId = '123456789012';
|
||||
const query = buildEventsSearchQuery({
|
||||
index: ['auditbeat-*'],
|
||||
|
@ -146,9 +194,8 @@ describe('create_signals', () => {
|
|||
to: 'today',
|
||||
filter: {},
|
||||
size: 100,
|
||||
searchAfterSortId: fakeSortId,
|
||||
searchAfterSortIds: [fakeSortId],
|
||||
timestampOverride: undefined,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
|
@ -170,12 +217,19 @@ describe('create_signals', () => {
|
|||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
|
@ -205,7 +259,7 @@ describe('create_signals', () => {
|
|||
},
|
||||
});
|
||||
});
|
||||
test('if searchAfterSortId is a valid sortId number', () => {
|
||||
test('if searchAfterSortIds is a valid sortId number', () => {
|
||||
const fakeSortIdNumber = 123456789012;
|
||||
const query = buildEventsSearchQuery({
|
||||
index: ['auditbeat-*'],
|
||||
|
@ -213,9 +267,8 @@ describe('create_signals', () => {
|
|||
to: 'today',
|
||||
filter: {},
|
||||
size: 100,
|
||||
searchAfterSortId: fakeSortIdNumber,
|
||||
searchAfterSortIds: [fakeSortIdNumber],
|
||||
timestampOverride: undefined,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
|
@ -237,12 +290,19 @@ describe('create_signals', () => {
|
|||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
|
@ -279,9 +339,8 @@ describe('create_signals', () => {
|
|||
to: 'today',
|
||||
filter: {},
|
||||
size: 100,
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
timestampOverride: undefined,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
|
@ -303,12 +362,19 @@ describe('create_signals', () => {
|
|||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
|
@ -352,9 +418,8 @@ describe('create_signals', () => {
|
|||
to: 'today',
|
||||
filter: {},
|
||||
size: 100,
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
timestampOverride: undefined,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(query).toEqual({
|
||||
allow_no_indices: true,
|
||||
|
@ -371,12 +436,19 @@ describe('create_signals', () => {
|
|||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-5m',
|
||||
lte: 'today',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
|
|
|
@ -5,6 +5,8 @@
|
|||
* 2.0.
|
||||
*/
|
||||
import type { estypes } from '@elastic/elasticsearch';
|
||||
import { SortResults } from '@elastic/elasticsearch/api/types';
|
||||
import { isEmpty } from 'lodash';
|
||||
import {
|
||||
SortOrderOrUndefined,
|
||||
TimestampOverrideOrUndefined,
|
||||
|
@ -18,9 +20,8 @@ interface BuildEventsSearchQuery {
|
|||
filter?: estypes.QueryContainer;
|
||||
size: number;
|
||||
sortOrder?: SortOrderOrUndefined;
|
||||
searchAfterSortId: string | number | undefined;
|
||||
searchAfterSortIds: SortResults | undefined;
|
||||
timestampOverride: TimestampOverrideOrUndefined;
|
||||
excludeDocsWithTimestampOverride: boolean;
|
||||
}
|
||||
|
||||
export const buildEventsSearchQuery = ({
|
||||
|
@ -30,10 +31,9 @@ export const buildEventsSearchQuery = ({
|
|||
to,
|
||||
filter,
|
||||
size,
|
||||
searchAfterSortId,
|
||||
searchAfterSortIds,
|
||||
sortOrder,
|
||||
timestampOverride,
|
||||
excludeDocsWithTimestampOverride,
|
||||
}: BuildEventsSearchQuery) => {
|
||||
const defaultTimeFields = ['@timestamp'];
|
||||
const timestamps =
|
||||
|
@ -43,36 +43,62 @@ export const buildEventsSearchQuery = ({
|
|||
format: 'strict_date_optional_time',
|
||||
}));
|
||||
|
||||
const sortField =
|
||||
timestampOverride != null && !excludeDocsWithTimestampOverride
|
||||
? timestampOverride
|
||||
: '@timestamp';
|
||||
|
||||
const rangeFilter: estypes.QueryContainer[] = [
|
||||
{
|
||||
range: {
|
||||
[sortField]: {
|
||||
lte: to,
|
||||
gte: from,
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
if (excludeDocsWithTimestampOverride) {
|
||||
rangeFilter.push({
|
||||
bool: {
|
||||
must_not: {
|
||||
exists: {
|
||||
field: timestampOverride,
|
||||
const rangeFilter: estypes.QueryContainer[] =
|
||||
timestampOverride != null
|
||||
? [
|
||||
{
|
||||
range: {
|
||||
[timestampOverride]: {
|
||||
lte: to,
|
||||
gte: from,
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
// @ts-expect-error undefined in not assignable to QueryContainer
|
||||
// but tests contain undefined, so I suppose it's desired behaviour
|
||||
const filterWithTime: estypes.QueryContainer[] = [filter, { bool: { filter: rangeFilter } }];
|
||||
{
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
lte: to,
|
||||
gte: from,
|
||||
// @ts-expect-error
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
bool: {
|
||||
must_not: {
|
||||
exists: {
|
||||
field: timestampOverride,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
lte: to,
|
||||
gte: from,
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const filterWithTime: estypes.QueryContainer[] = [
|
||||
// but tests contain undefined, so I suppose it's desired behaviour
|
||||
// @ts-expect-error undefined in not assignable to QueryContainer
|
||||
filter,
|
||||
{ bool: { filter: [{ bool: { should: [...rangeFilter], minimum_should_match: 1 } }] } },
|
||||
];
|
||||
|
||||
const searchQuery = {
|
||||
allow_no_indices: true,
|
||||
|
@ -99,22 +125,39 @@ export const buildEventsSearchQuery = ({
|
|||
],
|
||||
...(aggregations ? { aggregations } : {}),
|
||||
sort: [
|
||||
{
|
||||
[sortField]: {
|
||||
order: sortOrder ?? 'asc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
...(timestampOverride != null
|
||||
? [
|
||||
{
|
||||
[timestampOverride]: {
|
||||
order: sortOrder ?? 'asc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
{
|
||||
'@timestamp': {
|
||||
order: sortOrder ?? 'asc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
'@timestamp': {
|
||||
order: sortOrder ?? 'asc',
|
||||
unmapped_type: 'date',
|
||||
},
|
||||
},
|
||||
]),
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
if (searchAfterSortId) {
|
||||
if (searchAfterSortIds != null && !isEmpty(searchAfterSortIds)) {
|
||||
return {
|
||||
...searchQuery,
|
||||
body: {
|
||||
...searchQuery.body,
|
||||
search_after: [searchAfterSortId],
|
||||
search_after: searchAfterSortIds,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
@ -17,7 +17,7 @@ import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'
|
|||
describe('filterEventsAgainstList', () => {
|
||||
let listClient = listMock.getListClient();
|
||||
let exceptionItem = getExceptionListItemSchemaMock();
|
||||
let events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
@ -44,7 +44,7 @@ describe('filterEventsAgainstList', () => {
|
|||
},
|
||||
],
|
||||
};
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
@ -111,7 +111,7 @@ describe('filterEventsAgainstList', () => {
|
|||
});
|
||||
|
||||
test('it returns a single matched set as a JSON.stringify() set from the "events"', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
(exceptionItem.entries[0] as EntryList).field = 'source.ip';
|
||||
const [{ matchedSet }] = await createFieldAndSetTuples({
|
||||
listClient,
|
||||
|
@ -124,7 +124,10 @@ describe('filterEventsAgainstList', () => {
|
|||
});
|
||||
|
||||
test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('456', undefined, '2.2.2.2'),
|
||||
];
|
||||
(exceptionItem.entries[0] as EntryList).field = 'source.ip';
|
||||
const [{ matchedSet }] = await createFieldAndSetTuples({
|
||||
listClient,
|
||||
|
@ -137,7 +140,7 @@ describe('filterEventsAgainstList', () => {
|
|||
});
|
||||
|
||||
test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => {
|
||||
events = [sampleDocWithSortId('123', ['1.1.1.1', '2.2.2.2'])];
|
||||
events = [sampleDocWithSortId('123', undefined, ['1.1.1.1', '2.2.2.2'])];
|
||||
(exceptionItem.entries[0] as EntryList).field = 'source.ip';
|
||||
const [{ matchedSet }] = await createFieldAndSetTuples({
|
||||
listClient,
|
||||
|
@ -150,7 +153,10 @@ describe('filterEventsAgainstList', () => {
|
|||
});
|
||||
|
||||
test('it returns 2 fields when given two exception list items', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('456', undefined, '2.2.2.2'),
|
||||
];
|
||||
exceptionItem.entries = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
|
@ -182,7 +188,10 @@ describe('filterEventsAgainstList', () => {
|
|||
});
|
||||
|
||||
test('it returns two matched sets from two different events, one excluded, and one included', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('456', undefined, '2.2.2.2'),
|
||||
];
|
||||
exceptionItem.entries = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
|
@ -215,7 +224,10 @@ describe('filterEventsAgainstList', () => {
|
|||
});
|
||||
|
||||
test('it returns two fields from two different events', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('456', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('456', undefined, '2.2.2.2'),
|
||||
];
|
||||
exceptionItem.entries = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
|
@ -249,8 +261,8 @@ describe('filterEventsAgainstList', () => {
|
|||
|
||||
test('it returns two matches from two different events', async () => {
|
||||
events = [
|
||||
sampleDocWithSortId('123', '1.1.1.1', '3.3.3.3'),
|
||||
sampleDocWithSortId('456', '2.2.2.2', '5.5.5.5'),
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1', '3.3.3.3'),
|
||||
sampleDocWithSortId('456', undefined, '2.2.2.2', '5.5.5.5'),
|
||||
];
|
||||
exceptionItem.entries = [
|
||||
{
|
||||
|
|
|
@ -14,7 +14,7 @@ import { buildRuleMessageMock as buildRuleMessage } from '../rule_messages.mock'
|
|||
|
||||
describe('createSetToFilterAgainst', () => {
|
||||
let listClient = listMock.getListClient();
|
||||
let events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
@ -27,7 +27,7 @@ describe('createSetToFilterAgainst', () => {
|
|||
}))
|
||||
)
|
||||
);
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
@ -49,7 +49,7 @@ describe('createSetToFilterAgainst', () => {
|
|||
});
|
||||
|
||||
test('it returns 1 field if the list returns a single item', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
const field = await createSetToFilterAgainst({
|
||||
events,
|
||||
field: 'source.ip',
|
||||
|
@ -68,7 +68,10 @@ describe('createSetToFilterAgainst', () => {
|
|||
});
|
||||
|
||||
test('it returns 2 fields if the list returns 2 items', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('123', undefined, '2.2.2.2'),
|
||||
];
|
||||
const field = await createSetToFilterAgainst({
|
||||
events,
|
||||
field: 'source.ip',
|
||||
|
@ -87,7 +90,10 @@ describe('createSetToFilterAgainst', () => {
|
|||
});
|
||||
|
||||
test('it returns 0 fields if the field does not match up to a valid field within the event', async () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('123', undefined, '2.2.2.2'),
|
||||
];
|
||||
const field = await createSetToFilterAgainst({
|
||||
events,
|
||||
field: 'nonexistent.field', // field does not exist
|
||||
|
|
|
@ -14,7 +14,7 @@ import { FieldSet } from './types';
|
|||
|
||||
describe('filterEvents', () => {
|
||||
let listClient = listMock.getListClient();
|
||||
let events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
@ -27,7 +27,7 @@ describe('filterEvents', () => {
|
|||
}))
|
||||
)
|
||||
);
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
@ -35,7 +35,7 @@ describe('filterEvents', () => {
|
|||
});
|
||||
|
||||
test('it filters out the event if it is "included"', () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
const fieldAndSetTuples: FieldSet[] = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
|
@ -51,7 +51,7 @@ describe('filterEvents', () => {
|
|||
});
|
||||
|
||||
test('it does not filter out the event if it is "excluded"', () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
const fieldAndSetTuples: FieldSet[] = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
|
@ -67,7 +67,7 @@ describe('filterEvents', () => {
|
|||
});
|
||||
|
||||
test('it does NOT filter out the event if the field is not found', () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1')];
|
||||
events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];
|
||||
const fieldAndSetTuples: FieldSet[] = [
|
||||
{
|
||||
field: 'madeup.nonexistent', // field does not exist
|
||||
|
@ -83,7 +83,10 @@ describe('filterEvents', () => {
|
|||
});
|
||||
|
||||
test('it does NOT filter out the event if it is in both an inclusion and exclusion list', () => {
|
||||
events = [sampleDocWithSortId('123', '1.1.1.1'), sampleDocWithSortId('123', '2.2.2.2')];
|
||||
events = [
|
||||
sampleDocWithSortId('123', undefined, '1.1.1.1'),
|
||||
sampleDocWithSortId('123', undefined, '2.2.2.2'),
|
||||
];
|
||||
const fieldAndSetTuples: FieldSet[] = [
|
||||
{
|
||||
field: 'source.ip',
|
||||
|
|
|
@ -426,6 +426,84 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
|
||||
});
|
||||
|
||||
test('should return success when empty string sortId present', async () => {
|
||||
mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce(
|
||||
elasticsearchClientMock.createSuccessTransportRequestPromise({
|
||||
took: 100,
|
||||
errors: false,
|
||||
items: [
|
||||
{
|
||||
create: {
|
||||
_id: someGuids[0],
|
||||
_index: 'myfakeindex',
|
||||
status: 201,
|
||||
},
|
||||
},
|
||||
{
|
||||
create: {
|
||||
_id: someGuids[1],
|
||||
_index: 'myfakeindex',
|
||||
status: 201,
|
||||
},
|
||||
},
|
||||
{
|
||||
create: {
|
||||
_id: someGuids[2],
|
||||
_index: 'myfakeindex',
|
||||
status: 201,
|
||||
},
|
||||
},
|
||||
{
|
||||
create: {
|
||||
_id: someGuids[3],
|
||||
_index: 'myfakeindex',
|
||||
status: 201,
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
mockService.scopedClusterClient.asCurrentUser.search
|
||||
.mockResolvedValueOnce(
|
||||
elasticsearchClientMock.createSuccessTransportRequestPromise(
|
||||
repeatedSearchResultsWithSortId(
|
||||
4,
|
||||
4,
|
||||
someGuids.slice(0, 3),
|
||||
['1.1.1.1', '2.2.2.2', '2.2.2.2', '2.2.2.2'],
|
||||
// this is the case we are testing, if we receive an empty string for one of the sort ids.
|
||||
['', '2222222222222']
|
||||
)
|
||||
)
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
elasticsearchClientMock.createSuccessTransportRequestPromise(
|
||||
sampleDocSearchResultsNoSortIdNoHits()
|
||||
)
|
||||
);
|
||||
|
||||
const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
|
||||
ruleSO,
|
||||
tuples,
|
||||
listClient,
|
||||
exceptionsList: [],
|
||||
services: mockService,
|
||||
logger: mockLogger,
|
||||
eventsTelemetry: undefined,
|
||||
id: sampleRuleGuid,
|
||||
inputIndexPattern,
|
||||
signalsIndex: DEFAULT_SIGNALS_INDEX,
|
||||
pageSize: 1,
|
||||
filter: undefined,
|
||||
refresh: false,
|
||||
buildRuleMessage,
|
||||
});
|
||||
expect(success).toEqual(true);
|
||||
expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
|
||||
expect(createdSignalsCount).toEqual(4);
|
||||
expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
|
||||
});
|
||||
|
||||
test('should return success when all search results are in the allowlist and no sortId present', async () => {
|
||||
const searchListItems: SearchListItemArraySchema = [
|
||||
{ ...getSearchListItemResponseMock(), value: ['1.1.1.1'] },
|
||||
|
|
|
@ -5,9 +5,8 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
/* eslint-disable complexity */
|
||||
|
||||
import { identity } from 'lodash';
|
||||
import { SortResults } from '@elastic/elasticsearch/api/types';
|
||||
import { singleSearchAfter } from './single_search_after';
|
||||
import { singleBulkCreate } from './single_bulk_create';
|
||||
import { filterEventsAgainstList } from './filters/filter_events_against_list';
|
||||
|
@ -19,6 +18,7 @@ import {
|
|||
createTotalHitsFromSearchResult,
|
||||
mergeReturns,
|
||||
mergeSearchResults,
|
||||
getSafeSortIds,
|
||||
} from './utils';
|
||||
import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types';
|
||||
|
||||
|
@ -44,10 +44,8 @@ export const searchAfterAndBulkCreate = async ({
|
|||
let toReturn = createSearchAfterReturnType();
|
||||
|
||||
// sortId tells us where to start our next consecutive search_after query
|
||||
let sortId: string | undefined;
|
||||
let sortIds: SortResults | undefined;
|
||||
let hasSortId = true; // default to true so we execute the search on initial run
|
||||
let backupSortId: string | undefined;
|
||||
let hasBackupSortId = ruleParams.timestampOverride ? true : false;
|
||||
|
||||
// signalsCreatedCount keeps track of how many signals we have created,
|
||||
// to ensure we don't exceed maxSignals
|
||||
|
@ -69,60 +67,12 @@ export const searchAfterAndBulkCreate = async ({
|
|||
while (signalsCreatedCount < tuple.maxSignals) {
|
||||
try {
|
||||
let mergedSearchResults = createSearchResultReturnType();
|
||||
logger.debug(buildRuleMessage(`sortIds: ${sortId}`));
|
||||
|
||||
// if there is a timestampOverride param we always want to do a secondary search against @timestamp
|
||||
if (ruleParams.timestampOverride != null && hasBackupSortId) {
|
||||
// only execute search if we have something to sort on or if it is the first search
|
||||
const {
|
||||
searchResult: searchResultB,
|
||||
searchDuration: searchDurationB,
|
||||
searchErrors: searchErrorsB,
|
||||
} = await singleSearchAfter({
|
||||
buildRuleMessage,
|
||||
searchAfterSortId: backupSortId,
|
||||
index: inputIndexPattern,
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
logger,
|
||||
// @ts-expect-error please, declare a type explicitly instead of unknown
|
||||
filter,
|
||||
pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
excludeDocsWithTimestampOverride: true,
|
||||
});
|
||||
|
||||
// call this function setSortIdOrExit()
|
||||
const lastSortId = searchResultB?.hits?.hits[searchResultB.hits.hits.length - 1]?.sort;
|
||||
if (lastSortId != null && lastSortId.length !== 0) {
|
||||
// @ts-expect-error @elastic/elasticsearch SortResults contains null not assignable to backupSortId
|
||||
backupSortId = lastSortId[0];
|
||||
hasBackupSortId = true;
|
||||
} else {
|
||||
logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB'));
|
||||
hasBackupSortId = false;
|
||||
}
|
||||
|
||||
mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]);
|
||||
|
||||
toReturn = mergeReturns([
|
||||
toReturn,
|
||||
createSearchAfterReturnTypeFromResponse({
|
||||
searchResult: mergedSearchResults,
|
||||
timestampOverride: undefined,
|
||||
}),
|
||||
createSearchAfterReturnType({
|
||||
searchAfterTimes: [searchDurationB],
|
||||
errors: searchErrorsB,
|
||||
}),
|
||||
]);
|
||||
}
|
||||
logger.debug(buildRuleMessage(`sortIds: ${sortIds}`));
|
||||
|
||||
if (hasSortId) {
|
||||
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
|
||||
buildRuleMessage,
|
||||
searchAfterSortId: sortId,
|
||||
searchAfterSortIds: sortIds,
|
||||
index: inputIndexPattern,
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
|
@ -132,7 +82,6 @@ export const searchAfterAndBulkCreate = async ({
|
|||
filter,
|
||||
pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]);
|
||||
toReturn = mergeReturns([
|
||||
|
@ -147,10 +96,11 @@ export const searchAfterAndBulkCreate = async ({
|
|||
}),
|
||||
]);
|
||||
|
||||
const lastSortId = searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort;
|
||||
if (lastSortId != null && lastSortId.length !== 0) {
|
||||
// @ts-expect-error @elastic/elasticsearch SortResults contains null not assignable to sortId
|
||||
sortId = lastSortId[0];
|
||||
const lastSortIds = getSafeSortIds(
|
||||
searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort
|
||||
);
|
||||
if (lastSortIds != null && lastSortIds.length !== 0) {
|
||||
sortIds = lastSortIds;
|
||||
hasSortId = true;
|
||||
} else {
|
||||
hasSortId = false;
|
||||
|
@ -236,7 +186,7 @@ export const searchAfterAndBulkCreate = async ({
|
|||
sendAlertTelemetryEvents(logger, eventsTelemetry, filteredEvents, buildRuleMessage);
|
||||
}
|
||||
|
||||
if (!hasSortId && !hasBackupSortId) {
|
||||
if (!hasSortId) {
|
||||
logger.debug(buildRuleMessage('ran out of sort ids to sort on'));
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -34,7 +34,7 @@ describe('singleSearchAfter', () => {
|
|||
elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId())
|
||||
);
|
||||
const { searchResult } = await singleSearchAfter({
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
|
@ -44,7 +44,6 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(searchResult).toEqual(sampleDocSearchResultsNoSortId());
|
||||
});
|
||||
|
@ -53,7 +52,7 @@ describe('singleSearchAfter', () => {
|
|||
elasticsearchClientMock.createSuccessTransportRequestPromise(sampleDocSearchResultsNoSortId())
|
||||
);
|
||||
const { searchErrors } = await singleSearchAfter({
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
|
@ -63,7 +62,6 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(searchErrors).toEqual([]);
|
||||
});
|
||||
|
@ -104,7 +102,7 @@ describe('singleSearchAfter', () => {
|
|||
})
|
||||
);
|
||||
const { searchErrors } = await singleSearchAfter({
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
|
@ -114,21 +112,20 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(searchErrors).toEqual([
|
||||
'index: "index-123" reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"',
|
||||
]);
|
||||
});
|
||||
test('if singleSearchAfter works with a given sort id', async () => {
|
||||
const searchAfterSortId = '1234567891111';
|
||||
const searchAfterSortIds = ['1234567891111'];
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
elasticsearchClientMock.createSuccessTransportRequestPromise(
|
||||
sampleDocSearchResultsWithSortId()
|
||||
)
|
||||
);
|
||||
const { searchResult } = await singleSearchAfter({
|
||||
searchAfterSortId,
|
||||
searchAfterSortIds,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
|
@ -138,18 +135,17 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
expect(searchResult).toEqual(sampleDocSearchResultsWithSortId());
|
||||
});
|
||||
test('if singleSearchAfter throws error', async () => {
|
||||
const searchAfterSortId = '1234567891111';
|
||||
const searchAfterSortIds = ['1234567891111'];
|
||||
mockService.scopedClusterClient.asCurrentUser.search.mockResolvedValueOnce(
|
||||
elasticsearchClientMock.createErrorTransportRequestPromise(new Error('Fake Error'))
|
||||
);
|
||||
await expect(
|
||||
singleSearchAfter({
|
||||
searchAfterSortId,
|
||||
searchAfterSortIds,
|
||||
index: [],
|
||||
from: 'now-360s',
|
||||
to: 'now',
|
||||
|
@ -159,7 +155,6 @@ describe('singleSearchAfter', () => {
|
|||
filter: undefined,
|
||||
timestampOverride: undefined,
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
})
|
||||
).rejects.toThrow('Fake Error');
|
||||
});
|
||||
|
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
import type { estypes } from '@elastic/elasticsearch';
|
||||
import { performance } from 'perf_hooks';
|
||||
import { SearchRequest, SortResults } from '@elastic/elasticsearch/api/types';
|
||||
import {
|
||||
AlertInstanceContext,
|
||||
AlertInstanceState,
|
||||
|
@ -23,7 +24,7 @@ import {
|
|||
|
||||
interface SingleSearchAfterParams {
|
||||
aggregations?: Record<string, estypes.AggregationContainer>;
|
||||
searchAfterSortId: string | undefined;
|
||||
searchAfterSortIds: SortResults | undefined;
|
||||
index: string[];
|
||||
from: string;
|
||||
to: string;
|
||||
|
@ -34,13 +35,12 @@ interface SingleSearchAfterParams {
|
|||
filter?: estypes.QueryContainer;
|
||||
timestampOverride: TimestampOverrideOrUndefined;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
excludeDocsWithTimestampOverride: boolean;
|
||||
}
|
||||
|
||||
// utilize search_after for paging results into bulk.
|
||||
export const singleSearchAfter = async ({
|
||||
aggregations,
|
||||
searchAfterSortId,
|
||||
searchAfterSortIds,
|
||||
index,
|
||||
from,
|
||||
to,
|
||||
|
@ -51,7 +51,6 @@ export const singleSearchAfter = async ({
|
|||
sortOrder,
|
||||
timestampOverride,
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride,
|
||||
}: SingleSearchAfterParams): Promise<{
|
||||
searchResult: SignalSearchResponse;
|
||||
searchDuration: string;
|
||||
|
@ -66,15 +65,16 @@ export const singleSearchAfter = async ({
|
|||
filter,
|
||||
size: pageSize,
|
||||
sortOrder,
|
||||
searchAfterSortId,
|
||||
searchAfterSortIds,
|
||||
timestampOverride,
|
||||
excludeDocsWithTimestampOverride,
|
||||
});
|
||||
|
||||
const start = performance.now();
|
||||
const {
|
||||
body: nextSearchAfterResult,
|
||||
} = await services.scopedClusterClient.asCurrentUser.search<SignalSource>(searchAfterQuery);
|
||||
} = await services.scopedClusterClient.asCurrentUser.search<SignalSource>(
|
||||
searchAfterQuery as SearchRequest
|
||||
);
|
||||
const end = performance.now();
|
||||
const searchErrors = createErrorsFromShard({
|
||||
errors: nextSearchAfterResult._shards.failures ?? [],
|
||||
|
|
|
@ -71,7 +71,7 @@ export const findPreviousThresholdSignals = async ({
|
|||
};
|
||||
|
||||
return singleSearchAfter({
|
||||
searchAfterSortId: undefined,
|
||||
searchAfterSortIds: undefined,
|
||||
timestampOverride,
|
||||
index: indexPattern,
|
||||
from,
|
||||
|
@ -81,6 +81,5 @@ export const findPreviousThresholdSignals = async ({
|
|||
filter,
|
||||
pageSize: 10000, // TODO: multiple pages?
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
};
|
||||
|
|
|
@ -141,6 +141,5 @@ export const findThresholdSignals = async ({
|
|||
pageSize: 1,
|
||||
sortOrder: 'desc',
|
||||
buildRuleMessage,
|
||||
excludeDocsWithTimestampOverride: false,
|
||||
});
|
||||
};
|
||||
|
|
|
@ -13,6 +13,7 @@ import type { estypes } from '@elastic/elasticsearch';
|
|||
import { isEmpty, partition } from 'lodash';
|
||||
import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport';
|
||||
|
||||
import { SortResults } from '@elastic/elasticsearch/api/types';
|
||||
import {
|
||||
TimestampOverrideOrUndefined,
|
||||
Privilege,
|
||||
|
@ -846,3 +847,25 @@ export const isThreatParams = (params: RuleParams): params is ThreatRuleParams =
|
|||
params.type === 'threat_match';
|
||||
export const isMachineLearningParams = (params: RuleParams): params is MachineLearningRuleParams =>
|
||||
params.type === 'machine_learning';
|
||||
|
||||
/**
|
||||
* Prevent javascript from returning Number.MAX_SAFE_INTEGER when Elasticsearch expects
|
||||
* Java's Long.MAX_VALUE. This happens when sorting fields by date which are
|
||||
* unmapped in the provided index
|
||||
*
|
||||
* Ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620
|
||||
*
|
||||
* return stringified Long.MAX_VALUE if we receive Number.MAX_SAFE_INTEGER
|
||||
* @param sortIds SortResults | undefined
|
||||
* @returns SortResults
|
||||
*/
|
||||
export const getSafeSortIds = (sortIds: SortResults | undefined) => {
|
||||
return sortIds?.map((sortId) => {
|
||||
// haven't determined when we would receive a null value for a sort id
|
||||
// but in case we do, default to sending the stringified Java max_int
|
||||
if (sortId == null || sortId === '' || sortId >= Number.MAX_SAFE_INTEGER) {
|
||||
return '9223372036854775807';
|
||||
}
|
||||
return sortId;
|
||||
});
|
||||
};
|
||||
|
|
|
@ -1322,5 +1322,118 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Signals generated from events with timestamp override field and ensures search_after continues to work when documents are missing timestamp override field', () => {
|
||||
beforeEach(async () => {
|
||||
await createSignalsIndex(supertest);
|
||||
await esArchiver.load('auditbeat/hosts');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await deleteSignalsIndex(supertest);
|
||||
await deleteAllAlerts(supertest);
|
||||
await esArchiver.unload('auditbeat/hosts');
|
||||
});
|
||||
|
||||
/**
|
||||
* This represents our worst case scenario where this field is not mapped on any index
|
||||
* We want to check that our logic continues to function within the constraints of search after
|
||||
* Elasticsearch returns java's long.MAX_VALUE for unmapped date fields
|
||||
* Javascript does not support numbers this large, but without passing in a number of this size
|
||||
* The search_after will continue to return the same results and not iterate to the next set
|
||||
* So to circumvent this limitation of javascript we return the stringified version of Java's
|
||||
* Long.MAX_VALUE so that search_after does not enter into an infinite loop.
|
||||
*
|
||||
* ref: https://github.com/elastic/elasticsearch/issues/28806#issuecomment-369303620
|
||||
*/
|
||||
it('should generate 200 signals when timestamp override does not exist', async () => {
|
||||
const rule: QueryCreateSchema = {
|
||||
...getRuleForSignalTesting(['auditbeat-*']),
|
||||
timestamp_override: 'event.fakeingested',
|
||||
max_signals: 200,
|
||||
};
|
||||
|
||||
const { id } = await createRule(supertest, rule);
|
||||
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
|
||||
await waitForSignalsToBePresent(supertest, 200, [id]);
|
||||
const signalsResponse = await getSignalsByIds(supertest, [id], 200);
|
||||
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
|
||||
|
||||
expect(signals.length).equal(200);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Here we test the functionality of timestamp overrides. If the rule specifies a timestamp override,
|
||||
* then the documents will be queried and sorted using the timestamp override field.
|
||||
* If no timestamp override field exists in the indices but one was provided to the rule,
|
||||
* the rule's query will additionally search for events using the `@timestamp` field
|
||||
*/
|
||||
describe('Signals generated from events with timestamp override field', async () => {
|
||||
beforeEach(async () => {
|
||||
await deleteSignalsIndex(supertest);
|
||||
await createSignalsIndex(supertest);
|
||||
await esArchiver.load('security_solution/timestamp_override_1');
|
||||
await esArchiver.load('security_solution/timestamp_override_2');
|
||||
await esArchiver.load('security_solution/timestamp_override_3');
|
||||
await esArchiver.load('security_solution/timestamp_override_4');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await deleteSignalsIndex(supertest);
|
||||
await deleteAllAlerts(supertest);
|
||||
await esArchiver.unload('security_solution/timestamp_override_1');
|
||||
await esArchiver.unload('security_solution/timestamp_override_2');
|
||||
await esArchiver.unload('security_solution/timestamp_override_3');
|
||||
await esArchiver.unload('security_solution/timestamp_override_4');
|
||||
});
|
||||
|
||||
it('should generate signals with event.ingested, @timestamp and (event.ingested + timestamp)', async () => {
|
||||
const rule: QueryCreateSchema = {
|
||||
...getRuleForSignalTesting(['myfa*']),
|
||||
timestamp_override: 'event.ingested',
|
||||
};
|
||||
|
||||
const { id } = await createRule(supertest, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
|
||||
await waitForSignalsToBePresent(supertest, 3, [id]);
|
||||
const signalsResponse = await getSignalsByIds(supertest, [id], 3);
|
||||
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
|
||||
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
|
||||
|
||||
expect(signalsOrderedByEventId.length).equal(3);
|
||||
});
|
||||
|
||||
it('should generate 2 signals with @timestamp', async () => {
|
||||
const rule: QueryCreateSchema = getRuleForSignalTesting(['myfa*']);
|
||||
|
||||
const { id } = await createRule(supertest, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
|
||||
await waitForSignalsToBePresent(supertest, 2, [id]);
|
||||
const signalsResponse = await getSignalsByIds(supertest, [id]);
|
||||
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
|
||||
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
|
||||
|
||||
expect(signalsOrderedByEventId.length).equal(2);
|
||||
});
|
||||
|
||||
it('should generate 2 signals when timestamp override does not exist', async () => {
|
||||
const rule: QueryCreateSchema = {
|
||||
...getRuleForSignalTesting(['myfa*']),
|
||||
timestamp_override: 'event.fakeingestfield',
|
||||
};
|
||||
const { id } = await createRule(supertest, rule);
|
||||
|
||||
await waitForRuleSuccessOrStatus(supertest, id, 'partial failure');
|
||||
await waitForSignalsToBePresent(supertest, 2, [id]);
|
||||
const signalsResponse = await getSignalsByIds(supertest, [id, id]);
|
||||
const signals = signalsResponse.hits.hits.map((hit) => hit._source);
|
||||
const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc');
|
||||
|
||||
expect(signalsOrderedByEventId.length).equal(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
Binary file not shown.
|
@ -1,19 +1,19 @@
|
|||
{
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "myfakeindex-1",
|
||||
"mappings" : {
|
||||
"properties" : {
|
||||
"message" : {
|
||||
"type" : "text",
|
||||
"fields" : {
|
||||
"keyword" : {
|
||||
"type" : "keyword",
|
||||
"ignore_above" : 256
|
||||
}
|
||||
}
|
||||
}
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "myfakeindex-1",
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "myfakeindex-1",
|
||||
"source": {
|
||||
"message": "hello world 1"
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "myfakeindex-1",
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "myfakeindex-2",
|
||||
"source": {
|
||||
"message": "hello world 2",
|
||||
"event": {
|
||||
"ingested": "2020-12-16T15:16:18.570Z"
|
||||
}
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "myfakeindex-2",
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"event": {
|
||||
"properties": {
|
||||
"ingested": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "myfakeindex-3",
|
||||
"source": {
|
||||
"message": "hello world 3",
|
||||
"@timestamp": "2020-12-16T15:16:18.570Z"
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "myfakeindex-3",
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"type": "doc",
|
||||
"value": {
|
||||
"index": "myfakeindex-4",
|
||||
"source": {
|
||||
"message": "hello world 4",
|
||||
"@timestamp": "2020-12-16T15:16:18.570Z",
|
||||
"event": {
|
||||
"ingested": "2020-12-16T15:16:18.570Z"
|
||||
}
|
||||
},
|
||||
"type": "_doc"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"type": "index",
|
||||
"value": {
|
||||
"index": "myfakeindex-4",
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"message": {
|
||||
"type": "text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"type": "keyword",
|
||||
"ignore_above": 256
|
||||
}
|
||||
}
|
||||
},
|
||||
"event": {
|
||||
"properties": {
|
||||
"ingested": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue