maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Flatten

Browse source
This commit is contained in:
Madison Caldwell 2021-08-03 15:20:11 -04:00
parent 4cf8f7d9b1
commit a25dde02a2
4 changed files with 134 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

180
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { sampleDocNoSortId } from '../../../signals/__mocks__/es_results';
import { sampleDocNoSortIdWithTimestamp } from '../../../signals/__mocks__/es_results';
import { flatten } from './flatten';
import {
buildAlert,
@ -14,41 +14,33 @@ import {
additionalAlertFields,
removeClashes,
} from './build_alert';
import { Ancestor, BaseSignalHit, SignalSourceHit } from '../../../signals/types';
import { Ancestor, SignalSourceHit } from '../../../signals/types';
import {
getRulesSchemaMock,
ANCHOR_DATE,
} from '../../../../../../common/detection_engine/schemas/response/rules_schema.mocks';
import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock';
import { SIGNALS_TEMPLATE_VERSION } from '../../../routes/index/get_signals_template';
import { RACAlert } from '../../types';
type SignalDoc = SignalSourceHit & {
_source: Required<SignalSourceHit>['_source'] & { '@timestamp': string };
};
describe('buildAlert', () => {
beforeEach(() => {
jest.clearAllMocks();
});
test('it builds a signal as expected without original_event if event does not exist', () => {
const sampleDoc: SignalSourceHit & {
_source: Required<SignalSourceHit>['_source'];
} = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc: SignalSourceHit & {
_source: Required<SignalSourceHit>['_source']
} = {
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': new Date().toISOString(),
},
};
test('it builds an alert as expected without original_event if event does not exist', () => {
const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
delete doc._source.event;
const rule = getRulesSchemaMock();
const signal = {
const alert = {
...buildAlert([doc], rule),
...additionalAlertFields(doc),
};
const expected: RACAlert = {
'@timestamp': new Date().toISOString(),
const timestamp = alert['@timestamp'];
const expected = {
'@timestamp': timestamp,
'kibana.alert.ancestors': [
{
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
@ -59,6 +51,7 @@ describe('buildAlert', () => {
],
'kibana.alert.original_time': '2020-04-20T21:27:45.000Z',
'kibana.alert.status': 'open',
'kibana.alert.workflow_status': 'open',
...flatten('kibana.alert.rule', {
author: [],
id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
@ -70,7 +63,6 @@ describe('buildAlert', () => {
false_positives: ['false positive 1', 'false positive 2'],
from: 'now-6m',
immutable: false,
index: '',
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
references: ['test 1', 'test 2'],
@ -97,11 +89,11 @@ describe('buildAlert', () => {
}),
'kibana.alert.depth': 1,
};
expect(signal).toEqual(expected);
expect(alert).toEqual(expected);
});
test('it builds a signal as expected with original_event if is present', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
test('it builds an alert as expected with original_event if is present', () => {
const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
doc._source.event = {
action: 'socket_opened',
dataset: 'socket',
@ -109,21 +101,14 @@ describe('buildAlert', () => {
module: 'system',
};
const rule = getRulesSchemaMock();
const signal = {
const alert = {
...buildAlert([doc], rule),
...additionalAlertFields(doc),
};
const expected: RACAlert = {
_meta: {
version: SIGNALS_TEMPLATE_VERSION,
},
parent: {
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
type: 'event',
index: 'myFakeSignalIndex',
depth: 0,
},
parents: [
const timestamp = alert['@timestamp'];
const expected = {
'@timestamp': timestamp,
'kibana.alert.ancestors': [
{
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
type: 'event',
@ -131,23 +116,16 @@ describe('buildAlert', () => {
depth: 0,
},
],
ancestors: [
{
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
type: 'event',
index: 'myFakeSignalIndex',
depth: 0,
},
],
original_time: '2020-04-20T21:27:45.000Z',
original_event: {
'kibana.alert.original_time': '2020-04-20T21:27:45.000Z',
'kibana.alert.original_event': {
action: 'socket_opened',
dataset: 'socket',
kind: 'event',
module: 'system',
},
status: 'open',
rule: {
'kibana.alert.status': 'open',
'kibana.alert.workflow_status': 'open',
...flatten('kibana.alert.rule', {
author: [],
id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
created_at: new Date(ANCHOR_DATE).toISOString(),
@ -181,32 +159,32 @@ describe('buildAlert', () => {
rule_id: 'query-rule-id',
interval: '5m',
exceptions_list: getListArrayMock(),
},
depth: 1,
}),
'kibana.alert.depth': 1,
};
expect(signal).toEqual(expected);
expect(alert).toEqual(expected);
});
test('it builds a ancestor correctly if the parent does not exist', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
test('it builds an ancestor correctly if the parent does not exist', () => {
const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
doc._source.event = {
action: 'socket_opened',
dataset: 'socket',
kind: 'event',
module: 'system',
};
const signal = buildParent(doc);
const parent = buildParent(doc);
const expected: Ancestor = {
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
type: 'event',
index: 'myFakeSignalIndex',
depth: 0,
};
expect(signal).toEqual(expected);
expect(parent).toEqual(expected);
});
test('it builds a ancestor correctly if the parent does exist', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
test('it builds an ancestor correctly if the parent does exist', () => {
const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
doc._source.event = {
action: 'socket_opened',
dataset: 'socket',
@ -235,7 +213,7 @@ describe('buildAlert', () => {
id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
},
};
const signal = buildParent(doc);
const parent = buildParent(doc);
const expected: Ancestor = {
rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b',
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
@ -243,18 +221,25 @@ describe('buildAlert', () => {
index: 'myFakeSignalIndex',
depth: 1,
};
expect(signal).toEqual(expected);
expect(parent).toEqual(expected);
});
test('it builds a signal ancestor correctly if the parent does not exist', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
test('it builds an alert ancestor correctly if the parent does not exist', () => {
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc: SignalDoc = {
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': new Date().toISOString(),
},
};
doc._source.event = {
action: 'socket_opened',
dataset: 'socket',
kind: 'event',
module: 'system',
};
const signal = buildAncestors(doc);
const ancestor = buildAncestors(doc);
const expected: Ancestor[] = [
{
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
@ -263,11 +248,18 @@ describe('buildAlert', () => {
depth: 0,
},
];
expect(signal).toEqual(expected);
expect(ancestor).toEqual(expected);
});
test('it builds a signal ancestor correctly if the parent does exist', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
test('it builds an alert ancestor correctly if the parent does exist', () => {
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc: SignalDoc = {
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': new Date().toISOString(),
},
};
doc._source.event = {
action: 'socket_opened',
dataset: 'socket',
@ -296,7 +288,7 @@ describe('buildAlert', () => {
},
depth: 1,
};
const signal = buildAncestors(doc);
const ancestors = buildAncestors(doc);
const expected: Ancestor[] = [
{
id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87',
@ -312,63 +304,93 @@ describe('buildAlert', () => {
depth: 1,
},
];
expect(signal).toEqual(expected);
expect(ancestors).toEqual(expected);
});
describe('removeClashes', () => {
test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc: SignalDoc = {
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': new Date().toISOString(),
},
};
const output = removeClashes(doc);
expect(output).toBe(doc); // reference check
});
test('it will call renameClashes with a regular doc and not change anything', () => {
const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc: SignalDoc = {
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': new Date().toISOString(),
},
};
const output = removeClashes(doc);
expect(output).toEqual(doc); // deep equal check
});
test('it will remove a "signal" numeric clash', () => {
const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc = ({
...sampleDoc,
_source: {
...sampleDoc._source,
signal: 127,
},
} as unknown) as BaseSignalHit;
} as unknown) as SignalDoc;
const output = removeClashes(doc);
expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'));
const timestamp = output._source['@timestamp'];
expect(output).toEqual({
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': timestamp,
},
});
});
test('it will remove a "signal" object clash', () => {
const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc = ({
...sampleDoc,
_source: {
...sampleDoc._source,
signal: { child_1: { child_2: 'Test nesting' } },
},
} as unknown) as BaseSignalHit;
} as unknown) as SignalDoc;
const output = removeClashes(doc);
expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'));
const timestamp = output._source['@timestamp'];
expect(output).toEqual({
...sampleDoc,
_source: {
...sampleDoc._source,
'@timestamp': timestamp,
},
});
});
test('it will not remove a "signal" if that is signal is one of our signals', () => {
const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71');
const doc = ({
...sampleDoc,
_source: {
...sampleDoc._source,
signal: { rule: { id: '123' } },
},
} as unknown) as BaseSignalHit;
} as unknown) as SignalDoc;
const output = removeClashes(doc);
const timestamp = output._source['@timestamp'];
const expected = {
...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'),
...sampleDoc,
_source: {
...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')._source,
...sampleDoc._source,
signal: { rule: { id: '123' } },
'@timestamp': timestamp,
},
};
expect(output).toEqual(expected);

49
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts
View file

@ -6,10 +6,9 @@
*/
import { ALERT_STATUS, ALERT_WORKFLOW_STATUS } from '@kbn/rule-data-utils';
import { SearchTypes } from '../../../../../../common/detection_engine/types';
import { RulesSchema } from '../../../../../../common/detection_engine/schemas/response/rules_schema';
import { isEventTypeSignal } from '../../../signals/build_event_type_signal';
import { Ancestor, BaseSignalHit, SimpleHit, ThresholdResult } from '../../../signals/types';
import { Ancestor, BaseSignalHit, SimpleHit } from '../../../signals/types';
import {
getF,
getValidDateFromDoc,
@ -17,8 +16,8 @@ import {
isWrappedSignalHit,
} from '../../../signals/utils';
import { invariant } from '../../../../../../common/utils/invariant';
import { DEFAULT_MAX_SIGNALS } from '../../../../../../common/constants';
import { RACAlert } from '../../types';
import { flatten } from './flatten';
/**
* Takes an event document and extracts the information needed for the corresponding entry in the child
@ -26,7 +25,7 @@ import { RACAlert } from '../../types';
* @param doc The parent event
*/
export const buildParent = (doc: SimpleHit): Ancestor => {
const isSignal = isWrappedSignalHit(doc) || isWrappedRACAlert(doc);
const isSignal: boolean = isWrappedSignalHit(doc) || isWrappedRACAlert(doc);
const parent: Ancestor = {
id: doc._id,
type: isSignal ? 'signal' : 'event',
@ -93,35 +92,14 @@ export const buildAlert = (docs: SimpleHit[], rule: RulesSchema): RACAlert => {
[]
);
return {
return ({
'@timestamp': new Date().toISOString(),
'kibana.alert.ancestors': ancestors as object[],
'kibana.alert.ancestors': ancestors,
[ALERT_STATUS]: 'open',
[ALERT_WORKFLOW_STATUS]: 'open',
'kibana.alert.depth': depth,
'kibana.alert.rule.false_positives': rule.false_positives ?? [],
'kibana.alert.rule.id': rule.id,
'kibana.alert.rule.immutable': rule.immutable ? 'true' : 'false',
'kibana.alert.rule.index': rule.index ?? [],
'kibana.alert.rule.language': rule.language ?? 'kuery',
'kibana.alert.rule.max_signals': rule.max_signals ?? DEFAULT_MAX_SIGNALS,
'kibana.alert.rule.query': rule.query ?? '*:*',
'kibana.alert.rule.saved_id': rule.saved_id ?? '',
'kibana.alert.rule.threat_index': rule.threat_index,
'kibana.alert.rule.threat_indicator_path': rule.threat_indicator_path,
'kibana.alert.rule.threat_language': rule.threat_language,
'kibana.alert.rule.threat_mapping.field': '', // TODO
'kibana.alert.rule.threat_mapping.value': '', // TODO
'kibana.alert.rule.threat_mapping.type': '', // TODO
'kibana.alert.rule.threshold.field': rule.threshold?.field,
'kibana.alert.rule.threshold.value': rule.threshold?.value,
'kibana.alert.rule.threshold.cardinality.field': '', // TODO
'kibana.alert.rule.threshold.cardinality.value': 0, // TODO
};
};
const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => {
return typeof thresholdResult === 'object';
...flatten('kibana.alert.rule', rule),
} as unknown) as RACAlert;
};
/**
@ -130,17 +108,16 @@ const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is Thr
* @param doc The parent signal/event of the new signal to be built.
*/
export const additionalAlertFields = (doc: BaseSignalHit) => {
const thresholdResult = doc._source?.threshold_result;
if (thresholdResult != null && !isThresholdResult(thresholdResult)) {
throw new Error(`threshold_result failed to validate: ${thresholdResult}`);
}
const originalTime = getValidDateFromDoc({
doc,
timestampOverride: undefined,
});
return {
const additionalFields: Record<string, unknown> = {
'kibana.alert.original_time': originalTime != null ? originalTime.toISOString() : undefined,
'kibana.alert.original_event': doc._source?.event ?? undefined,
'kibana.alert.threshold_result': thresholdResult,
};
const event = doc._source?.event;
if (event != null) {
additionalFields['kibana.alert.original_event'] = event;
}
return additionalFields;
};

16
x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file

@ -168,6 +168,22 @@ export const sampleDocNoSortId = (
sort: [],
});
export const sampleDocNoSortIdWithTimestamp = (
someUuid: string = sampleIdGuid,
ip?: string
): SignalSourceHit & {
_source: Required<SignalSourceHit>['_source'] & { '@timestamp': string };
} => {
const doc = sampleDocNoSortId(someUuid, ip);
return {
...doc,
_source: {
...doc._source,
'@timestamp': new Date().toISOString(),
},
};
};
export const sampleDocSeverity = (severity?: unknown, fieldName?: string): SignalSourceHit => {
const doc = {
_index: 'myFakeSignalIndex',

9
x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts
View file

@ -5,11 +5,11 @@
* 2.0.
*/
import { createHash } from 'crypto';
import { chunk, get, isEmpty, partition } from 'lodash';
import moment from 'moment';
import uuidv5 from 'uuid/v5';
import dateMath from '@elastic/datemath';
import type { estypes } from '@elastic/elasticsearch';
import { chunk, isEmpty, partition } from 'lodash';
import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport';
import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
@ -946,12 +946,11 @@ export const isWrappedRACAlert = (event: SimpleHit): event is WrappedRACAlert =>
};
export const getF = <T extends SearchTypes>(event: SimpleHit, field: string): T | undefined => {
const sourceField = `_source.${field}`;
if (isWrappedRACAlert(event)) {
return event._source[sourceField.replace('signal', 'kibana.alert')] as T; // TODO: handle special cases
return event._source, field.replace('signal', 'kibana.alert') as T; // TODO: handle special cases
} else if (isWrappedSignalHit(event)) {
return event._source[sourceField] as T;
return get(event._source, field) as T;
} else if (isWrappedEventHit(event)) {
return event._source[sourceField] as T;
return get(event._source, field) as T;
}
};