maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Security Solution] adds wrapSequences method (RAC) (#102106) (#103490)

Browse source 
adds wrapSequences method
This commit is contained in:
Ece Özalp 2021-06-28 12:36:42 -04:00 committed by GitHub
parent 2998d9c971
commit a526dcb62a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 107 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts
View file

@ -78,6 +78,8 @@ describe('eql_executor', () => {
logger,
searchAfterSize,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
});
expect(response.warningMessages.length).toEqual(1);
});

30
x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts
View file

@ -21,18 +21,19 @@ import { isOutdated } from '../../migrations/helpers';
import { getIndexVersion } from '../../routes/index/get_index_version';
import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template';
import { EqlRuleParams } from '../../schemas/rule_schemas';
import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body';
import { getInputIndex } from '../get_input_output_index';
import { filterDuplicateSignals } from '../filter_duplicate_signals';
import {
AlertAttributes,
BulkCreate,
WrapHits,
WrapSequences,
EqlSignalSearchResponse,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
WrappedSignalHit,
SimpleHit,
} from '../types';
import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils';
import { createSearchAfterReturnType, makeFloatString } from '../utils';
export const eqlExecutor = async ({
rule,
@ -43,6 +44,8 @@ export const eqlExecutor = async ({
logger,
searchAfterSize,
bulkCreate,
wrapHits,
wrapSequences,
}: {
rule: SavedObject<AlertAttributes<EqlRuleParams>>;
tuple: RuleRangeTuple;
@ -52,6 +55,8 @@ export const eqlExecutor = async ({
logger: Logger;
searchAfterSize: number;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
wrapSequences: WrapSequences;
}): Promise<SearchAfterAndBulkCreateReturnType> => {
const result = createSearchAfterReturnType();
const ruleParams = rule.attributes.params;
@ -104,27 +109,18 @@ export const eqlExecutor = async ({
const eqlSignalSearchEnd = performance.now();
const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart);
result.searchAfterTimes = [eqlSearchDuration];
let newSignals: WrappedSignalHit[] | undefined;
let newSignals: SimpleHit[] | undefined;
if (response.hits.sequences !== undefined) {
newSignals = response.hits.sequences.reduce(
(acc: WrappedSignalHit[], sequence) =>
acc.concat(buildSignalGroupFromSequence(sequence, rule, ruleParams.outputIndex)),
[]
);
newSignals = wrapSequences(response.hits.sequences);
} else if (response.hits.events !== undefined) {
newSignals = filterDuplicateSignals(
rule.id,
response.hits.events.map((event) =>
wrapSignal(buildSignalFromEvent(event, rule, true), ruleParams.outputIndex)
)
);
newSignals = wrapHits(response.hits.events);
} else {
throw new Error(
'eql query response should have either `sequences` or `events` but had neither'
);
}
if (newSignals.length > 0) {
if (newSignals?.length) {
const insertResult = await bulkCreate(newSignals);
result.bulkCreateTimes.push(insertResult.bulkCreateDuration);
result.createdSignalsCount += insertResult.createdItemsCount;

12
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts
View file

@ -36,11 +36,13 @@ const mockSignals = [
];
describe('filterDuplicateSignals', () => {
it('filters duplicate signals', () => {
expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1);
});
describe('detection engine implementation', () => {
it('filters duplicate signals', () => {
expect(filterDuplicateSignals(mockRuleId1, mockSignals, false).length).toEqual(1);
});
it('does not filter non-duplicate signals', () => {
expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2);
it('does not filter non-duplicate signals', () => {
expect(filterDuplicateSignals(mockRuleId3, mockSignals, false).length).toEqual(2);
});
});
});

26
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts
View file

@ -5,10 +5,26 @@
* 2.0.
*/
import { WrappedSignalHit } from './types';
import { SimpleHit, WrappedSignalHit } from './types';
export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => {
return signals.filter(
(doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
);
const isWrappedSignalHit = (
signals: SimpleHit[],
isRuleRegistryEnabled: boolean
): signals is WrappedSignalHit[] => {
return !isRuleRegistryEnabled;
};
export const filterDuplicateSignals = (
ruleId: string,
signals: SimpleHit[],
isRuleRegistryEnabled: boolean
) => {
if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) {
return signals.filter(
(doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
);
} else {
// TODO: filter duplicate signals for RAC
return [];
}
};

5
x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts
View file

@ -66,7 +66,10 @@ describe('searchAfterAndBulkCreate', () => {
buildRuleMessage,
false
);
wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX });
wrapHits = wrapHitsFactory({
ruleSO,
signalsIndex: DEFAULT_SIGNALS_INDEX,
});
});
test('should return success with number of searches less than max signals', async () => {

8
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file

@ -67,6 +67,7 @@ import {
} from '../schemas/rule_schemas';
import { bulkCreateFactory } from './bulk_create_factory';
import { wrapHitsFactory } from './wrap_hits_factory';
import { wrapSequencesFactory } from './wrap_sequences_factory';
export const signalRulesAlertType = ({
logger,
@ -233,6 +234,11 @@ export const signalRulesAlertType = ({
signalsIndex: params.outputIndex,
});
const wrapSequences = wrapSequencesFactory({
ruleSO: savedObject,
signalsIndex: params.outputIndex,
});
if (isMlRule(type)) {
const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams);
for (const tuple of tuples) {
@ -313,6 +319,8 @@ export const signalRulesAlertType = ({
searchAfterSize,
bulkCreate,
logger,
wrapHits,
wrapSequences,
});
}
} else {

9
x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
View file

@ -25,6 +25,7 @@ import {
BaseHit,
RuleAlertAction,
SearchTypes,
EqlSequence,
} from '../../../../common/detection_engine/types';
import { ListClient } from '../../../../../lists/server';
import { Logger, SavedObject } from '../../../../../../../src/core/server';
@ -257,9 +258,11 @@ export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise<Signa
export type BulkCreate = <T>(docs: Array<BaseHit<T>>) => Promise<GenericBulkCreateResponse<T>>;
export type WrapHits = (
hits: Array<estypes.SearchHit<unknown>>
) => Array<BaseHit<{ '@timestamp': string }>>;
export type SimpleHit = BaseHit<{ '@timestamp': string }>;
export type WrapHits = (hits: Array<estypes.SearchHit<SignalSource>>) => SimpleHit[];
export type WrapSequences = (sequences: Array<EqlSequence<SignalSource>>) => SimpleHit[];
export interface SearchAfterAndBulkCreateParams {
tuple: {

19
x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts
View file

@ -5,12 +5,7 @@
* 2.0.
*/
import {
SearchAfterAndBulkCreateParams,
SignalSourceHit,
WrapHits,
WrappedSignalHit,
} from './types';
import { SearchAfterAndBulkCreateParams, WrapHits, WrappedSignalHit } from './types';
import { generateId } from './utils';
import { buildBulkBody } from './build_bulk_body';
import { filterDuplicateSignals } from './filter_duplicate_signals';
@ -25,11 +20,15 @@ export const wrapHitsFactory = ({
const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [
{
_index: signalsIndex,
// TODO: bring back doc._version
_id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''),
_source: buildBulkBody(ruleSO, doc as SignalSourceHit),
_id: generateId(
doc._index,
doc._id,
String(doc._version),
ruleSO.attributes.params.ruleId ?? ''
),
_source: buildBulkBody(ruleSO, doc),
},
]);
return filterDuplicateSignals(ruleSO.id, wrappedDocs);
return filterDuplicateSignals(ruleSO.id, wrappedDocs, false);
};

24
x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SearchAfterAndBulkCreateParams, WrappedSignalHit, WrapSequences } from './types';
import { buildSignalGroupFromSequence } from './build_bulk_body';
export const wrapSequencesFactory = ({
ruleSO,
signalsIndex,
}: {
ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
signalsIndex: string;
}): WrapSequences => (sequences) =>
sequences.reduce(
(acc: WrappedSignalHit[], sequence) => [
...acc,
...buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex),
],
[]
);

6
x-pack/plugins/security_solution/server/plugin.ts
View file

@ -211,8 +211,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
});
// TODO: Once we are past experimental phase this check can be removed along with legacy registration of rules
const isRuleRegistryEnabled = experimentalFeatures.ruleRegistryEnabled;
let ruleDataClient: RuleDataClient | null = null;
if (experimentalFeatures.ruleRegistryEnabled) {
if (isRuleRegistryEnabled) {
const { ruleDataService } = plugins.ruleRegistry;
const start = () => core.getStartServices().then(([coreStart]) => coreStart);
@ -296,7 +298,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
const ruleTypes = [
SIGNALS_ID,
NOTIFICATIONS_ID,
...(experimentalFeatures.ruleRegistryEnabled ? referenceRuleTypes : []),
...(isRuleRegistryEnabled ? referenceRuleTypes : []),
];
plugins.features.registerKibanaFeature({

18
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts
View file

@ -227,7 +227,7 @@ export default ({ getService }: FtrProviderContext) => {
parents: [
{
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -242,7 +242,7 @@ export default ({ getService }: FtrProviderContext) => {
},
{
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -252,7 +252,7 @@ export default ({ getService }: FtrProviderContext) => {
depth: 2,
parent: {
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1265,7 +1265,7 @@ export default ({ getService }: FtrProviderContext) => {
parents: [
{
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1280,7 +1280,7 @@ export default ({ getService }: FtrProviderContext) => {
},
{
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1290,7 +1290,7 @@ export default ({ getService }: FtrProviderContext) => {
depth: 2,
parent: {
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1423,7 +1423,7 @@ export default ({ getService }: FtrProviderContext) => {
parents: [
{
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1438,7 +1438,7 @@ export default ({ getService }: FtrProviderContext) => {
},
{
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,
@ -1448,7 +1448,7 @@ export default ({ getService }: FtrProviderContext) => {
depth: 2,
parent: {
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
type: 'signal',
index: '.siem-signals-default-000001',
depth: 1,