adds wrapSequences method
This commit is contained in:
parent
2998d9c971
commit
a526dcb62a
11 changed files with 107 additions and 52 deletions
|
@ -78,6 +78,8 @@ describe('eql_executor', () => {
|
|||
logger,
|
||||
searchAfterSize,
|
||||
bulkCreate: jest.fn(),
|
||||
wrapHits: jest.fn(),
|
||||
wrapSequences: jest.fn(),
|
||||
});
|
||||
expect(response.warningMessages.length).toEqual(1);
|
||||
});
|
||||
|
|
|
@ -21,18 +21,19 @@ import { isOutdated } from '../../migrations/helpers';
|
|||
import { getIndexVersion } from '../../routes/index/get_index_version';
|
||||
import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template';
|
||||
import { EqlRuleParams } from '../../schemas/rule_schemas';
|
||||
import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body';
|
||||
import { getInputIndex } from '../get_input_output_index';
|
||||
import { filterDuplicateSignals } from '../filter_duplicate_signals';
|
||||
|
||||
import {
|
||||
AlertAttributes,
|
||||
BulkCreate,
|
||||
WrapHits,
|
||||
WrapSequences,
|
||||
EqlSignalSearchResponse,
|
||||
RuleRangeTuple,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
WrappedSignalHit,
|
||||
SimpleHit,
|
||||
} from '../types';
|
||||
import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils';
|
||||
import { createSearchAfterReturnType, makeFloatString } from '../utils';
|
||||
|
||||
export const eqlExecutor = async ({
|
||||
rule,
|
||||
|
@ -43,6 +44,8 @@ export const eqlExecutor = async ({
|
|||
logger,
|
||||
searchAfterSize,
|
||||
bulkCreate,
|
||||
wrapHits,
|
||||
wrapSequences,
|
||||
}: {
|
||||
rule: SavedObject<AlertAttributes<EqlRuleParams>>;
|
||||
tuple: RuleRangeTuple;
|
||||
|
@ -52,6 +55,8 @@ export const eqlExecutor = async ({
|
|||
logger: Logger;
|
||||
searchAfterSize: number;
|
||||
bulkCreate: BulkCreate;
|
||||
wrapHits: WrapHits;
|
||||
wrapSequences: WrapSequences;
|
||||
}): Promise<SearchAfterAndBulkCreateReturnType> => {
|
||||
const result = createSearchAfterReturnType();
|
||||
const ruleParams = rule.attributes.params;
|
||||
|
@ -104,27 +109,18 @@ export const eqlExecutor = async ({
|
|||
const eqlSignalSearchEnd = performance.now();
|
||||
const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart);
|
||||
result.searchAfterTimes = [eqlSearchDuration];
|
||||
let newSignals: WrappedSignalHit[] | undefined;
|
||||
let newSignals: SimpleHit[] | undefined;
|
||||
if (response.hits.sequences !== undefined) {
|
||||
newSignals = response.hits.sequences.reduce(
|
||||
(acc: WrappedSignalHit[], sequence) =>
|
||||
acc.concat(buildSignalGroupFromSequence(sequence, rule, ruleParams.outputIndex)),
|
||||
[]
|
||||
);
|
||||
newSignals = wrapSequences(response.hits.sequences);
|
||||
} else if (response.hits.events !== undefined) {
|
||||
newSignals = filterDuplicateSignals(
|
||||
rule.id,
|
||||
response.hits.events.map((event) =>
|
||||
wrapSignal(buildSignalFromEvent(event, rule, true), ruleParams.outputIndex)
|
||||
)
|
||||
);
|
||||
newSignals = wrapHits(response.hits.events);
|
||||
} else {
|
||||
throw new Error(
|
||||
'eql query response should have either `sequences` or `events` but had neither'
|
||||
);
|
||||
}
|
||||
|
||||
if (newSignals.length > 0) {
|
||||
if (newSignals?.length) {
|
||||
const insertResult = await bulkCreate(newSignals);
|
||||
result.bulkCreateTimes.push(insertResult.bulkCreateDuration);
|
||||
result.createdSignalsCount += insertResult.createdItemsCount;
|
||||
|
|
|
@ -36,11 +36,13 @@ const mockSignals = [
|
|||
];
|
||||
|
||||
describe('filterDuplicateSignals', () => {
|
||||
it('filters duplicate signals', () => {
|
||||
expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1);
|
||||
});
|
||||
describe('detection engine implementation', () => {
|
||||
it('filters duplicate signals', () => {
|
||||
expect(filterDuplicateSignals(mockRuleId1, mockSignals, false).length).toEqual(1);
|
||||
});
|
||||
|
||||
it('does not filter non-duplicate signals', () => {
|
||||
expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2);
|
||||
it('does not filter non-duplicate signals', () => {
|
||||
expect(filterDuplicateSignals(mockRuleId3, mockSignals, false).length).toEqual(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -5,10 +5,26 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { WrappedSignalHit } from './types';
|
||||
import { SimpleHit, WrappedSignalHit } from './types';
|
||||
|
||||
export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => {
|
||||
return signals.filter(
|
||||
(doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
|
||||
);
|
||||
const isWrappedSignalHit = (
|
||||
signals: SimpleHit[],
|
||||
isRuleRegistryEnabled: boolean
|
||||
): signals is WrappedSignalHit[] => {
|
||||
return !isRuleRegistryEnabled;
|
||||
};
|
||||
|
||||
export const filterDuplicateSignals = (
|
||||
ruleId: string,
|
||||
signals: SimpleHit[],
|
||||
isRuleRegistryEnabled: boolean
|
||||
) => {
|
||||
if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) {
|
||||
return signals.filter(
|
||||
(doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
|
||||
);
|
||||
} else {
|
||||
// TODO: filter duplicate signals for RAC
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
|
|
@ -66,7 +66,10 @@ describe('searchAfterAndBulkCreate', () => {
|
|||
buildRuleMessage,
|
||||
false
|
||||
);
|
||||
wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX });
|
||||
wrapHits = wrapHitsFactory({
|
||||
ruleSO,
|
||||
signalsIndex: DEFAULT_SIGNALS_INDEX,
|
||||
});
|
||||
});
|
||||
|
||||
test('should return success with number of searches less than max signals', async () => {
|
||||
|
|
|
@ -67,6 +67,7 @@ import {
|
|||
} from '../schemas/rule_schemas';
|
||||
import { bulkCreateFactory } from './bulk_create_factory';
|
||||
import { wrapHitsFactory } from './wrap_hits_factory';
|
||||
import { wrapSequencesFactory } from './wrap_sequences_factory';
|
||||
|
||||
export const signalRulesAlertType = ({
|
||||
logger,
|
||||
|
@ -233,6 +234,11 @@ export const signalRulesAlertType = ({
|
|||
signalsIndex: params.outputIndex,
|
||||
});
|
||||
|
||||
const wrapSequences = wrapSequencesFactory({
|
||||
ruleSO: savedObject,
|
||||
signalsIndex: params.outputIndex,
|
||||
});
|
||||
|
||||
if (isMlRule(type)) {
|
||||
const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams);
|
||||
for (const tuple of tuples) {
|
||||
|
@ -313,6 +319,8 @@ export const signalRulesAlertType = ({
|
|||
searchAfterSize,
|
||||
bulkCreate,
|
||||
logger,
|
||||
wrapHits,
|
||||
wrapSequences,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
|
|
|
@ -25,6 +25,7 @@ import {
|
|||
BaseHit,
|
||||
RuleAlertAction,
|
||||
SearchTypes,
|
||||
EqlSequence,
|
||||
} from '../../../../common/detection_engine/types';
|
||||
import { ListClient } from '../../../../../lists/server';
|
||||
import { Logger, SavedObject } from '../../../../../../../src/core/server';
|
||||
|
@ -257,9 +258,11 @@ export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise<Signa
|
|||
|
||||
export type BulkCreate = <T>(docs: Array<BaseHit<T>>) => Promise<GenericBulkCreateResponse<T>>;
|
||||
|
||||
export type WrapHits = (
|
||||
hits: Array<estypes.SearchHit<unknown>>
|
||||
) => Array<BaseHit<{ '@timestamp': string }>>;
|
||||
export type SimpleHit = BaseHit<{ '@timestamp': string }>;
|
||||
|
||||
export type WrapHits = (hits: Array<estypes.SearchHit<SignalSource>>) => SimpleHit[];
|
||||
|
||||
export type WrapSequences = (sequences: Array<EqlSequence<SignalSource>>) => SimpleHit[];
|
||||
|
||||
export interface SearchAfterAndBulkCreateParams {
|
||||
tuple: {
|
||||
|
|
|
@ -5,12 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
SearchAfterAndBulkCreateParams,
|
||||
SignalSourceHit,
|
||||
WrapHits,
|
||||
WrappedSignalHit,
|
||||
} from './types';
|
||||
import { SearchAfterAndBulkCreateParams, WrapHits, WrappedSignalHit } from './types';
|
||||
import { generateId } from './utils';
|
||||
import { buildBulkBody } from './build_bulk_body';
|
||||
import { filterDuplicateSignals } from './filter_duplicate_signals';
|
||||
|
@ -25,11 +20,15 @@ export const wrapHitsFactory = ({
|
|||
const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [
|
||||
{
|
||||
_index: signalsIndex,
|
||||
// TODO: bring back doc._version
|
||||
_id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''),
|
||||
_source: buildBulkBody(ruleSO, doc as SignalSourceHit),
|
||||
_id: generateId(
|
||||
doc._index,
|
||||
doc._id,
|
||||
String(doc._version),
|
||||
ruleSO.attributes.params.ruleId ?? ''
|
||||
),
|
||||
_source: buildBulkBody(ruleSO, doc),
|
||||
},
|
||||
]);
|
||||
|
||||
return filterDuplicateSignals(ruleSO.id, wrappedDocs);
|
||||
return filterDuplicateSignals(ruleSO.id, wrappedDocs, false);
|
||||
};
|
||||
|
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { SearchAfterAndBulkCreateParams, WrappedSignalHit, WrapSequences } from './types';
|
||||
import { buildSignalGroupFromSequence } from './build_bulk_body';
|
||||
|
||||
export const wrapSequencesFactory = ({
|
||||
ruleSO,
|
||||
signalsIndex,
|
||||
}: {
|
||||
ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
|
||||
signalsIndex: string;
|
||||
}): WrapSequences => (sequences) =>
|
||||
sequences.reduce(
|
||||
(acc: WrappedSignalHit[], sequence) => [
|
||||
...acc,
|
||||
...buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex),
|
||||
],
|
||||
[]
|
||||
);
|
|
@ -211,8 +211,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
|
|||
});
|
||||
|
||||
// TODO: Once we are past experimental phase this check can be removed along with legacy registration of rules
|
||||
const isRuleRegistryEnabled = experimentalFeatures.ruleRegistryEnabled;
|
||||
|
||||
let ruleDataClient: RuleDataClient | null = null;
|
||||
if (experimentalFeatures.ruleRegistryEnabled) {
|
||||
if (isRuleRegistryEnabled) {
|
||||
const { ruleDataService } = plugins.ruleRegistry;
|
||||
const start = () => core.getStartServices().then(([coreStart]) => coreStart);
|
||||
|
||||
|
@ -296,7 +298,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
|
|||
const ruleTypes = [
|
||||
SIGNALS_ID,
|
||||
NOTIFICATIONS_ID,
|
||||
...(experimentalFeatures.ruleRegistryEnabled ? referenceRuleTypes : []),
|
||||
...(isRuleRegistryEnabled ? referenceRuleTypes : []),
|
||||
];
|
||||
|
||||
plugins.features.registerKibanaFeature({
|
||||
|
|
|
@ -227,7 +227,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
parents: [
|
||||
{
|
||||
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
|
||||
id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
|
||||
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -242,7 +242,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
},
|
||||
{
|
||||
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
|
||||
id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
|
||||
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -252,7 +252,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
depth: 2,
|
||||
parent: {
|
||||
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
|
||||
id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
|
||||
id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -1265,7 +1265,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
parents: [
|
||||
{
|
||||
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
|
||||
id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
|
||||
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -1280,7 +1280,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
},
|
||||
{
|
||||
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
|
||||
id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
|
||||
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -1290,7 +1290,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
depth: 2,
|
||||
parent: {
|
||||
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
|
||||
id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
|
||||
id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -1423,7 +1423,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
parents: [
|
||||
{
|
||||
rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
|
||||
id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
|
||||
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -1438,7 +1438,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
},
|
||||
{
|
||||
rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
|
||||
id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
|
||||
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
@ -1448,7 +1448,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
depth: 2,
|
||||
parent: {
|
||||
rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
|
||||
id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
|
||||
id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
|
||||
type: 'signal',
|
||||
index: '.siem-signals-default-000001',
|
||||
depth: 1,
|
||||
|
|
Loading…
Reference in a new issue