[RAC][Security Solution] Adds Threshold rule type and removes reliance on outputIndex (#111437)
* Initial commit * Properly handle signal history * Fix #95258 - cardinality sort bug * Init threshold rule * Create working threshold rule * Fix threshold signal generation * Fix tests * Update mappings * ALERT_TYPE_ID => RULE_TYPE_ID * Add tests * Fix types
This commit is contained in:
parent
93e8a7d91b
commit
cdb9fbac83
|
@ -197,7 +197,6 @@ export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const;
|
|||
export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const;
|
||||
export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
|
||||
export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
|
||||
export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
|
||||
export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
|
||||
|
||||
/**
|
||||
|
|
|
@ -5,7 +5,35 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
SPACE_IDS,
|
||||
ALERT_RULE_CONSUMER,
|
||||
ALERT_REASON,
|
||||
ALERT_STATUS,
|
||||
ALERT_STATUS_ACTIVE,
|
||||
ALERT_WORKFLOW_STATUS,
|
||||
ALERT_RULE_NAMESPACE,
|
||||
ALERT_INSTANCE_ID,
|
||||
ALERT_UUID,
|
||||
ALERT_RULE_TYPE_ID,
|
||||
ALERT_RULE_PRODUCER,
|
||||
ALERT_RULE_CATEGORY,
|
||||
ALERT_RULE_UUID,
|
||||
ALERT_RULE_NAME,
|
||||
} from '@kbn/rule-data-utils';
|
||||
import { TypeOfFieldMap } from '../../../../../../rule_registry/common/field_map';
|
||||
import { SERVER_APP_ID } from '../../../../../common/constants';
|
||||
import { ANCHOR_DATE } from '../../../../../common/detection_engine/schemas/response/rules_schema.mocks';
|
||||
import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
|
||||
import { sampleDocNoSortId } from '../../signals/__mocks__/es_results';
|
||||
import { flattenWithPrefix } from '../factories/utils/flatten_with_prefix';
|
||||
import { RulesFieldMap } from '../field_maps';
|
||||
import {
|
||||
ALERT_ANCESTORS,
|
||||
ALERT_ORIGINAL_TIME,
|
||||
ALERT_ORIGINAL_EVENT,
|
||||
} from '../field_maps/field_names';
|
||||
import { WrappedRACAlert } from '../types';
|
||||
|
||||
export const mockThresholdResults = {
|
||||
rawResponse: {
|
||||
|
@ -59,3 +87,79 @@ export const mockThresholdResults = {
|
|||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const sampleThresholdAlert: WrappedRACAlert = {
|
||||
_id: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d',
|
||||
_index: 'some-index',
|
||||
_source: {
|
||||
'@timestamp': '2020-04-20T21:26:30.000Z',
|
||||
[SPACE_IDS]: ['default'],
|
||||
[ALERT_INSTANCE_ID]: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d',
|
||||
[ALERT_UUID]: '310158f7-994d-4a38-8cdc-152139ac4d29',
|
||||
[ALERT_RULE_CONSUMER]: SERVER_APP_ID,
|
||||
[ALERT_ANCESTORS]: [
|
||||
{
|
||||
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
|
||||
type: 'event',
|
||||
index: 'myFakeSignalIndex',
|
||||
depth: 0,
|
||||
},
|
||||
],
|
||||
[ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z',
|
||||
[ALERT_ORIGINAL_EVENT]: {
|
||||
action: 'socket_opened',
|
||||
dataset: 'socket',
|
||||
kind: 'event',
|
||||
module: 'system',
|
||||
},
|
||||
[ALERT_REASON]: 'alert reasonable reason',
|
||||
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
|
||||
[ALERT_WORKFLOW_STATUS]: 'open',
|
||||
'source.ip': '127.0.0.1',
|
||||
'host.name': 'garden-gnomes',
|
||||
[ALERT_RULE_CATEGORY]: 'security',
|
||||
[ALERT_RULE_NAME]: 'a threshold rule',
|
||||
[ALERT_RULE_PRODUCER]: 'siem',
|
||||
[ALERT_RULE_TYPE_ID]: 'query-rule-id',
|
||||
[ALERT_RULE_UUID]: '151af49f-2e82-4b6f-831b-7f8cb341a5ff',
|
||||
...(flattenWithPrefix(ALERT_RULE_NAMESPACE, {
|
||||
author: [],
|
||||
uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
|
||||
created_at: new Date(ANCHOR_DATE).toISOString(),
|
||||
updated_at: new Date(ANCHOR_DATE).toISOString(),
|
||||
created_by: 'elastic',
|
||||
description: 'some description',
|
||||
enabled: true,
|
||||
false_positives: ['false positive 1', 'false positive 2'],
|
||||
from: 'now-6m',
|
||||
immutable: false,
|
||||
name: 'Query with a rule id',
|
||||
query: 'user.name: root or user.name: admin',
|
||||
references: ['test 1', 'test 2'],
|
||||
severity: 'high',
|
||||
severity_mapping: [],
|
||||
updated_by: 'elastic_kibana',
|
||||
tags: ['some fake tag 1', 'some fake tag 2'],
|
||||
to: 'now',
|
||||
type: 'query',
|
||||
threat: [],
|
||||
threshold: {
|
||||
field: ['source.ip', 'host.name'],
|
||||
value: 1,
|
||||
},
|
||||
version: 1,
|
||||
status: 'succeeded',
|
||||
status_date: '2020-02-22T16:47:50.047Z',
|
||||
last_success_at: '2020-02-22T16:47:50.047Z',
|
||||
last_success_message: 'succeeded',
|
||||
max_signals: 100,
|
||||
risk_score: 55,
|
||||
risk_score_mapping: [],
|
||||
language: 'kuery',
|
||||
rule_id: 'f88a544c-1d4e-4652-ae2a-c953b38da5d0',
|
||||
interval: '5m',
|
||||
exceptions_list: getListArrayMock(),
|
||||
}) as TypeOfFieldMap<RulesFieldMap>),
|
||||
'kibana.alert.depth': 1,
|
||||
},
|
||||
};
|
||||
|
|
|
@ -5,6 +5,8 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils';
|
||||
|
||||
import { performance } from 'perf_hooks';
|
||||
import { countBy, isEmpty } from 'lodash';
|
||||
|
||||
|
@ -62,11 +64,15 @@ export const bulkCreateFactory = <TContext extends AlertInstanceContext>(
|
|||
);
|
||||
|
||||
const createdItems = wrappedDocs
|
||||
.map((doc, index) => ({
|
||||
_id: response.body.items[index].index?._id ?? '',
|
||||
_index: response.body.items[index].index?._index ?? '',
|
||||
...doc._source,
|
||||
}))
|
||||
.map((doc, index) => {
|
||||
const responseIndex = response.body.items[index].index;
|
||||
return {
|
||||
_id: responseIndex?._id ?? '',
|
||||
_index: responseIndex?._index ?? '',
|
||||
[ALERT_INSTANCE_ID]: responseIndex?._id ?? '',
|
||||
...doc._source,
|
||||
};
|
||||
})
|
||||
.filter((_, index) => response.body.items[index].index?.status === 201);
|
||||
const createdItemsCount = createdItems.length;
|
||||
|
||||
|
|
|
@ -102,7 +102,6 @@ describe('buildAlert', () => {
|
|||
status_date: '2020-02-22T16:47:50.047Z',
|
||||
last_success_at: '2020-02-22T16:47:50.047Z',
|
||||
last_success_message: 'succeeded',
|
||||
output_index: '.siem-signals-default',
|
||||
max_signals: 100,
|
||||
risk_score: 55,
|
||||
risk_score_mapping: [],
|
||||
|
@ -179,7 +178,6 @@ describe('buildAlert', () => {
|
|||
status_date: '2020-02-22T16:47:50.047Z',
|
||||
last_success_at: '2020-02-22T16:47:50.047Z',
|
||||
last_success_message: 'succeeded',
|
||||
output_index: '.siem-signals-default',
|
||||
max_signals: 100,
|
||||
risk_score: 55,
|
||||
risk_score_mapping: [],
|
||||
|
|
|
@ -120,7 +120,7 @@ export const buildAlert = (
|
|||
[]
|
||||
);
|
||||
|
||||
const { id, ...mappedRule } = rule;
|
||||
const { id, output_index: outputIndex, ...mappedRule } = rule;
|
||||
mappedRule.uuid = id;
|
||||
|
||||
return ({
|
||||
|
|
|
@ -5,16 +5,26 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { isPlainObject } from 'lodash';
|
||||
import { SearchTypes } from '../../../../../../common/detection_engine/types';
|
||||
|
||||
export const flattenWithPrefix = (
|
||||
prefix: string,
|
||||
obj: Record<string, SearchTypes>
|
||||
maybeObj: unknown
|
||||
): Record<string, SearchTypes> => {
|
||||
return Object.keys(obj).reduce((acc: Record<string, SearchTypes>, key) => {
|
||||
if (maybeObj != null && isPlainObject(maybeObj)) {
|
||||
return Object.keys(maybeObj as Record<string, SearchTypes>).reduce(
|
||||
(acc: Record<string, SearchTypes>, key) => {
|
||||
return {
|
||||
...acc,
|
||||
...flattenWithPrefix(`${prefix}.${key}`, (maybeObj as Record<string, SearchTypes>)[key]),
|
||||
};
|
||||
},
|
||||
{}
|
||||
);
|
||||
} else {
|
||||
return {
|
||||
...acc,
|
||||
[`${prefix}.${key}`]: obj[key],
|
||||
[prefix]: maybeObj as SearchTypes,
|
||||
};
|
||||
}, {});
|
||||
}
|
||||
};
|
||||
|
|
|
@ -43,11 +43,6 @@ export const alertsFieldMap: FieldMap = {
|
|||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.group': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.group.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
|
@ -58,11 +53,6 @@ export const alertsFieldMap: FieldMap = {
|
|||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.original_event': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.original_event.action': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
|
@ -198,81 +188,6 @@ export const alertsFieldMap: FieldMap = {
|
|||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.threat': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.threat.framework': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.tactic': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.tactic.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.tactic.name': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.tactic.reference': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.name': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.reference': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.subtechnique': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.subtechnique.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.subtechnique.name': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threat.technique.subtechnique.reference': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.threshold_result': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.threshold_result.cardinality': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
|
@ -300,7 +215,7 @@ export const alertsFieldMap: FieldMap = {
|
|||
},
|
||||
'kibana.alert.threshold_result.terms': {
|
||||
type: 'object',
|
||||
array: false,
|
||||
array: true,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.threshold_result.terms.field': {
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
|
||||
import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils';
|
||||
|
||||
export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const;
|
||||
export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const;
|
||||
|
@ -14,3 +14,6 @@ export const ALERT_GROUP_ID = `${ALERT_NAMESPACE}.group.id` as const;
|
|||
export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const;
|
||||
export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const;
|
||||
export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const;
|
||||
|
||||
const ALERT_RULE_THRESHOLD = `${ALERT_RULE_NAMESPACE}.threshold` as const;
|
||||
export const ALERT_RULE_THRESHOLD_FIELD = `${ALERT_RULE_THRESHOLD}.field` as const;
|
||||
|
|
|
@ -11,6 +11,11 @@ export const rulesFieldMap = {
|
|||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.exceptions_list': {
|
||||
type: 'object',
|
||||
array: true,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.false_positives': {
|
||||
type: 'keyword',
|
||||
array: true,
|
||||
|
@ -46,6 +51,56 @@ export const rulesFieldMap = {
|
|||
array: true,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.framework': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.tactic.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.tactic.name': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.tactic.reference': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.technique.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.technique.name': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.technique.reference': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.technique.subtechnique.id': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.technique.subtechnique.name': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat.technique.subtechnique.reference': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
'kibana.alert.rule.threat_filters': {
|
||||
type: 'keyword',
|
||||
array: true,
|
||||
|
@ -91,11 +146,6 @@ export const rulesFieldMap = {
|
|||
array: true,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.threshold': {
|
||||
type: 'object',
|
||||
array: true,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.threshold.field': {
|
||||
type: 'keyword',
|
||||
array: true,
|
||||
|
@ -103,7 +153,7 @@ export const rulesFieldMap = {
|
|||
},
|
||||
'kibana.alert.rule.threshold.value': {
|
||||
type: 'float', // TODO: should be 'long' (eventually, after we stabilize)
|
||||
array: true,
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.threshold.cardinality': {
|
||||
|
@ -113,12 +163,12 @@ export const rulesFieldMap = {
|
|||
},
|
||||
'kibana.alert.rule.threshold.cardinality.field': {
|
||||
type: 'keyword',
|
||||
array: true,
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.threshold.cardinality.value': {
|
||||
type: 'long',
|
||||
array: true,
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'kibana.alert.rule.timeline_id': {
|
||||
|
|
|
@ -7,5 +7,6 @@
|
|||
|
||||
export { createEqlAlertType } from './eql/create_eql_alert_type';
|
||||
export { createIndicatorMatchAlertType } from './indicator_match/create_indicator_match_alert_type';
|
||||
export { createQueryAlertType } from './query/create_query_alert_type';
|
||||
export { createMlAlertType } from './ml/create_ml_alert_type';
|
||||
export { createQueryAlertType } from './query/create_query_alert_type';
|
||||
export { createThresholdAlertType } from './threshold/create_threshold_alert_type';
|
||||
|
|
|
@ -24,7 +24,7 @@ jest.mock('../utils/get_list_client', () => ({
|
|||
|
||||
jest.mock('../../rule_execution_log/rule_execution_log_client');
|
||||
|
||||
describe('Custom query alerts', () => {
|
||||
describe('Custom Query Alerts', () => {
|
||||
it('does not send an alert when no events found', async () => {
|
||||
const { services, dependencies, executor } = createRuleTypeMocks();
|
||||
const queryAlertType = createQueryAlertType({
|
||||
|
|
|
@ -0,0 +1,63 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \
|
||||
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
|
||||
-H 'kbn-xsrf: true' \
|
||||
-H 'Content-Type: application/json' \
|
||||
--verbose \
|
||||
-d '
|
||||
{
|
||||
"params":{
|
||||
"author": [],
|
||||
"description": "Basic threshold rule",
|
||||
"exceptionsList": [],
|
||||
"falsePositives": [],
|
||||
"from": "now-300s",
|
||||
"query": "*:*",
|
||||
"immutable": false,
|
||||
"index": ["*"],
|
||||
"language": "kuery",
|
||||
"maxSignals": 10,
|
||||
"outputIndex": "",
|
||||
"references": [],
|
||||
"riskScore": 21,
|
||||
"riskScoreMapping": [],
|
||||
"ruleId": "52dec1ba-b779-469c-9667-6b0e865fb89a",
|
||||
"severity": "low",
|
||||
"severityMapping": [],
|
||||
"threat": [],
|
||||
"threshold": {
|
||||
"field": ["source.ip"],
|
||||
"value": 2,
|
||||
"cardinality": [
|
||||
{
|
||||
"field": "source.ip",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"to": "now",
|
||||
"type": "threshold",
|
||||
"version": 1
|
||||
},
|
||||
"consumer":"alerts",
|
||||
"alertTypeId":"siem.thresholdRule",
|
||||
"schedule":{
|
||||
"interval":"1m"
|
||||
},
|
||||
"actions":[],
|
||||
"tags":[
|
||||
"custom",
|
||||
"persistence"
|
||||
],
|
||||
"notifyWhen":"onActionGroupChange",
|
||||
"name":"Basic threshold rule"
|
||||
}'
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { allowedExperimentalValues } from '../../../../../common/experimental_features';
|
||||
import { createThresholdAlertType } from './create_threshold_alert_type';
|
||||
import { createRuleTypeMocks } from '../__mocks__/rule_type';
|
||||
import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock';
|
||||
|
||||
jest.mock('../../rule_execution_log/rule_execution_log_client');
|
||||
|
||||
describe('Threshold Alerts', () => {
|
||||
it('does not send an alert when no events found', async () => {
|
||||
const params = getThresholdRuleParams();
|
||||
const { dependencies, executor } = createRuleTypeMocks('threshold', params);
|
||||
const thresholdAlertTpe = createThresholdAlertType({
|
||||
experimentalFeatures: allowedExperimentalValues,
|
||||
lists: dependencies.lists,
|
||||
logger: dependencies.logger,
|
||||
mergeStrategy: 'allFields',
|
||||
ignoreFields: [],
|
||||
ruleDataClient: dependencies.ruleDataClient,
|
||||
ruleDataService: dependencies.ruleDataService,
|
||||
version: '1.0.0',
|
||||
});
|
||||
dependencies.alerting.registerType(thresholdAlertTpe);
|
||||
|
||||
await executor({ params });
|
||||
expect(dependencies.ruleDataClient.getWriter).not.toBeCalled();
|
||||
});
|
||||
});
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
|
||||
|
||||
import { PersistenceServices } from '../../../../../../rule_registry/server';
|
||||
import { THRESHOLD_RULE_TYPE_ID } from '../../../../../common/constants';
|
||||
import { thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas';
|
||||
import { thresholdExecutor } from '../../signals/executors/threshold';
|
||||
import { ThresholdAlertState } from '../../signals/types';
|
||||
import { createSecurityRuleTypeFactory } from '../create_security_rule_type_factory';
|
||||
import { CreateRuleOptions } from '../types';
|
||||
|
||||
export const createThresholdAlertType = (createOptions: CreateRuleOptions) => {
|
||||
const {
|
||||
experimentalFeatures,
|
||||
lists,
|
||||
logger,
|
||||
mergeStrategy,
|
||||
ignoreFields,
|
||||
ruleDataClient,
|
||||
version,
|
||||
ruleDataService,
|
||||
} = createOptions;
|
||||
const createSecurityRuleType = createSecurityRuleTypeFactory({
|
||||
lists,
|
||||
logger,
|
||||
mergeStrategy,
|
||||
ignoreFields,
|
||||
ruleDataClient,
|
||||
ruleDataService,
|
||||
});
|
||||
return createSecurityRuleType<ThresholdRuleParams, {}, PersistenceServices, ThresholdAlertState>({
|
||||
id: THRESHOLD_RULE_TYPE_ID,
|
||||
name: 'Threshold Rule',
|
||||
validate: {
|
||||
params: {
|
||||
validate: (object: unknown): ThresholdRuleParams => {
|
||||
const [validated, errors] = validateNonExact(object, thresholdRuleParams);
|
||||
if (errors != null) {
|
||||
throw new Error(errors);
|
||||
}
|
||||
if (validated == null) {
|
||||
throw new Error('Validation of rule params failed');
|
||||
}
|
||||
return validated;
|
||||
},
|
||||
},
|
||||
},
|
||||
actionGroups: [
|
||||
{
|
||||
id: 'default',
|
||||
name: 'Default',
|
||||
},
|
||||
],
|
||||
defaultActionGroupId: 'default',
|
||||
actionVariables: {
|
||||
context: [{ name: 'server', description: 'the server' }],
|
||||
},
|
||||
minimumLicenseRequired: 'basic',
|
||||
isExportable: false,
|
||||
producer: 'security-solution',
|
||||
async executor(execOptions) {
|
||||
const {
|
||||
runOpts: { buildRuleMessage, bulkCreate, exceptionItems, rule, tuple, wrapHits },
|
||||
services,
|
||||
startedAt,
|
||||
state,
|
||||
} = execOptions;
|
||||
|
||||
// console.log(JSON.stringify(state));
|
||||
|
||||
const result = await thresholdExecutor({
|
||||
buildRuleMessage,
|
||||
bulkCreate,
|
||||
exceptionItems,
|
||||
experimentalFeatures,
|
||||
logger,
|
||||
rule,
|
||||
services,
|
||||
startedAt,
|
||||
state,
|
||||
tuple,
|
||||
version,
|
||||
wrapHits,
|
||||
});
|
||||
|
||||
return result;
|
||||
},
|
||||
});
|
||||
};
|
|
@ -7,15 +7,19 @@
|
|||
|
||||
import { getFilter } from './find_rules';
|
||||
import {
|
||||
EQL_RULE_TYPE_ID,
|
||||
INDICATOR_RULE_TYPE_ID,
|
||||
ML_RULE_TYPE_ID,
|
||||
QUERY_RULE_TYPE_ID,
|
||||
THRESHOLD_RULE_TYPE_ID,
|
||||
SIGNALS_ID,
|
||||
} from '../../../../common/constants';
|
||||
|
||||
const allAlertTypeIds = `(alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID}
|
||||
const allAlertTypeIds = `(alert.attributes.alertTypeId: ${EQL_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${QUERY_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID})`.replace(/[\n\r]/g, '');
|
||||
OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID}
|
||||
OR alert.attributes.alertTypeId: ${THRESHOLD_RULE_TYPE_ID})`.replace(/[\n\r]/g, '');
|
||||
|
||||
describe('find_rules', () => {
|
||||
const fullFilterTestCases: Array<[boolean, string]> = [
|
||||
|
|
|
@ -69,6 +69,8 @@ import {
|
|||
INDICATOR_RULE_TYPE_ID,
|
||||
ML_RULE_TYPE_ID,
|
||||
QUERY_RULE_TYPE_ID,
|
||||
EQL_RULE_TYPE_ID,
|
||||
THRESHOLD_RULE_TYPE_ID,
|
||||
} from '../../../../common/constants';
|
||||
|
||||
const nonEqlLanguages = t.keyof({ kuery: null, lucene: null });
|
||||
|
@ -206,12 +208,11 @@ export const notifyWhen = t.union([
|
|||
|
||||
export const allRuleTypes = t.union([
|
||||
t.literal(SIGNALS_ID),
|
||||
// t.literal(EQL_RULE_TYPE_ID),
|
||||
t.literal(EQL_RULE_TYPE_ID),
|
||||
t.literal(ML_RULE_TYPE_ID),
|
||||
t.literal(QUERY_RULE_TYPE_ID),
|
||||
// t.literal(SAVED_QUERY_RULE_TYPE_ID),
|
||||
t.literal(INDICATOR_RULE_TYPE_ID),
|
||||
// t.literal(THRESHOLD_RULE_TYPE_ID),
|
||||
t.literal(THRESHOLD_RULE_TYPE_ID),
|
||||
]);
|
||||
export type AllRuleTypes = t.TypeOf<typeof allRuleTypes>;
|
||||
|
||||
|
|
|
@ -73,6 +73,7 @@ describe('threshold_executor', () => {
|
|||
exceptionItems,
|
||||
experimentalFeatures: allowedExperimentalValues,
|
||||
services: alertServices,
|
||||
state: { initialized: true, signalHistory: {} },
|
||||
version,
|
||||
logger,
|
||||
buildRuleMessage,
|
||||
|
|
|
@ -5,9 +5,12 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { SearchHit } from '@elastic/elasticsearch/api/types';
|
||||
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
|
||||
|
||||
import { Logger } from 'src/core/server';
|
||||
import { SavedObject } from 'src/core/types';
|
||||
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
|
||||
|
||||
import {
|
||||
AlertInstanceContext,
|
||||
AlertInstanceState,
|
||||
|
@ -28,6 +31,7 @@ import {
|
|||
BulkCreate,
|
||||
RuleRangeTuple,
|
||||
SearchAfterAndBulkCreateReturnType,
|
||||
ThresholdAlertState,
|
||||
WrapHits,
|
||||
} from '../types';
|
||||
import {
|
||||
|
@ -37,6 +41,7 @@ import {
|
|||
} from '../utils';
|
||||
import { BuildRuleMessage } from '../rule_messages';
|
||||
import { ExperimentalFeatures } from '../../../../../common/experimental_features';
|
||||
import { buildThresholdSignalHistory } from '../threshold/build_signal_history';
|
||||
|
||||
export const thresholdExecutor = async ({
|
||||
rule,
|
||||
|
@ -48,6 +53,7 @@ export const thresholdExecutor = async ({
|
|||
logger,
|
||||
buildRuleMessage,
|
||||
startedAt,
|
||||
state,
|
||||
bulkCreate,
|
||||
wrapHits,
|
||||
}: {
|
||||
|
@ -60,17 +66,48 @@ export const thresholdExecutor = async ({
|
|||
logger: Logger;
|
||||
buildRuleMessage: BuildRuleMessage;
|
||||
startedAt: Date;
|
||||
state: ThresholdAlertState;
|
||||
bulkCreate: BulkCreate;
|
||||
wrapHits: WrapHits;
|
||||
}): Promise<SearchAfterAndBulkCreateReturnType> => {
|
||||
}): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
|
||||
let result = createSearchAfterReturnType();
|
||||
const ruleParams = rule.attributes.params;
|
||||
|
||||
// Get state or build initial state (on upgrade)
|
||||
const { signalHistory, searchErrors: previousSearchErrors } = state.initialized
|
||||
? { signalHistory: state.signalHistory, searchErrors: [] }
|
||||
: await getThresholdSignalHistory({
|
||||
indexPattern: ['*'], // TODO: get outputIndex?
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
logger,
|
||||
ruleId: ruleParams.ruleId,
|
||||
bucketByFields: ruleParams.threshold.field,
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
buildRuleMessage,
|
||||
});
|
||||
|
||||
if (!state.initialized) {
|
||||
// Clean up any signal history that has fallen outside the window
|
||||
const toDelete: string[] = [];
|
||||
for (const [hash, entry] of Object.entries(signalHistory)) {
|
||||
if (entry.lastSignalTimestamp < tuple.from.valueOf()) {
|
||||
toDelete.push(hash);
|
||||
}
|
||||
}
|
||||
for (const hash of toDelete) {
|
||||
delete signalHistory[hash];
|
||||
}
|
||||
}
|
||||
|
||||
if (hasLargeValueItem(exceptionItems)) {
|
||||
result.warningMessages.push(
|
||||
'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules'
|
||||
);
|
||||
result.warning = true;
|
||||
}
|
||||
|
||||
const inputIndex = await getInputIndex({
|
||||
experimentalFeatures,
|
||||
services,
|
||||
|
@ -78,23 +115,8 @@ export const thresholdExecutor = async ({
|
|||
index: ruleParams.index,
|
||||
});
|
||||
|
||||
const {
|
||||
thresholdSignalHistory,
|
||||
searchErrors: previousSearchErrors,
|
||||
} = await getThresholdSignalHistory({
|
||||
indexPattern: [ruleParams.outputIndex],
|
||||
from: tuple.from.toISOString(),
|
||||
to: tuple.to.toISOString(),
|
||||
services,
|
||||
logger,
|
||||
ruleId: ruleParams.ruleId,
|
||||
bucketByFields: ruleParams.threshold.field,
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
buildRuleMessage,
|
||||
});
|
||||
|
||||
const bucketFilters = await getThresholdBucketFilters({
|
||||
thresholdSignalHistory,
|
||||
signalHistory,
|
||||
timestampOverride: ruleParams.timestampOverride,
|
||||
});
|
||||
|
||||
|
@ -141,7 +163,7 @@ export const thresholdExecutor = async ({
|
|||
signalsIndex: ruleParams.outputIndex,
|
||||
startedAt,
|
||||
from: tuple.from.toDate(),
|
||||
thresholdSignalHistory,
|
||||
signalHistory,
|
||||
bulkCreate,
|
||||
wrapHits,
|
||||
});
|
||||
|
@ -161,5 +183,31 @@ export const thresholdExecutor = async ({
|
|||
searchAfterTimes: [thresholdSearchDuration],
|
||||
}),
|
||||
]);
|
||||
return result;
|
||||
|
||||
const createdAlerts = createdItems.map((alert) => {
|
||||
const { _id, _index, ...source } = alert as { _id: string; _index: string };
|
||||
return {
|
||||
_id,
|
||||
_index,
|
||||
_source: {
|
||||
...source,
|
||||
},
|
||||
} as SearchHit<unknown>;
|
||||
});
|
||||
|
||||
const newSignalHistory = buildThresholdSignalHistory({
|
||||
alerts: createdAlerts,
|
||||
});
|
||||
|
||||
return {
|
||||
...result,
|
||||
state: {
|
||||
...state,
|
||||
initialized: true,
|
||||
signalHistory: {
|
||||
...signalHistory,
|
||||
...newSignalHistory,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
|
@ -29,7 +29,7 @@ import {
|
|||
} from '../../../../common/detection_engine/utils';
|
||||
import { SetupPlugins } from '../../../plugin';
|
||||
import { getInputIndex } from './get_input_output_index';
|
||||
import { AlertAttributes, SignalRuleAlertTypeDefinition } from './types';
|
||||
import { AlertAttributes, SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types';
|
||||
import {
|
||||
getListsClient,
|
||||
getExceptions,
|
||||
|
@ -125,6 +125,7 @@ export const signalRulesAlertType = ({
|
|||
async executor({
|
||||
previousStartedAt,
|
||||
startedAt,
|
||||
state,
|
||||
alertId,
|
||||
services,
|
||||
params,
|
||||
|
@ -316,6 +317,7 @@ export const signalRulesAlertType = ({
|
|||
logger,
|
||||
buildRuleMessage,
|
||||
startedAt,
|
||||
state: state as ThresholdAlertState,
|
||||
bulkCreate,
|
||||
wrapHits,
|
||||
});
|
||||
|
|
|
@ -75,9 +75,11 @@ export const singleSearchAfter = async ({
|
|||
searchAfterQuery as estypes.SearchRequest
|
||||
);
|
||||
const end = performance.now();
|
||||
|
||||
const searchErrors = createErrorsFromShard({
|
||||
errors: nextSearchAfterResult._shards.failures ?? [],
|
||||
});
|
||||
|
||||
return {
|
||||
searchResult: nextSearchAfterResult,
|
||||
searchDuration: makeFloatString(end - start),
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ALERT_ORIGINAL_TIME } from '../../rule_types/field_maps/field_names';
|
||||
import { sampleThresholdAlert } from '../../rule_types/__mocks__/threshold';
|
||||
import { buildThresholdSignalHistory } from './build_signal_history';
|
||||
|
||||
describe('buildSignalHistory', () => {
|
||||
it('builds a signal history from an alert', () => {
|
||||
const signalHistory = buildThresholdSignalHistory({ alerts: [sampleThresholdAlert] });
|
||||
expect(signalHistory).toEqual({
|
||||
'7a75c5c2db61f57ec166c669cb8244b91f812f0b2f1d4f8afd528d4f8b4e199b': {
|
||||
lastSignalTimestamp: Date.parse(
|
||||
sampleThresholdAlert._source[ALERT_ORIGINAL_TIME] as string
|
||||
),
|
||||
terms: [
|
||||
{
|
||||
field: 'host.name',
|
||||
value: 'garden-gnomes',
|
||||
},
|
||||
{
|
||||
field: 'source.ip',
|
||||
value: '127.0.0.1',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,75 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { SearchHit } from '@elastic/elasticsearch/api/types';
|
||||
import {
|
||||
ALERT_ORIGINAL_TIME,
|
||||
ALERT_RULE_THRESHOLD_FIELD,
|
||||
} from '../../rule_types/field_maps/field_names';
|
||||
|
||||
import { SimpleHit, ThresholdSignalHistory } from '../types';
|
||||
import { getThresholdTermsHash, isWrappedRACAlert, isWrappedSignalHit } from '../utils';
|
||||
|
||||
interface GetThresholdSignalHistoryParams {
|
||||
alerts: Array<SearchHit<unknown>>;
|
||||
}
|
||||
|
||||
const getTerms = (alert: SimpleHit) => {
|
||||
if (isWrappedRACAlert(alert)) {
|
||||
return (alert._source[ALERT_RULE_THRESHOLD_FIELD] as string[]).map((field) => ({
|
||||
field,
|
||||
value: alert._source[field] as string,
|
||||
}));
|
||||
} else if (isWrappedSignalHit(alert)) {
|
||||
return alert._source.signal?.threshold_result?.terms ?? [];
|
||||
} else {
|
||||
// We shouldn't be here
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const getOriginalTime = (alert: SimpleHit) => {
|
||||
if (isWrappedRACAlert(alert)) {
|
||||
const originalTime = alert._source[ALERT_ORIGINAL_TIME];
|
||||
return originalTime != null ? new Date(originalTime as string).getTime() : undefined;
|
||||
} else if (isWrappedSignalHit(alert)) {
|
||||
const originalTime = alert._source.signal?.original_time;
|
||||
return originalTime != null ? new Date(originalTime).getTime() : undefined;
|
||||
} else {
|
||||
// We shouldn't be here
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
export const buildThresholdSignalHistory = ({
|
||||
alerts,
|
||||
}: GetThresholdSignalHistoryParams): ThresholdSignalHistory => {
|
||||
const signalHistory = alerts.reduce<ThresholdSignalHistory>((acc, alert) => {
|
||||
if (!alert._source) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
const terms = getTerms(alert as SimpleHit);
|
||||
const hash = getThresholdTermsHash(terms);
|
||||
const existing = acc[hash];
|
||||
const originalTime = getOriginalTime(alert as SimpleHit);
|
||||
|
||||
if (existing != null) {
|
||||
if (originalTime && originalTime > existing.lastSignalTimestamp) {
|
||||
acc[hash].lastSignalTimestamp = originalTime;
|
||||
}
|
||||
} else if (originalTime) {
|
||||
acc[hash] = {
|
||||
terms,
|
||||
lastSignalTimestamp: originalTime,
|
||||
};
|
||||
}
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
return signalHistory;
|
||||
};
|
|
@ -46,7 +46,7 @@ interface BulkCreateThresholdSignalsParams {
|
|||
signalsIndex: string;
|
||||
startedAt: Date;
|
||||
from: Date;
|
||||
thresholdSignalHistory: ThresholdSignalHistory;
|
||||
signalHistory: ThresholdSignalHistory;
|
||||
bulkCreate: BulkCreate;
|
||||
wrapHits: WrapHits;
|
||||
}
|
||||
|
@ -61,7 +61,7 @@ const getTransformedHits = (
|
|||
ruleId: string,
|
||||
filter: unknown,
|
||||
timestampOverride: TimestampOverrideOrUndefined,
|
||||
thresholdSignalHistory: ThresholdSignalHistory
|
||||
signalHistory: ThresholdSignalHistory
|
||||
) => {
|
||||
const aggParts = threshold.field.length
|
||||
? results.aggregations && getThresholdAggregationParts(results.aggregations)
|
||||
|
@ -148,7 +148,7 @@ const getTransformedHits = (
|
|||
}
|
||||
|
||||
const termsHash = getThresholdTermsHash(bucket.terms);
|
||||
const signalHit = thresholdSignalHistory[termsHash];
|
||||
const signalHit = signalHistory[termsHash];
|
||||
|
||||
const source = {
|
||||
'@timestamp': timestamp,
|
||||
|
@ -202,7 +202,7 @@ export const transformThresholdResultsToEcs = (
|
|||
threshold: ThresholdNormalized,
|
||||
ruleId: string,
|
||||
timestampOverride: TimestampOverrideOrUndefined,
|
||||
thresholdSignalHistory: ThresholdSignalHistory
|
||||
signalHistory: ThresholdSignalHistory
|
||||
): SignalSearchResponse => {
|
||||
const transformedHits = getTransformedHits(
|
||||
results,
|
||||
|
@ -214,7 +214,7 @@ export const transformThresholdResultsToEcs = (
|
|||
ruleId,
|
||||
filter,
|
||||
timestampOverride,
|
||||
thresholdSignalHistory
|
||||
signalHistory
|
||||
);
|
||||
const thresholdResults = {
|
||||
...results,
|
||||
|
@ -246,7 +246,7 @@ export const bulkCreateThresholdSignals = async (
|
|||
ruleParams.threshold,
|
||||
ruleParams.ruleId,
|
||||
ruleParams.timestampOverride,
|
||||
params.thresholdSignalHistory
|
||||
params.signalHistory
|
||||
);
|
||||
|
||||
return params.bulkCreate(
|
||||
|
|
|
@ -227,6 +227,7 @@ describe('findThresholdSignals', () => {
|
|||
'threshold_1:user.name': {
|
||||
terms: {
|
||||
field: 'user.name',
|
||||
order: { cardinality_count: 'desc' },
|
||||
min_doc_count: 100,
|
||||
size: 10000,
|
||||
},
|
||||
|
@ -302,6 +303,7 @@ describe('findThresholdSignals', () => {
|
|||
lang: 'painless',
|
||||
},
|
||||
min_doc_count: 200,
|
||||
order: { cardinality_count: 'desc' },
|
||||
},
|
||||
aggs: {
|
||||
cardinality_count: {
|
||||
|
|
|
@ -89,6 +89,13 @@ export const findThresholdSignals = async ({
|
|||
|
||||
const thresholdFields = threshold.field;
|
||||
|
||||
// order buckets by cardinality (https://github.com/elastic/kibana/issues/95258)
|
||||
const thresholdFieldCount = thresholdFields.length;
|
||||
const orderByCardinality = (i: number = 0) =>
|
||||
(thresholdFieldCount === 0 || i === thresholdFieldCount - 1) && threshold.cardinality?.length
|
||||
? { order: { cardinality_count: 'desc' } }
|
||||
: {};
|
||||
|
||||
// Generate a nested terms aggregation for each threshold grouping field provided, appending leaf
|
||||
// aggregations to 1) filter out buckets that don't meet the cardinality threshold, if provided, and
|
||||
// 2) return the latest hit for each bucket so that we can persist the timestamp of the event in the
|
||||
|
@ -104,6 +111,7 @@ export const findThresholdSignals = async ({
|
|||
set(acc, aggPath, {
|
||||
terms: {
|
||||
field,
|
||||
...orderByCardinality(i),
|
||||
min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set
|
||||
size: 10000, // max 10k buckets
|
||||
},
|
||||
|
@ -121,6 +129,7 @@ export const findThresholdSignals = async ({
|
|||
source: '""', // Group everything in the same bucket
|
||||
lang: 'painless',
|
||||
},
|
||||
...orderByCardinality(),
|
||||
min_doc_count: threshold.value,
|
||||
},
|
||||
aggs: leafAggs,
|
||||
|
|
|
@ -11,7 +11,7 @@ import { getThresholdBucketFilters } from './get_threshold_bucket_filters';
|
|||
describe('getThresholdBucketFilters', () => {
|
||||
it('should generate filters for threshold signal detection with dupe mitigation', async () => {
|
||||
const result = await getThresholdBucketFilters({
|
||||
thresholdSignalHistory: sampleThresholdSignalHistory(),
|
||||
signalHistory: sampleThresholdSignalHistory(),
|
||||
timestampOverride: undefined,
|
||||
});
|
||||
expect(result).toEqual([
|
||||
|
|
|
@ -9,14 +9,18 @@ import { Filter } from 'src/plugins/data/common';
|
|||
import { ESFilter } from '../../../../../../../../src/core/types/elasticsearch';
|
||||
import { ThresholdSignalHistory, ThresholdSignalHistoryRecord } from '../types';
|
||||
|
||||
/*
|
||||
* Returns a filter to exclude events that have already been included in a
|
||||
* previous threshold signal. Uses the threshold signal history to achieve this.
|
||||
*/
|
||||
export const getThresholdBucketFilters = async ({
|
||||
thresholdSignalHistory,
|
||||
signalHistory,
|
||||
timestampOverride,
|
||||
}: {
|
||||
thresholdSignalHistory: ThresholdSignalHistory;
|
||||
signalHistory: ThresholdSignalHistory;
|
||||
timestampOverride: string | undefined;
|
||||
}): Promise<Filter[]> => {
|
||||
const filters = Object.values(thresholdSignalHistory).reduce(
|
||||
const filters = Object.values(signalHistory).reduce(
|
||||
(acc: ESFilter[], bucket: ThresholdSignalHistoryRecord): ESFilter[] => {
|
||||
const filter = {
|
||||
bool: {
|
||||
|
@ -24,6 +28,7 @@ export const getThresholdBucketFilters = async ({
|
|||
{
|
||||
range: {
|
||||
[timestampOverride ?? '@timestamp']: {
|
||||
// Timestamp of last event signaled on for this set of terms.
|
||||
lte: new Date(bucket.lastSignalTimestamp).toISOString(),
|
||||
},
|
||||
},
|
||||
|
@ -32,6 +37,7 @@ export const getThresholdBucketFilters = async ({
|
|||
},
|
||||
} as ESFilter;
|
||||
|
||||
// Terms to filter events older than `lastSignalTimestamp`.
|
||||
bucket.terms.forEach((term) => {
|
||||
if (term.field != null) {
|
||||
(filter.bool!.filter as ESFilter[]).push({
|
||||
|
|
|
@ -5,7 +5,6 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema';
|
||||
import { TimestampOverrideOrUndefined } from '../../../../../common/detection_engine/schemas/common/schemas';
|
||||
import {
|
||||
AlertInstanceContext,
|
||||
|
@ -16,7 +15,7 @@ import { Logger } from '../../../../../../../../src/core/server';
|
|||
import { ThresholdSignalHistory } from '../types';
|
||||
import { BuildRuleMessage } from '../rule_messages';
|
||||
import { findPreviousThresholdSignals } from './find_previous_threshold_signals';
|
||||
import { getThresholdTermsHash } from '../utils';
|
||||
import { buildThresholdSignalHistory } from './build_signal_history';
|
||||
|
||||
interface GetThresholdSignalHistoryParams {
|
||||
from: string;
|
||||
|
@ -41,7 +40,7 @@ export const getThresholdSignalHistory = async ({
|
|||
timestampOverride,
|
||||
buildRuleMessage,
|
||||
}: GetThresholdSignalHistoryParams): Promise<{
|
||||
thresholdSignalHistory: ThresholdSignalHistory;
|
||||
signalHistory: ThresholdSignalHistory;
|
||||
searchErrors: string[];
|
||||
}> => {
|
||||
const { searchResult, searchErrors } = await findPreviousThresholdSignals({
|
||||
|
@ -56,51 +55,10 @@ export const getThresholdSignalHistory = async ({
|
|||
buildRuleMessage,
|
||||
});
|
||||
|
||||
const thresholdSignalHistory = searchResult.hits.hits.reduce<ThresholdSignalHistory>(
|
||||
(acc, hit) => {
|
||||
if (!hit._source) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
const terms =
|
||||
hit._source.signal?.threshold_result?.terms != null
|
||||
? hit._source.signal.threshold_result.terms
|
||||
: [
|
||||
// Pre-7.12 signals
|
||||
{
|
||||
field:
|
||||
(((hit._source.signal?.rule as RulesSchema).threshold as unknown) as {
|
||||
field: string;
|
||||
}).field ?? '',
|
||||
value: ((hit._source.signal?.threshold_result as unknown) as { value: string })
|
||||
.value,
|
||||
},
|
||||
];
|
||||
|
||||
const hash = getThresholdTermsHash(terms);
|
||||
const existing = acc[hash];
|
||||
const originalTime =
|
||||
hit._source.signal?.original_time != null
|
||||
? new Date(hit._source.signal?.original_time).getTime()
|
||||
: undefined;
|
||||
|
||||
if (existing != null) {
|
||||
if (originalTime && originalTime > existing.lastSignalTimestamp) {
|
||||
acc[hash].lastSignalTimestamp = originalTime;
|
||||
}
|
||||
} else if (originalTime) {
|
||||
acc[hash] = {
|
||||
terms,
|
||||
lastSignalTimestamp: originalTime,
|
||||
};
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
return {
|
||||
thresholdSignalHistory,
|
||||
signalHistory: buildThresholdSignalHistory({
|
||||
alerts: searchResult.hits.hits,
|
||||
}),
|
||||
searchErrors,
|
||||
};
|
||||
};
|
||||
|
|
|
@ -362,3 +362,8 @@ export interface ThresholdQueryBucket extends TermAggregationBucket {
|
|||
value_as_string: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ThresholdAlertState extends AlertTypeState {
|
||||
initialized: boolean;
|
||||
signalHistory: ThresholdSignalHistory;
|
||||
}
|
||||
|
|
|
@ -63,10 +63,12 @@ import { WrappedRACAlert } from '../rule_types/types';
|
|||
import { SearchTypes } from '../../../../common/detection_engine/types';
|
||||
import { IRuleExecutionLogClient } from '../rule_execution_log/types';
|
||||
import {
|
||||
EQL_RULE_TYPE_ID,
|
||||
INDICATOR_RULE_TYPE_ID,
|
||||
ML_RULE_TYPE_ID,
|
||||
QUERY_RULE_TYPE_ID,
|
||||
SIGNALS_ID,
|
||||
THRESHOLD_RULE_TYPE_ID,
|
||||
} from '../../../../common/constants';
|
||||
|
||||
interface SortExceptionsReturn {
|
||||
|
@ -1013,10 +1015,10 @@ export const getField = <T extends SearchTypes>(event: SimpleHit, field: string)
|
|||
* Maps legacy rule types to RAC rule type IDs.
|
||||
*/
|
||||
export const ruleTypeMappings = {
|
||||
eql: SIGNALS_ID,
|
||||
eql: EQL_RULE_TYPE_ID,
|
||||
machine_learning: ML_RULE_TYPE_ID,
|
||||
query: QUERY_RULE_TYPE_ID,
|
||||
saved_query: SIGNALS_ID,
|
||||
threat_match: INDICATOR_RULE_TYPE_ID,
|
||||
threshold: SIGNALS_ID,
|
||||
threshold: THRESHOLD_RULE_TYPE_ID,
|
||||
};
|
||||
|
|
|
@ -53,6 +53,7 @@ import {
|
|||
createIndicatorMatchAlertType,
|
||||
createMlAlertType,
|
||||
createQueryAlertType,
|
||||
createThresholdAlertType,
|
||||
} from './lib/detection_engine/rule_types';
|
||||
import { initRoutes } from './routes';
|
||||
import { isAlertExecutor } from './lib/detection_engine/signals/types';
|
||||
|
@ -264,9 +265,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
|
|||
};
|
||||
|
||||
this.setupPlugins.alerting.registerType(createEqlAlertType(createRuleOptions));
|
||||
this.setupPlugins.alerting.registerType(createQueryAlertType(createRuleOptions));
|
||||
this.setupPlugins.alerting.registerType(createIndicatorMatchAlertType(createRuleOptions));
|
||||
this.setupPlugins.alerting.registerType(createMlAlertType(createRuleOptions));
|
||||
this.setupPlugins.alerting.registerType(createQueryAlertType(createRuleOptions));
|
||||
this.setupPlugins.alerting.registerType(createThresholdAlertType(createRuleOptions));
|
||||
}
|
||||
|
||||
// TODO We need to get the endpoint routes inside of initRoutes
|
||||
|
|
Loading…
Reference in a new issue