[RAC][Security Solution] Adds Threshold rule type and removes reliance on outputIndex (#111437)

* Initial commit

* Properly handle signal history

* Fix #95258 - cardinality sort bug

* Init threshold rule

* Create working threshold rule

* Fix threshold signal generation

* Fix tests

* Update mappings

* ALERT_TYPE_ID => RULE_TYPE_ID

* Add tests

* Fix types
This commit is contained in:
Madison Caldwell 2021-09-14 14:41:29 -04:00 committed by GitHub
parent 93e8a7d91b
commit cdb9fbac83
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 625 additions and 197 deletions

View file

@ -197,7 +197,6 @@ export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const;
export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const;
export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
/**

View file

@ -5,7 +5,35 @@
* 2.0.
*/
import {
SPACE_IDS,
ALERT_RULE_CONSUMER,
ALERT_REASON,
ALERT_STATUS,
ALERT_STATUS_ACTIVE,
ALERT_WORKFLOW_STATUS,
ALERT_RULE_NAMESPACE,
ALERT_INSTANCE_ID,
ALERT_UUID,
ALERT_RULE_TYPE_ID,
ALERT_RULE_PRODUCER,
ALERT_RULE_CATEGORY,
ALERT_RULE_UUID,
ALERT_RULE_NAME,
} from '@kbn/rule-data-utils';
import { TypeOfFieldMap } from '../../../../../../rule_registry/common/field_map';
import { SERVER_APP_ID } from '../../../../../common/constants';
import { ANCHOR_DATE } from '../../../../../common/detection_engine/schemas/response/rules_schema.mocks';
import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock';
import { sampleDocNoSortId } from '../../signals/__mocks__/es_results';
import { flattenWithPrefix } from '../factories/utils/flatten_with_prefix';
import { RulesFieldMap } from '../field_maps';
import {
ALERT_ANCESTORS,
ALERT_ORIGINAL_TIME,
ALERT_ORIGINAL_EVENT,
} from '../field_maps/field_names';
import { WrappedRACAlert } from '../types';
export const mockThresholdResults = {
rawResponse: {
@ -59,3 +87,79 @@ export const mockThresholdResults = {
},
},
};
export const sampleThresholdAlert: WrappedRACAlert = {
_id: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d',
_index: 'some-index',
_source: {
'@timestamp': '2020-04-20T21:26:30.000Z',
[SPACE_IDS]: ['default'],
[ALERT_INSTANCE_ID]: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d',
[ALERT_UUID]: '310158f7-994d-4a38-8cdc-152139ac4d29',
[ALERT_RULE_CONSUMER]: SERVER_APP_ID,
[ALERT_ANCESTORS]: [
{
id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71',
type: 'event',
index: 'myFakeSignalIndex',
depth: 0,
},
],
[ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z',
[ALERT_ORIGINAL_EVENT]: {
action: 'socket_opened',
dataset: 'socket',
kind: 'event',
module: 'system',
},
[ALERT_REASON]: 'alert reasonable reason',
[ALERT_STATUS]: ALERT_STATUS_ACTIVE,
[ALERT_WORKFLOW_STATUS]: 'open',
'source.ip': '127.0.0.1',
'host.name': 'garden-gnomes',
[ALERT_RULE_CATEGORY]: 'security',
[ALERT_RULE_NAME]: 'a threshold rule',
[ALERT_RULE_PRODUCER]: 'siem',
[ALERT_RULE_TYPE_ID]: 'query-rule-id',
[ALERT_RULE_UUID]: '151af49f-2e82-4b6f-831b-7f8cb341a5ff',
...(flattenWithPrefix(ALERT_RULE_NAMESPACE, {
author: [],
uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
created_at: new Date(ANCHOR_DATE).toISOString(),
updated_at: new Date(ANCHOR_DATE).toISOString(),
created_by: 'elastic',
description: 'some description',
enabled: true,
false_positives: ['false positive 1', 'false positive 2'],
from: 'now-6m',
immutable: false,
name: 'Query with a rule id',
query: 'user.name: root or user.name: admin',
references: ['test 1', 'test 2'],
severity: 'high',
severity_mapping: [],
updated_by: 'elastic_kibana',
tags: ['some fake tag 1', 'some fake tag 2'],
to: 'now',
type: 'query',
threat: [],
threshold: {
field: ['source.ip', 'host.name'],
value: 1,
},
version: 1,
status: 'succeeded',
status_date: '2020-02-22T16:47:50.047Z',
last_success_at: '2020-02-22T16:47:50.047Z',
last_success_message: 'succeeded',
max_signals: 100,
risk_score: 55,
risk_score_mapping: [],
language: 'kuery',
rule_id: 'f88a544c-1d4e-4652-ae2a-c953b38da5d0',
interval: '5m',
exceptions_list: getListArrayMock(),
}) as TypeOfFieldMap<RulesFieldMap>),
'kibana.alert.depth': 1,
},
};

View file

@ -5,6 +5,8 @@
* 2.0.
*/
import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils';
import { performance } from 'perf_hooks';
import { countBy, isEmpty } from 'lodash';
@ -62,11 +64,15 @@ export const bulkCreateFactory = <TContext extends AlertInstanceContext>(
);
const createdItems = wrappedDocs
.map((doc, index) => ({
_id: response.body.items[index].index?._id ?? '',
_index: response.body.items[index].index?._index ?? '',
...doc._source,
}))
.map((doc, index) => {
const responseIndex = response.body.items[index].index;
return {
_id: responseIndex?._id ?? '',
_index: responseIndex?._index ?? '',
[ALERT_INSTANCE_ID]: responseIndex?._id ?? '',
...doc._source,
};
})
.filter((_, index) => response.body.items[index].index?.status === 201);
const createdItemsCount = createdItems.length;

View file

@ -102,7 +102,6 @@ describe('buildAlert', () => {
status_date: '2020-02-22T16:47:50.047Z',
last_success_at: '2020-02-22T16:47:50.047Z',
last_success_message: 'succeeded',
output_index: '.siem-signals-default',
max_signals: 100,
risk_score: 55,
risk_score_mapping: [],
@ -179,7 +178,6 @@ describe('buildAlert', () => {
status_date: '2020-02-22T16:47:50.047Z',
last_success_at: '2020-02-22T16:47:50.047Z',
last_success_message: 'succeeded',
output_index: '.siem-signals-default',
max_signals: 100,
risk_score: 55,
risk_score_mapping: [],

View file

@ -120,7 +120,7 @@ export const buildAlert = (
[]
);
const { id, ...mappedRule } = rule;
const { id, output_index: outputIndex, ...mappedRule } = rule;
mappedRule.uuid = id;
return ({

View file

@ -5,16 +5,26 @@
* 2.0.
*/
import { isPlainObject } from 'lodash';
import { SearchTypes } from '../../../../../../common/detection_engine/types';
export const flattenWithPrefix = (
prefix: string,
obj: Record<string, SearchTypes>
maybeObj: unknown
): Record<string, SearchTypes> => {
return Object.keys(obj).reduce((acc: Record<string, SearchTypes>, key) => {
if (maybeObj != null && isPlainObject(maybeObj)) {
return Object.keys(maybeObj as Record<string, SearchTypes>).reduce(
(acc: Record<string, SearchTypes>, key) => {
return {
...acc,
...flattenWithPrefix(`${prefix}.${key}`, (maybeObj as Record<string, SearchTypes>)[key]),
};
},
{}
);
} else {
return {
...acc,
[`${prefix}.${key}`]: obj[key],
[prefix]: maybeObj as SearchTypes,
};
}, {});
}
};

View file

@ -43,11 +43,6 @@ export const alertsFieldMap: FieldMap = {
array: false,
required: true,
},
'kibana.alert.group': {
type: 'object',
array: false,
required: false,
},
'kibana.alert.group.id': {
type: 'keyword',
array: false,
@ -58,11 +53,6 @@ export const alertsFieldMap: FieldMap = {
array: false,
required: false,
},
'kibana.alert.original_event': {
type: 'object',
array: false,
required: false,
},
'kibana.alert.original_event.action': {
type: 'keyword',
array: false,
@ -198,81 +188,6 @@ export const alertsFieldMap: FieldMap = {
array: false,
required: false,
},
'kibana.alert.threat': {
type: 'object',
array: false,
required: false,
},
'kibana.alert.threat.framework': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.tactic': {
type: 'object',
array: false,
required: true,
},
'kibana.alert.threat.tactic.id': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.tactic.name': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.tactic.reference': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.technique': {
type: 'object',
array: false,
required: true,
},
'kibana.alert.threat.technique.id': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.technique.name': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.technique.reference': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.technique.subtechnique': {
type: 'object',
array: false,
required: true,
},
'kibana.alert.threat.technique.subtechnique.id': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.technique.subtechnique.name': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threat.technique.subtechnique.reference': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.threshold_result': {
type: 'object',
array: false,
required: false,
},
'kibana.alert.threshold_result.cardinality': {
type: 'object',
array: false,
@ -300,7 +215,7 @@ export const alertsFieldMap: FieldMap = {
},
'kibana.alert.threshold_result.terms': {
type: 'object',
array: false,
array: true,
required: false,
},
'kibana.alert.threshold_result.terms.field': {

View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils';
export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const;
export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const;
@ -14,3 +14,6 @@ export const ALERT_GROUP_ID = `${ALERT_NAMESPACE}.group.id` as const;
export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const;
export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const;
export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const;
const ALERT_RULE_THRESHOLD = `${ALERT_RULE_NAMESPACE}.threshold` as const;
export const ALERT_RULE_THRESHOLD_FIELD = `${ALERT_RULE_THRESHOLD}.field` as const;

View file

@ -11,6 +11,11 @@ export const rulesFieldMap = {
array: false,
required: false,
},
'kibana.alert.rule.exceptions_list': {
type: 'object',
array: true,
required: false,
},
'kibana.alert.rule.false_positives': {
type: 'keyword',
array: true,
@ -46,6 +51,56 @@ export const rulesFieldMap = {
array: true,
required: true,
},
'kibana.alert.rule.threat.framework': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.tactic.id': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.tactic.name': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.tactic.reference': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.technique.id': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.technique.name': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.technique.reference': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.technique.subtechnique.id': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.technique.subtechnique.name': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat.technique.subtechnique.reference': {
type: 'keyword',
array: false,
required: true,
},
'kibana.alert.rule.threat_filters': {
type: 'keyword',
array: true,
@ -91,11 +146,6 @@ export const rulesFieldMap = {
array: true,
required: false,
},
'kibana.alert.rule.threshold': {
type: 'object',
array: true,
required: false,
},
'kibana.alert.rule.threshold.field': {
type: 'keyword',
array: true,
@ -103,7 +153,7 @@ export const rulesFieldMap = {
},
'kibana.alert.rule.threshold.value': {
type: 'float', // TODO: should be 'long' (eventually, after we stabilize)
array: true,
array: false,
required: false,
},
'kibana.alert.rule.threshold.cardinality': {
@ -113,12 +163,12 @@ export const rulesFieldMap = {
},
'kibana.alert.rule.threshold.cardinality.field': {
type: 'keyword',
array: true,
array: false,
required: false,
},
'kibana.alert.rule.threshold.cardinality.value': {
type: 'long',
array: true,
array: false,
required: false,
},
'kibana.alert.rule.timeline_id': {

View file

@ -7,5 +7,6 @@
export { createEqlAlertType } from './eql/create_eql_alert_type';
export { createIndicatorMatchAlertType } from './indicator_match/create_indicator_match_alert_type';
export { createQueryAlertType } from './query/create_query_alert_type';
export { createMlAlertType } from './ml/create_ml_alert_type';
export { createQueryAlertType } from './query/create_query_alert_type';
export { createThresholdAlertType } from './threshold/create_threshold_alert_type';

View file

@ -24,7 +24,7 @@ jest.mock('../utils/get_list_client', () => ({
jest.mock('../../rule_execution_log/rule_execution_log_client');
describe('Custom query alerts', () => {
describe('Custom Query Alerts', () => {
it('does not send an alert when no events found', async () => {
const { services, dependencies, executor } = createRuleTypeMocks();
const queryAlertType = createQueryAlertType({

View file

@ -0,0 +1,63 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License
# 2.0; you may not use this file except in compliance with the Elastic License
# 2.0.
#
curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-H 'kbn-xsrf: true' \
-H 'Content-Type: application/json' \
--verbose \
-d '
{
"params":{
"author": [],
"description": "Basic threshold rule",
"exceptionsList": [],
"falsePositives": [],
"from": "now-300s",
"query": "*:*",
"immutable": false,
"index": ["*"],
"language": "kuery",
"maxSignals": 10,
"outputIndex": "",
"references": [],
"riskScore": 21,
"riskScoreMapping": [],
"ruleId": "52dec1ba-b779-469c-9667-6b0e865fb89a",
"severity": "low",
"severityMapping": [],
"threat": [],
"threshold": {
"field": ["source.ip"],
"value": 2,
"cardinality": [
{
"field": "source.ip",
"value": 1
}
]
},
"to": "now",
"type": "threshold",
"version": 1
},
"consumer":"alerts",
"alertTypeId":"siem.thresholdRule",
"schedule":{
"interval":"1m"
},
"actions":[],
"tags":[
"custom",
"persistence"
],
"notifyWhen":"onActionGroupChange",
"name":"Basic threshold rule"
}'

View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { allowedExperimentalValues } from '../../../../../common/experimental_features';
import { createThresholdAlertType } from './create_threshold_alert_type';
import { createRuleTypeMocks } from '../__mocks__/rule_type';
import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock';
jest.mock('../../rule_execution_log/rule_execution_log_client');
describe('Threshold Alerts', () => {
it('does not send an alert when no events found', async () => {
const params = getThresholdRuleParams();
const { dependencies, executor } = createRuleTypeMocks('threshold', params);
const thresholdAlertTpe = createThresholdAlertType({
experimentalFeatures: allowedExperimentalValues,
lists: dependencies.lists,
logger: dependencies.logger,
mergeStrategy: 'allFields',
ignoreFields: [],
ruleDataClient: dependencies.ruleDataClient,
ruleDataService: dependencies.ruleDataService,
version: '1.0.0',
});
dependencies.alerting.registerType(thresholdAlertTpe);
await executor({ params });
expect(dependencies.ruleDataClient.getWriter).not.toBeCalled();
});
});

View file

@ -0,0 +1,95 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { PersistenceServices } from '../../../../../../rule_registry/server';
import { THRESHOLD_RULE_TYPE_ID } from '../../../../../common/constants';
import { thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas';
import { thresholdExecutor } from '../../signals/executors/threshold';
import { ThresholdAlertState } from '../../signals/types';
import { createSecurityRuleTypeFactory } from '../create_security_rule_type_factory';
import { CreateRuleOptions } from '../types';
export const createThresholdAlertType = (createOptions: CreateRuleOptions) => {
const {
experimentalFeatures,
lists,
logger,
mergeStrategy,
ignoreFields,
ruleDataClient,
version,
ruleDataService,
} = createOptions;
const createSecurityRuleType = createSecurityRuleTypeFactory({
lists,
logger,
mergeStrategy,
ignoreFields,
ruleDataClient,
ruleDataService,
});
return createSecurityRuleType<ThresholdRuleParams, {}, PersistenceServices, ThresholdAlertState>({
id: THRESHOLD_RULE_TYPE_ID,
name: 'Threshold Rule',
validate: {
params: {
validate: (object: unknown): ThresholdRuleParams => {
const [validated, errors] = validateNonExact(object, thresholdRuleParams);
if (errors != null) {
throw new Error(errors);
}
if (validated == null) {
throw new Error('Validation of rule params failed');
}
return validated;
},
},
},
actionGroups: [
{
id: 'default',
name: 'Default',
},
],
defaultActionGroupId: 'default',
actionVariables: {
context: [{ name: 'server', description: 'the server' }],
},
minimumLicenseRequired: 'basic',
isExportable: false,
producer: 'security-solution',
async executor(execOptions) {
const {
runOpts: { buildRuleMessage, bulkCreate, exceptionItems, rule, tuple, wrapHits },
services,
startedAt,
state,
} = execOptions;
// console.log(JSON.stringify(state));
const result = await thresholdExecutor({
buildRuleMessage,
bulkCreate,
exceptionItems,
experimentalFeatures,
logger,
rule,
services,
startedAt,
state,
tuple,
version,
wrapHits,
});
return result;
},
});
};

View file

@ -7,15 +7,19 @@
import { getFilter } from './find_rules';
import {
EQL_RULE_TYPE_ID,
INDICATOR_RULE_TYPE_ID,
ML_RULE_TYPE_ID,
QUERY_RULE_TYPE_ID,
THRESHOLD_RULE_TYPE_ID,
SIGNALS_ID,
} from '../../../../common/constants';
const allAlertTypeIds = `(alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID}
const allAlertTypeIds = `(alert.attributes.alertTypeId: ${EQL_RULE_TYPE_ID}
OR alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID}
OR alert.attributes.alertTypeId: ${QUERY_RULE_TYPE_ID}
OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID})`.replace(/[\n\r]/g, '');
OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID}
OR alert.attributes.alertTypeId: ${THRESHOLD_RULE_TYPE_ID})`.replace(/[\n\r]/g, '');
describe('find_rules', () => {
const fullFilterTestCases: Array<[boolean, string]> = [

View file

@ -69,6 +69,8 @@ import {
INDICATOR_RULE_TYPE_ID,
ML_RULE_TYPE_ID,
QUERY_RULE_TYPE_ID,
EQL_RULE_TYPE_ID,
THRESHOLD_RULE_TYPE_ID,
} from '../../../../common/constants';
const nonEqlLanguages = t.keyof({ kuery: null, lucene: null });
@ -206,12 +208,11 @@ export const notifyWhen = t.union([
export const allRuleTypes = t.union([
t.literal(SIGNALS_ID),
// t.literal(EQL_RULE_TYPE_ID),
t.literal(EQL_RULE_TYPE_ID),
t.literal(ML_RULE_TYPE_ID),
t.literal(QUERY_RULE_TYPE_ID),
// t.literal(SAVED_QUERY_RULE_TYPE_ID),
t.literal(INDICATOR_RULE_TYPE_ID),
// t.literal(THRESHOLD_RULE_TYPE_ID),
t.literal(THRESHOLD_RULE_TYPE_ID),
]);
export type AllRuleTypes = t.TypeOf<typeof allRuleTypes>;

View file

@ -73,6 +73,7 @@ describe('threshold_executor', () => {
exceptionItems,
experimentalFeatures: allowedExperimentalValues,
services: alertServices,
state: { initialized: true, signalHistory: {} },
version,
logger,
buildRuleMessage,

View file

@ -5,9 +5,12 @@
* 2.0.
*/
import { SearchHit } from '@elastic/elasticsearch/api/types';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import { Logger } from 'src/core/server';
import { SavedObject } from 'src/core/types';
import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
import {
AlertInstanceContext,
AlertInstanceState,
@ -28,6 +31,7 @@ import {
BulkCreate,
RuleRangeTuple,
SearchAfterAndBulkCreateReturnType,
ThresholdAlertState,
WrapHits,
} from '../types';
import {
@ -37,6 +41,7 @@ import {
} from '../utils';
import { BuildRuleMessage } from '../rule_messages';
import { ExperimentalFeatures } from '../../../../../common/experimental_features';
import { buildThresholdSignalHistory } from '../threshold/build_signal_history';
export const thresholdExecutor = async ({
rule,
@ -48,6 +53,7 @@ export const thresholdExecutor = async ({
logger,
buildRuleMessage,
startedAt,
state,
bulkCreate,
wrapHits,
}: {
@ -60,17 +66,48 @@ export const thresholdExecutor = async ({
logger: Logger;
buildRuleMessage: BuildRuleMessage;
startedAt: Date;
state: ThresholdAlertState;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
}): Promise<SearchAfterAndBulkCreateReturnType> => {
}): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
let result = createSearchAfterReturnType();
const ruleParams = rule.attributes.params;
// Get state or build initial state (on upgrade)
const { signalHistory, searchErrors: previousSearchErrors } = state.initialized
? { signalHistory: state.signalHistory, searchErrors: [] }
: await getThresholdSignalHistory({
indexPattern: ['*'], // TODO: get outputIndex?
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
services,
logger,
ruleId: ruleParams.ruleId,
bucketByFields: ruleParams.threshold.field,
timestampOverride: ruleParams.timestampOverride,
buildRuleMessage,
});
if (!state.initialized) {
// Clean up any signal history that has fallen outside the window
const toDelete: string[] = [];
for (const [hash, entry] of Object.entries(signalHistory)) {
if (entry.lastSignalTimestamp < tuple.from.valueOf()) {
toDelete.push(hash);
}
}
for (const hash of toDelete) {
delete signalHistory[hash];
}
}
if (hasLargeValueItem(exceptionItems)) {
result.warningMessages.push(
'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules'
);
result.warning = true;
}
const inputIndex = await getInputIndex({
experimentalFeatures,
services,
@ -78,23 +115,8 @@ export const thresholdExecutor = async ({
index: ruleParams.index,
});
const {
thresholdSignalHistory,
searchErrors: previousSearchErrors,
} = await getThresholdSignalHistory({
indexPattern: [ruleParams.outputIndex],
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
services,
logger,
ruleId: ruleParams.ruleId,
bucketByFields: ruleParams.threshold.field,
timestampOverride: ruleParams.timestampOverride,
buildRuleMessage,
});
const bucketFilters = await getThresholdBucketFilters({
thresholdSignalHistory,
signalHistory,
timestampOverride: ruleParams.timestampOverride,
});
@ -141,7 +163,7 @@ export const thresholdExecutor = async ({
signalsIndex: ruleParams.outputIndex,
startedAt,
from: tuple.from.toDate(),
thresholdSignalHistory,
signalHistory,
bulkCreate,
wrapHits,
});
@ -161,5 +183,31 @@ export const thresholdExecutor = async ({
searchAfterTimes: [thresholdSearchDuration],
}),
]);
return result;
const createdAlerts = createdItems.map((alert) => {
const { _id, _index, ...source } = alert as { _id: string; _index: string };
return {
_id,
_index,
_source: {
...source,
},
} as SearchHit<unknown>;
});
const newSignalHistory = buildThresholdSignalHistory({
alerts: createdAlerts,
});
return {
...result,
state: {
...state,
initialized: true,
signalHistory: {
...signalHistory,
...newSignalHistory,
},
},
};
};

View file

@ -29,7 +29,7 @@ import {
} from '../../../../common/detection_engine/utils';
import { SetupPlugins } from '../../../plugin';
import { getInputIndex } from './get_input_output_index';
import { AlertAttributes, SignalRuleAlertTypeDefinition } from './types';
import { AlertAttributes, SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types';
import {
getListsClient,
getExceptions,
@ -125,6 +125,7 @@ export const signalRulesAlertType = ({
async executor({
previousStartedAt,
startedAt,
state,
alertId,
services,
params,
@ -316,6 +317,7 @@ export const signalRulesAlertType = ({
logger,
buildRuleMessage,
startedAt,
state: state as ThresholdAlertState,
bulkCreate,
wrapHits,
});

View file

@ -75,9 +75,11 @@ export const singleSearchAfter = async ({
searchAfterQuery as estypes.SearchRequest
);
const end = performance.now();
const searchErrors = createErrorsFromShard({
errors: nextSearchAfterResult._shards.failures ?? [],
});
return {
searchResult: nextSearchAfterResult,
searchDuration: makeFloatString(end - start),

View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ALERT_ORIGINAL_TIME } from '../../rule_types/field_maps/field_names';
import { sampleThresholdAlert } from '../../rule_types/__mocks__/threshold';
import { buildThresholdSignalHistory } from './build_signal_history';
describe('buildSignalHistory', () => {
it('builds a signal history from an alert', () => {
const signalHistory = buildThresholdSignalHistory({ alerts: [sampleThresholdAlert] });
expect(signalHistory).toEqual({
'7a75c5c2db61f57ec166c669cb8244b91f812f0b2f1d4f8afd528d4f8b4e199b': {
lastSignalTimestamp: Date.parse(
sampleThresholdAlert._source[ALERT_ORIGINAL_TIME] as string
),
terms: [
{
field: 'host.name',
value: 'garden-gnomes',
},
{
field: 'source.ip',
value: '127.0.0.1',
},
],
},
});
});
});

View file

@ -0,0 +1,75 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SearchHit } from '@elastic/elasticsearch/api/types';
import {
ALERT_ORIGINAL_TIME,
ALERT_RULE_THRESHOLD_FIELD,
} from '../../rule_types/field_maps/field_names';
import { SimpleHit, ThresholdSignalHistory } from '../types';
import { getThresholdTermsHash, isWrappedRACAlert, isWrappedSignalHit } from '../utils';
interface GetThresholdSignalHistoryParams {
alerts: Array<SearchHit<unknown>>;
}
const getTerms = (alert: SimpleHit) => {
if (isWrappedRACAlert(alert)) {
return (alert._source[ALERT_RULE_THRESHOLD_FIELD] as string[]).map((field) => ({
field,
value: alert._source[field] as string,
}));
} else if (isWrappedSignalHit(alert)) {
return alert._source.signal?.threshold_result?.terms ?? [];
} else {
// We shouldn't be here
return [];
}
};
const getOriginalTime = (alert: SimpleHit) => {
if (isWrappedRACAlert(alert)) {
const originalTime = alert._source[ALERT_ORIGINAL_TIME];
return originalTime != null ? new Date(originalTime as string).getTime() : undefined;
} else if (isWrappedSignalHit(alert)) {
const originalTime = alert._source.signal?.original_time;
return originalTime != null ? new Date(originalTime).getTime() : undefined;
} else {
// We shouldn't be here
return undefined;
}
};
export const buildThresholdSignalHistory = ({
alerts,
}: GetThresholdSignalHistoryParams): ThresholdSignalHistory => {
const signalHistory = alerts.reduce<ThresholdSignalHistory>((acc, alert) => {
if (!alert._source) {
return acc;
}
const terms = getTerms(alert as SimpleHit);
const hash = getThresholdTermsHash(terms);
const existing = acc[hash];
const originalTime = getOriginalTime(alert as SimpleHit);
if (existing != null) {
if (originalTime && originalTime > existing.lastSignalTimestamp) {
acc[hash].lastSignalTimestamp = originalTime;
}
} else if (originalTime) {
acc[hash] = {
terms,
lastSignalTimestamp: originalTime,
};
}
return acc;
}, {});
return signalHistory;
};

View file

@ -46,7 +46,7 @@ interface BulkCreateThresholdSignalsParams {
signalsIndex: string;
startedAt: Date;
from: Date;
thresholdSignalHistory: ThresholdSignalHistory;
signalHistory: ThresholdSignalHistory;
bulkCreate: BulkCreate;
wrapHits: WrapHits;
}
@ -61,7 +61,7 @@ const getTransformedHits = (
ruleId: string,
filter: unknown,
timestampOverride: TimestampOverrideOrUndefined,
thresholdSignalHistory: ThresholdSignalHistory
signalHistory: ThresholdSignalHistory
) => {
const aggParts = threshold.field.length
? results.aggregations && getThresholdAggregationParts(results.aggregations)
@ -148,7 +148,7 @@ const getTransformedHits = (
}
const termsHash = getThresholdTermsHash(bucket.terms);
const signalHit = thresholdSignalHistory[termsHash];
const signalHit = signalHistory[termsHash];
const source = {
'@timestamp': timestamp,
@ -202,7 +202,7 @@ export const transformThresholdResultsToEcs = (
threshold: ThresholdNormalized,
ruleId: string,
timestampOverride: TimestampOverrideOrUndefined,
thresholdSignalHistory: ThresholdSignalHistory
signalHistory: ThresholdSignalHistory
): SignalSearchResponse => {
const transformedHits = getTransformedHits(
results,
@ -214,7 +214,7 @@ export const transformThresholdResultsToEcs = (
ruleId,
filter,
timestampOverride,
thresholdSignalHistory
signalHistory
);
const thresholdResults = {
...results,
@ -246,7 +246,7 @@ export const bulkCreateThresholdSignals = async (
ruleParams.threshold,
ruleParams.ruleId,
ruleParams.timestampOverride,
params.thresholdSignalHistory
params.signalHistory
);
return params.bulkCreate(

View file

@ -227,6 +227,7 @@ describe('findThresholdSignals', () => {
'threshold_1:user.name': {
terms: {
field: 'user.name',
order: { cardinality_count: 'desc' },
min_doc_count: 100,
size: 10000,
},
@ -302,6 +303,7 @@ describe('findThresholdSignals', () => {
lang: 'painless',
},
min_doc_count: 200,
order: { cardinality_count: 'desc' },
},
aggs: {
cardinality_count: {

View file

@ -89,6 +89,13 @@ export const findThresholdSignals = async ({
const thresholdFields = threshold.field;
// order buckets by cardinality (https://github.com/elastic/kibana/issues/95258)
const thresholdFieldCount = thresholdFields.length;
const orderByCardinality = (i: number = 0) =>
(thresholdFieldCount === 0 || i === thresholdFieldCount - 1) && threshold.cardinality?.length
? { order: { cardinality_count: 'desc' } }
: {};
// Generate a nested terms aggregation for each threshold grouping field provided, appending leaf
// aggregations to 1) filter out buckets that don't meet the cardinality threshold, if provided, and
// 2) return the latest hit for each bucket so that we can persist the timestamp of the event in the
@ -104,6 +111,7 @@ export const findThresholdSignals = async ({
set(acc, aggPath, {
terms: {
field,
...orderByCardinality(i),
min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set
size: 10000, // max 10k buckets
},
@ -121,6 +129,7 @@ export const findThresholdSignals = async ({
source: '""', // Group everything in the same bucket
lang: 'painless',
},
...orderByCardinality(),
min_doc_count: threshold.value,
},
aggs: leafAggs,

View file

@ -11,7 +11,7 @@ import { getThresholdBucketFilters } from './get_threshold_bucket_filters';
describe('getThresholdBucketFilters', () => {
it('should generate filters for threshold signal detection with dupe mitigation', async () => {
const result = await getThresholdBucketFilters({
thresholdSignalHistory: sampleThresholdSignalHistory(),
signalHistory: sampleThresholdSignalHistory(),
timestampOverride: undefined,
});
expect(result).toEqual([

View file

@ -9,14 +9,18 @@ import { Filter } from 'src/plugins/data/common';
import { ESFilter } from '../../../../../../../../src/core/types/elasticsearch';
import { ThresholdSignalHistory, ThresholdSignalHistoryRecord } from '../types';
/*
* Returns a filter to exclude events that have already been included in a
* previous threshold signal. Uses the threshold signal history to achieve this.
*/
export const getThresholdBucketFilters = async ({
thresholdSignalHistory,
signalHistory,
timestampOverride,
}: {
thresholdSignalHistory: ThresholdSignalHistory;
signalHistory: ThresholdSignalHistory;
timestampOverride: string | undefined;
}): Promise<Filter[]> => {
const filters = Object.values(thresholdSignalHistory).reduce(
const filters = Object.values(signalHistory).reduce(
(acc: ESFilter[], bucket: ThresholdSignalHistoryRecord): ESFilter[] => {
const filter = {
bool: {
@ -24,6 +28,7 @@ export const getThresholdBucketFilters = async ({
{
range: {
[timestampOverride ?? '@timestamp']: {
// Timestamp of last event signaled on for this set of terms.
lte: new Date(bucket.lastSignalTimestamp).toISOString(),
},
},
@ -32,6 +37,7 @@ export const getThresholdBucketFilters = async ({
},
} as ESFilter;
// Terms to filter events older than `lastSignalTimestamp`.
bucket.terms.forEach((term) => {
if (term.field != null) {
(filter.bool!.filter as ESFilter[]).push({

View file

@ -5,7 +5,6 @@
* 2.0.
*/
import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema';
import { TimestampOverrideOrUndefined } from '../../../../../common/detection_engine/schemas/common/schemas';
import {
AlertInstanceContext,
@ -16,7 +15,7 @@ import { Logger } from '../../../../../../../../src/core/server';
import { ThresholdSignalHistory } from '../types';
import { BuildRuleMessage } from '../rule_messages';
import { findPreviousThresholdSignals } from './find_previous_threshold_signals';
import { getThresholdTermsHash } from '../utils';
import { buildThresholdSignalHistory } from './build_signal_history';
interface GetThresholdSignalHistoryParams {
from: string;
@ -41,7 +40,7 @@ export const getThresholdSignalHistory = async ({
timestampOverride,
buildRuleMessage,
}: GetThresholdSignalHistoryParams): Promise<{
thresholdSignalHistory: ThresholdSignalHistory;
signalHistory: ThresholdSignalHistory;
searchErrors: string[];
}> => {
const { searchResult, searchErrors } = await findPreviousThresholdSignals({
@ -56,51 +55,10 @@ export const getThresholdSignalHistory = async ({
buildRuleMessage,
});
const thresholdSignalHistory = searchResult.hits.hits.reduce<ThresholdSignalHistory>(
(acc, hit) => {
if (!hit._source) {
return acc;
}
const terms =
hit._source.signal?.threshold_result?.terms != null
? hit._source.signal.threshold_result.terms
: [
// Pre-7.12 signals
{
field:
(((hit._source.signal?.rule as RulesSchema).threshold as unknown) as {
field: string;
}).field ?? '',
value: ((hit._source.signal?.threshold_result as unknown) as { value: string })
.value,
},
];
const hash = getThresholdTermsHash(terms);
const existing = acc[hash];
const originalTime =
hit._source.signal?.original_time != null
? new Date(hit._source.signal?.original_time).getTime()
: undefined;
if (existing != null) {
if (originalTime && originalTime > existing.lastSignalTimestamp) {
acc[hash].lastSignalTimestamp = originalTime;
}
} else if (originalTime) {
acc[hash] = {
terms,
lastSignalTimestamp: originalTime,
};
}
return acc;
},
{}
);
return {
thresholdSignalHistory,
signalHistory: buildThresholdSignalHistory({
alerts: searchResult.hits.hits,
}),
searchErrors,
};
};

View file

@ -362,3 +362,8 @@ export interface ThresholdQueryBucket extends TermAggregationBucket {
value_as_string: string;
};
}
export interface ThresholdAlertState extends AlertTypeState {
initialized: boolean;
signalHistory: ThresholdSignalHistory;
}

View file

@ -63,10 +63,12 @@ import { WrappedRACAlert } from '../rule_types/types';
import { SearchTypes } from '../../../../common/detection_engine/types';
import { IRuleExecutionLogClient } from '../rule_execution_log/types';
import {
EQL_RULE_TYPE_ID,
INDICATOR_RULE_TYPE_ID,
ML_RULE_TYPE_ID,
QUERY_RULE_TYPE_ID,
SIGNALS_ID,
THRESHOLD_RULE_TYPE_ID,
} from '../../../../common/constants';
interface SortExceptionsReturn {
@ -1013,10 +1015,10 @@ export const getField = <T extends SearchTypes>(event: SimpleHit, field: string)
* Maps legacy rule types to RAC rule type IDs.
*/
export const ruleTypeMappings = {
eql: SIGNALS_ID,
eql: EQL_RULE_TYPE_ID,
machine_learning: ML_RULE_TYPE_ID,
query: QUERY_RULE_TYPE_ID,
saved_query: SIGNALS_ID,
threat_match: INDICATOR_RULE_TYPE_ID,
threshold: SIGNALS_ID,
threshold: THRESHOLD_RULE_TYPE_ID,
};

View file

@ -53,6 +53,7 @@ import {
createIndicatorMatchAlertType,
createMlAlertType,
createQueryAlertType,
createThresholdAlertType,
} from './lib/detection_engine/rule_types';
import { initRoutes } from './routes';
import { isAlertExecutor } from './lib/detection_engine/signals/types';
@ -264,9 +265,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
};
this.setupPlugins.alerting.registerType(createEqlAlertType(createRuleOptions));
this.setupPlugins.alerting.registerType(createQueryAlertType(createRuleOptions));
this.setupPlugins.alerting.registerType(createIndicatorMatchAlertType(createRuleOptions));
this.setupPlugins.alerting.registerType(createMlAlertType(createRuleOptions));
this.setupPlugins.alerting.registerType(createQueryAlertType(createRuleOptions));
this.setupPlugins.alerting.registerType(createThresholdAlertType(createRuleOptions));
}
// TODO We need to get the endpoint routes inside of initRoutes