diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index 8076caf60f69..a93439b29069 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -197,7 +197,6 @@ export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const; export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const; export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const; export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const; -export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const; export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const; /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts index 40d2ed37a557..554672806c12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/threshold.ts @@ -5,7 +5,35 @@ * 2.0. */ +import { + SPACE_IDS, + ALERT_RULE_CONSUMER, + ALERT_REASON, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_WORKFLOW_STATUS, + ALERT_RULE_NAMESPACE, + ALERT_INSTANCE_ID, + ALERT_UUID, + ALERT_RULE_TYPE_ID, + ALERT_RULE_PRODUCER, + ALERT_RULE_CATEGORY, + ALERT_RULE_UUID, + ALERT_RULE_NAME, +} from '@kbn/rule-data-utils'; +import { TypeOfFieldMap } from '../../../../../../rule_registry/common/field_map'; +import { SERVER_APP_ID } from '../../../../../common/constants'; +import { ANCHOR_DATE } from '../../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { sampleDocNoSortId } from '../../signals/__mocks__/es_results'; +import { flattenWithPrefix } from '../factories/utils/flatten_with_prefix'; +import { RulesFieldMap } from '../field_maps'; +import { + ALERT_ANCESTORS, + ALERT_ORIGINAL_TIME, + ALERT_ORIGINAL_EVENT, +} from '../field_maps/field_names'; +import { WrappedRACAlert } from '../types'; export const mockThresholdResults = { rawResponse: { @@ -59,3 +87,79 @@ export const mockThresholdResults = { }, }, }; + +export const sampleThresholdAlert: WrappedRACAlert = { + _id: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d', + _index: 'some-index', + _source: { + '@timestamp': '2020-04-20T21:26:30.000Z', + [SPACE_IDS]: ['default'], + [ALERT_INSTANCE_ID]: 'b3ad77a4-65bd-4c4e-89cf-13c46f54bc4d', + [ALERT_UUID]: '310158f7-994d-4a38-8cdc-152139ac4d29', + [ALERT_RULE_CONSUMER]: SERVER_APP_ID, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', + [ALERT_ORIGINAL_EVENT]: { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + [ALERT_REASON]: 'alert reasonable reason', + [ALERT_STATUS]: ALERT_STATUS_ACTIVE, + [ALERT_WORKFLOW_STATUS]: 'open', + 'source.ip': '127.0.0.1', + 'host.name': 'garden-gnomes', + [ALERT_RULE_CATEGORY]: 'security', + [ALERT_RULE_NAME]: 'a threshold rule', + [ALERT_RULE_PRODUCER]: 'siem', + [ALERT_RULE_TYPE_ID]: 'query-rule-id', + [ALERT_RULE_UUID]: '151af49f-2e82-4b6f-831b-7f8cb341a5ff', + ...(flattenWithPrefix(ALERT_RULE_NAMESPACE, { + author: [], + uuid: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + threshold: { + field: ['source.ip', 'host.name'], + value: 1, + }, + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'f88a544c-1d4e-4652-ae2a-c953b38da5d0', + interval: '5m', + exceptions_list: getListArrayMock(), + }) as TypeOfFieldMap), + 'kibana.alert.depth': 1, + }, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts index c600e187bc8f..cd7a5150ad38 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; + import { performance } from 'perf_hooks'; import { countBy, isEmpty } from 'lodash'; @@ -62,11 +64,15 @@ export const bulkCreateFactory = ( ); const createdItems = wrappedDocs - .map((doc, index) => ({ - _id: response.body.items[index].index?._id ?? '', - _index: response.body.items[index].index?._index ?? '', - ...doc._source, - })) + .map((doc, index) => { + const responseIndex = response.body.items[index].index; + return { + _id: responseIndex?._id ?? '', + _index: responseIndex?._index ?? '', + [ALERT_INSTANCE_ID]: responseIndex?._id ?? '', + ...doc._source, + }; + }) .filter((_, index) => response.body.items[index].index?.status === 201); const createdItemsCount = createdItems.length; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts index 9be18a1a1345..453c4d8b785d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -102,7 +102,6 @@ describe('buildAlert', () => { status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], @@ -179,7 +178,6 @@ describe('buildAlert', () => { status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index 5fbca2dc6178..d39d6fa2eb80 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -120,7 +120,7 @@ export const buildAlert = ( [] ); - const { id, ...mappedRule } = rule; + const { id, output_index: outputIndex, ...mappedRule } = rule; mappedRule.uuid = id; return ({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts index d472dc5885e5..02f418a15188 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/flatten_with_prefix.ts @@ -5,16 +5,26 @@ * 2.0. */ +import { isPlainObject } from 'lodash'; import { SearchTypes } from '../../../../../../common/detection_engine/types'; export const flattenWithPrefix = ( prefix: string, - obj: Record + maybeObj: unknown ): Record => { - return Object.keys(obj).reduce((acc: Record, key) => { + if (maybeObj != null && isPlainObject(maybeObj)) { + return Object.keys(maybeObj as Record).reduce( + (acc: Record, key) => { + return { + ...acc, + ...flattenWithPrefix(`${prefix}.${key}`, (maybeObj as Record)[key]), + }; + }, + {} + ); + } else { return { - ...acc, - [`${prefix}.${key}`]: obj[key], + [prefix]: maybeObj as SearchTypes, }; - }, {}); + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts index 1c4b7f03fd73..f21fc5b6ad39 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts @@ -43,11 +43,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: true, }, - 'kibana.alert.group': { - type: 'object', - array: false, - required: false, - }, 'kibana.alert.group.id': { type: 'keyword', array: false, @@ -58,11 +53,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: false, }, - 'kibana.alert.original_event': { - type: 'object', - array: false, - required: false, - }, 'kibana.alert.original_event.action': { type: 'keyword', array: false, @@ -198,81 +188,6 @@ export const alertsFieldMap: FieldMap = { array: false, required: false, }, - 'kibana.alert.threat': { - type: 'object', - array: false, - required: false, - }, - 'kibana.alert.threat.framework': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic': { - type: 'object', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic.id': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic.name': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.tactic.reference': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique': { - type: 'object', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.id': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.name': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.reference': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique': { - type: 'object', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique.id': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique.name': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threat.technique.subtechnique.reference': { - type: 'keyword', - array: false, - required: true, - }, - 'kibana.alert.threshold_result': { - type: 'object', - array: false, - required: false, - }, 'kibana.alert.threshold_result.cardinality': { type: 'object', array: false, @@ -300,7 +215,7 @@ export const alertsFieldMap: FieldMap = { }, 'kibana.alert.threshold_result.terms': { type: 'object', - array: false, + array: true, required: false, }, 'kibana.alert.threshold_result.terms.field': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts index fb9e597a3044..68d08e08086a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/field_names.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ALERT_NAMESPACE } from '@kbn/rule-data-utils'; +import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils'; export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const; export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const; @@ -14,3 +14,6 @@ export const ALERT_GROUP_ID = `${ALERT_NAMESPACE}.group.id` as const; export const ALERT_GROUP_INDEX = `${ALERT_NAMESPACE}.group.index` as const; export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const; export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const; + +const ALERT_RULE_THRESHOLD = `${ALERT_RULE_NAMESPACE}.threshold` as const; +export const ALERT_RULE_THRESHOLD_FIELD = `${ALERT_RULE_THRESHOLD}.field` as const; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts index 21405672fdf7..87b55e092ec5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts @@ -11,6 +11,11 @@ export const rulesFieldMap = { array: false, required: false, }, + 'kibana.alert.rule.exceptions_list': { + type: 'object', + array: true, + required: false, + }, 'kibana.alert.rule.false_positives': { type: 'keyword', array: true, @@ -46,6 +51,56 @@ export const rulesFieldMap = { array: true, required: true, }, + 'kibana.alert.rule.threat.framework': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic.id': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic.name': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.tactic.reference': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.id': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.name': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.reference': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique.id': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique.name': { + type: 'keyword', + array: false, + required: true, + }, + 'kibana.alert.rule.threat.technique.subtechnique.reference': { + type: 'keyword', + array: false, + required: true, + }, 'kibana.alert.rule.threat_filters': { type: 'keyword', array: true, @@ -91,11 +146,6 @@ export const rulesFieldMap = { array: true, required: false, }, - 'kibana.alert.rule.threshold': { - type: 'object', - array: true, - required: false, - }, 'kibana.alert.rule.threshold.field': { type: 'keyword', array: true, @@ -103,7 +153,7 @@ export const rulesFieldMap = { }, 'kibana.alert.rule.threshold.value': { type: 'float', // TODO: should be 'long' (eventually, after we stabilize) - array: true, + array: false, required: false, }, 'kibana.alert.rule.threshold.cardinality': { @@ -113,12 +163,12 @@ export const rulesFieldMap = { }, 'kibana.alert.rule.threshold.cardinality.field': { type: 'keyword', - array: true, + array: false, required: false, }, 'kibana.alert.rule.threshold.cardinality.value': { type: 'long', - array: true, + array: false, required: false, }, 'kibana.alert.rule.timeline_id': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts index 39325cab2c76..1787a15588b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts @@ -7,5 +7,6 @@ export { createEqlAlertType } from './eql/create_eql_alert_type'; export { createIndicatorMatchAlertType } from './indicator_match/create_indicator_match_alert_type'; -export { createQueryAlertType } from './query/create_query_alert_type'; export { createMlAlertType } from './ml/create_ml_alert_type'; +export { createQueryAlertType } from './query/create_query_alert_type'; +export { createThresholdAlertType } from './threshold/create_threshold_alert_type'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index ed791af08890..e45d8440386f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -24,7 +24,7 @@ jest.mock('../utils/get_list_client', () => ({ jest.mock('../../rule_execution_log/rule_execution_log_client'); -describe('Custom query alerts', () => { +describe('Custom Query Alerts', () => { it('does not send an alert when no events found', async () => { const { services, dependencies, executor } = createRuleTypeMocks(); const queryAlertType = createQueryAlertType({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh new file mode 100644 index 000000000000..47c5cb4eda2e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_threshold.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' \ + --verbose \ + -d ' +{ + "params":{ + "author": [], + "description": "Basic threshold rule", + "exceptionsList": [], + "falsePositives": [], + "from": "now-300s", + "query": "*:*", + "immutable": false, + "index": ["*"], + "language": "kuery", + "maxSignals": 10, + "outputIndex": "", + "references": [], + "riskScore": 21, + "riskScoreMapping": [], + "ruleId": "52dec1ba-b779-469c-9667-6b0e865fb89a", + "severity": "low", + "severityMapping": [], + "threat": [], + "threshold": { + "field": ["source.ip"], + "value": 2, + "cardinality": [ + { + "field": "source.ip", + "value": 1 + } + ] + }, + "to": "now", + "type": "threshold", + "version": 1 + }, + "consumer":"alerts", + "alertTypeId":"siem.thresholdRule", + "schedule":{ + "interval":"1m" + }, + "actions":[], + "tags":[ + "custom", + "persistence" + ], + "notifyWhen":"onActionGroupChange", + "name":"Basic threshold rule" +}' + + diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts new file mode 100644 index 000000000000..74435cb30047 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { allowedExperimentalValues } from '../../../../../common/experimental_features'; +import { createThresholdAlertType } from './create_threshold_alert_type'; +import { createRuleTypeMocks } from '../__mocks__/rule_type'; +import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock'; + +jest.mock('../../rule_execution_log/rule_execution_log_client'); + +describe('Threshold Alerts', () => { + it('does not send an alert when no events found', async () => { + const params = getThresholdRuleParams(); + const { dependencies, executor } = createRuleTypeMocks('threshold', params); + const thresholdAlertTpe = createThresholdAlertType({ + experimentalFeatures: allowedExperimentalValues, + lists: dependencies.lists, + logger: dependencies.logger, + mergeStrategy: 'allFields', + ignoreFields: [], + ruleDataClient: dependencies.ruleDataClient, + ruleDataService: dependencies.ruleDataService, + version: '1.0.0', + }); + dependencies.alerting.registerType(thresholdAlertTpe); + + await executor({ params }); + expect(dependencies.ruleDataClient.getWriter).not.toBeCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts new file mode 100644 index 000000000000..a503cf5aedbe --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; + +import { PersistenceServices } from '../../../../../../rule_registry/server'; +import { THRESHOLD_RULE_TYPE_ID } from '../../../../../common/constants'; +import { thresholdRuleParams, ThresholdRuleParams } from '../../schemas/rule_schemas'; +import { thresholdExecutor } from '../../signals/executors/threshold'; +import { ThresholdAlertState } from '../../signals/types'; +import { createSecurityRuleTypeFactory } from '../create_security_rule_type_factory'; +import { CreateRuleOptions } from '../types'; + +export const createThresholdAlertType = (createOptions: CreateRuleOptions) => { + const { + experimentalFeatures, + lists, + logger, + mergeStrategy, + ignoreFields, + ruleDataClient, + version, + ruleDataService, + } = createOptions; + const createSecurityRuleType = createSecurityRuleTypeFactory({ + lists, + logger, + mergeStrategy, + ignoreFields, + ruleDataClient, + ruleDataService, + }); + return createSecurityRuleType({ + id: THRESHOLD_RULE_TYPE_ID, + name: 'Threshold Rule', + validate: { + params: { + validate: (object: unknown): ThresholdRuleParams => { + const [validated, errors] = validateNonExact(object, thresholdRuleParams); + if (errors != null) { + throw new Error(errors); + } + if (validated == null) { + throw new Error('Validation of rule params failed'); + } + return validated; + }, + }, + }, + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + actionVariables: { + context: [{ name: 'server', description: 'the server' }], + }, + minimumLicenseRequired: 'basic', + isExportable: false, + producer: 'security-solution', + async executor(execOptions) { + const { + runOpts: { buildRuleMessage, bulkCreate, exceptionItems, rule, tuple, wrapHits }, + services, + startedAt, + state, + } = execOptions; + + // console.log(JSON.stringify(state)); + + const result = await thresholdExecutor({ + buildRuleMessage, + bulkCreate, + exceptionItems, + experimentalFeatures, + logger, + rule, + services, + startedAt, + state, + tuple, + version, + wrapHits, + }); + + return result; + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts index 0f7545c4df93..ebde1d0ad6df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/find_rules.test.ts @@ -7,15 +7,19 @@ import { getFilter } from './find_rules'; import { + EQL_RULE_TYPE_ID, INDICATOR_RULE_TYPE_ID, ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, SIGNALS_ID, } from '../../../../common/constants'; -const allAlertTypeIds = `(alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID} +const allAlertTypeIds = `(alert.attributes.alertTypeId: ${EQL_RULE_TYPE_ID} + OR alert.attributes.alertTypeId: ${ML_RULE_TYPE_ID} OR alert.attributes.alertTypeId: ${QUERY_RULE_TYPE_ID} - OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID})`.replace(/[\n\r]/g, ''); + OR alert.attributes.alertTypeId: ${INDICATOR_RULE_TYPE_ID} + OR alert.attributes.alertTypeId: ${THRESHOLD_RULE_TYPE_ID})`.replace(/[\n\r]/g, ''); describe('find_rules', () => { const fullFilterTestCases: Array<[boolean, string]> = [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts index e9215084614c..578d8c4926b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts @@ -69,6 +69,8 @@ import { INDICATOR_RULE_TYPE_ID, ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, + EQL_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, } from '../../../../common/constants'; const nonEqlLanguages = t.keyof({ kuery: null, lucene: null }); @@ -206,12 +208,11 @@ export const notifyWhen = t.union([ export const allRuleTypes = t.union([ t.literal(SIGNALS_ID), - // t.literal(EQL_RULE_TYPE_ID), + t.literal(EQL_RULE_TYPE_ID), t.literal(ML_RULE_TYPE_ID), t.literal(QUERY_RULE_TYPE_ID), - // t.literal(SAVED_QUERY_RULE_TYPE_ID), t.literal(INDICATOR_RULE_TYPE_ID), - // t.literal(THRESHOLD_RULE_TYPE_ID), + t.literal(THRESHOLD_RULE_TYPE_ID), ]); export type AllRuleTypes = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index afcb3707591f..5766390099e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -73,6 +73,7 @@ describe('threshold_executor', () => { exceptionItems, experimentalFeatures: allowedExperimentalValues, services: alertServices, + state: { initialized: true, signalHistory: {} }, version, logger, buildRuleMessage, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index ffd90f3b90b9..524bc6a6c524 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -5,9 +5,12 @@ * 2.0. */ +import { SearchHit } from '@elastic/elasticsearch/api/types'; +import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; + import { Logger } from 'src/core/server'; import { SavedObject } from 'src/core/types'; -import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; + import { AlertInstanceContext, AlertInstanceState, @@ -28,6 +31,7 @@ import { BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, + ThresholdAlertState, WrapHits, } from '../types'; import { @@ -37,6 +41,7 @@ import { } from '../utils'; import { BuildRuleMessage } from '../rule_messages'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; +import { buildThresholdSignalHistory } from '../threshold/build_signal_history'; export const thresholdExecutor = async ({ rule, @@ -48,6 +53,7 @@ export const thresholdExecutor = async ({ logger, buildRuleMessage, startedAt, + state, bulkCreate, wrapHits, }: { @@ -60,17 +66,48 @@ export const thresholdExecutor = async ({ logger: Logger; buildRuleMessage: BuildRuleMessage; startedAt: Date; + state: ThresholdAlertState; bulkCreate: BulkCreate; wrapHits: WrapHits; -}): Promise => { +}): Promise => { let result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; + + // Get state or build initial state (on upgrade) + const { signalHistory, searchErrors: previousSearchErrors } = state.initialized + ? { signalHistory: state.signalHistory, searchErrors: [] } + : await getThresholdSignalHistory({ + indexPattern: ['*'], // TODO: get outputIndex? + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + ruleId: ruleParams.ruleId, + bucketByFields: ruleParams.threshold.field, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); + + if (!state.initialized) { + // Clean up any signal history that has fallen outside the window + const toDelete: string[] = []; + for (const [hash, entry] of Object.entries(signalHistory)) { + if (entry.lastSignalTimestamp < tuple.from.valueOf()) { + toDelete.push(hash); + } + } + for (const hash of toDelete) { + delete signalHistory[hash]; + } + } + if (hasLargeValueItem(exceptionItems)) { result.warningMessages.push( 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' ); result.warning = true; } + const inputIndex = await getInputIndex({ experimentalFeatures, services, @@ -78,23 +115,8 @@ export const thresholdExecutor = async ({ index: ruleParams.index, }); - const { - thresholdSignalHistory, - searchErrors: previousSearchErrors, - } = await getThresholdSignalHistory({ - indexPattern: [ruleParams.outputIndex], - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - ruleId: ruleParams.ruleId, - bucketByFields: ruleParams.threshold.field, - timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); - const bucketFilters = await getThresholdBucketFilters({ - thresholdSignalHistory, + signalHistory, timestampOverride: ruleParams.timestampOverride, }); @@ -141,7 +163,7 @@ export const thresholdExecutor = async ({ signalsIndex: ruleParams.outputIndex, startedAt, from: tuple.from.toDate(), - thresholdSignalHistory, + signalHistory, bulkCreate, wrapHits, }); @@ -161,5 +183,31 @@ export const thresholdExecutor = async ({ searchAfterTimes: [thresholdSearchDuration], }), ]); - return result; + + const createdAlerts = createdItems.map((alert) => { + const { _id, _index, ...source } = alert as { _id: string; _index: string }; + return { + _id, + _index, + _source: { + ...source, + }, + } as SearchHit; + }); + + const newSignalHistory = buildThresholdSignalHistory({ + alerts: createdAlerts, + }); + + return { + ...result, + state: { + ...state, + initialized: true, + signalHistory: { + ...signalHistory, + ...newSignalHistory, + }, + }, + }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 68d60f7757e4..9a6c099ed176 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -29,7 +29,7 @@ import { } from '../../../../common/detection_engine/utils'; import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; -import { AlertAttributes, SignalRuleAlertTypeDefinition } from './types'; +import { AlertAttributes, SignalRuleAlertTypeDefinition, ThresholdAlertState } from './types'; import { getListsClient, getExceptions, @@ -125,6 +125,7 @@ export const signalRulesAlertType = ({ async executor({ previousStartedAt, startedAt, + state, alertId, services, params, @@ -316,6 +317,7 @@ export const signalRulesAlertType = ({ logger, buildRuleMessage, startedAt, + state: state as ThresholdAlertState, bulkCreate, wrapHits, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index ff49fb5892f5..7b30f70701e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -75,9 +75,11 @@ export const singleSearchAfter = async ({ searchAfterQuery as estypes.SearchRequest ); const end = performance.now(); + const searchErrors = createErrorsFromShard({ errors: nextSearchAfterResult._shards.failures ?? [], }); + return { searchResult: nextSearchAfterResult, searchDuration: makeFloatString(end - start), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts new file mode 100644 index 000000000000..8362942af15b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALERT_ORIGINAL_TIME } from '../../rule_types/field_maps/field_names'; +import { sampleThresholdAlert } from '../../rule_types/__mocks__/threshold'; +import { buildThresholdSignalHistory } from './build_signal_history'; + +describe('buildSignalHistory', () => { + it('builds a signal history from an alert', () => { + const signalHistory = buildThresholdSignalHistory({ alerts: [sampleThresholdAlert] }); + expect(signalHistory).toEqual({ + '7a75c5c2db61f57ec166c669cb8244b91f812f0b2f1d4f8afd528d4f8b4e199b': { + lastSignalTimestamp: Date.parse( + sampleThresholdAlert._source[ALERT_ORIGINAL_TIME] as string + ), + terms: [ + { + field: 'host.name', + value: 'garden-gnomes', + }, + { + field: 'source.ip', + value: '127.0.0.1', + }, + ], + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts new file mode 100644 index 000000000000..81b12d2d4f22 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/build_signal_history.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchHit } from '@elastic/elasticsearch/api/types'; +import { + ALERT_ORIGINAL_TIME, + ALERT_RULE_THRESHOLD_FIELD, +} from '../../rule_types/field_maps/field_names'; + +import { SimpleHit, ThresholdSignalHistory } from '../types'; +import { getThresholdTermsHash, isWrappedRACAlert, isWrappedSignalHit } from '../utils'; + +interface GetThresholdSignalHistoryParams { + alerts: Array>; +} + +const getTerms = (alert: SimpleHit) => { + if (isWrappedRACAlert(alert)) { + return (alert._source[ALERT_RULE_THRESHOLD_FIELD] as string[]).map((field) => ({ + field, + value: alert._source[field] as string, + })); + } else if (isWrappedSignalHit(alert)) { + return alert._source.signal?.threshold_result?.terms ?? []; + } else { + // We shouldn't be here + return []; + } +}; + +const getOriginalTime = (alert: SimpleHit) => { + if (isWrappedRACAlert(alert)) { + const originalTime = alert._source[ALERT_ORIGINAL_TIME]; + return originalTime != null ? new Date(originalTime as string).getTime() : undefined; + } else if (isWrappedSignalHit(alert)) { + const originalTime = alert._source.signal?.original_time; + return originalTime != null ? new Date(originalTime).getTime() : undefined; + } else { + // We shouldn't be here + return undefined; + } +}; + +export const buildThresholdSignalHistory = ({ + alerts, +}: GetThresholdSignalHistoryParams): ThresholdSignalHistory => { + const signalHistory = alerts.reduce((acc, alert) => { + if (!alert._source) { + return acc; + } + + const terms = getTerms(alert as SimpleHit); + const hash = getThresholdTermsHash(terms); + const existing = acc[hash]; + const originalTime = getOriginalTime(alert as SimpleHit); + + if (existing != null) { + if (originalTime && originalTime > existing.lastSignalTimestamp) { + acc[hash].lastSignalTimestamp = originalTime; + } + } else if (originalTime) { + acc[hash] = { + terms, + lastSignalTimestamp: originalTime, + }; + } + return acc; + }, {}); + + return signalHistory; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts index afb0353c4ba0..ce8ee4542d60 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts @@ -46,7 +46,7 @@ interface BulkCreateThresholdSignalsParams { signalsIndex: string; startedAt: Date; from: Date; - thresholdSignalHistory: ThresholdSignalHistory; + signalHistory: ThresholdSignalHistory; bulkCreate: BulkCreate; wrapHits: WrapHits; } @@ -61,7 +61,7 @@ const getTransformedHits = ( ruleId: string, filter: unknown, timestampOverride: TimestampOverrideOrUndefined, - thresholdSignalHistory: ThresholdSignalHistory + signalHistory: ThresholdSignalHistory ) => { const aggParts = threshold.field.length ? results.aggregations && getThresholdAggregationParts(results.aggregations) @@ -148,7 +148,7 @@ const getTransformedHits = ( } const termsHash = getThresholdTermsHash(bucket.terms); - const signalHit = thresholdSignalHistory[termsHash]; + const signalHit = signalHistory[termsHash]; const source = { '@timestamp': timestamp, @@ -202,7 +202,7 @@ export const transformThresholdResultsToEcs = ( threshold: ThresholdNormalized, ruleId: string, timestampOverride: TimestampOverrideOrUndefined, - thresholdSignalHistory: ThresholdSignalHistory + signalHistory: ThresholdSignalHistory ): SignalSearchResponse => { const transformedHits = getTransformedHits( results, @@ -214,7 +214,7 @@ export const transformThresholdResultsToEcs = ( ruleId, filter, timestampOverride, - thresholdSignalHistory + signalHistory ); const thresholdResults = { ...results, @@ -246,7 +246,7 @@ export const bulkCreateThresholdSignals = async ( ruleParams.threshold, ruleParams.ruleId, ruleParams.timestampOverride, - params.thresholdSignalHistory + params.signalHistory ); return params.bulkCreate( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts index e84b4f31fb15..41d46925770b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.test.ts @@ -227,6 +227,7 @@ describe('findThresholdSignals', () => { 'threshold_1:user.name': { terms: { field: 'user.name', + order: { cardinality_count: 'desc' }, min_doc_count: 100, size: 10000, }, @@ -302,6 +303,7 @@ describe('findThresholdSignals', () => { lang: 'painless', }, min_doc_count: 200, + order: { cardinality_count: 'desc' }, }, aggs: { cardinality_count: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts index ca7f22e4a757..740ba281cfcf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/find_threshold_signals.ts @@ -89,6 +89,13 @@ export const findThresholdSignals = async ({ const thresholdFields = threshold.field; + // order buckets by cardinality (https://github.com/elastic/kibana/issues/95258) + const thresholdFieldCount = thresholdFields.length; + const orderByCardinality = (i: number = 0) => + (thresholdFieldCount === 0 || i === thresholdFieldCount - 1) && threshold.cardinality?.length + ? { order: { cardinality_count: 'desc' } } + : {}; + // Generate a nested terms aggregation for each threshold grouping field provided, appending leaf // aggregations to 1) filter out buckets that don't meet the cardinality threshold, if provided, and // 2) return the latest hit for each bucket so that we can persist the timestamp of the event in the @@ -104,6 +111,7 @@ export const findThresholdSignals = async ({ set(acc, aggPath, { terms: { field, + ...orderByCardinality(i), min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set size: 10000, // max 10k buckets }, @@ -121,6 +129,7 @@ export const findThresholdSignals = async ({ source: '""', // Group everything in the same bucket lang: 'painless', }, + ...orderByCardinality(), min_doc_count: threshold.value, }, aggs: leafAggs, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts index d621868a0956..e67a6fa3dfa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.test.ts @@ -11,7 +11,7 @@ import { getThresholdBucketFilters } from './get_threshold_bucket_filters'; describe('getThresholdBucketFilters', () => { it('should generate filters for threshold signal detection with dupe mitigation', async () => { const result = await getThresholdBucketFilters({ - thresholdSignalHistory: sampleThresholdSignalHistory(), + signalHistory: sampleThresholdSignalHistory(), timestampOverride: undefined, }); expect(result).toEqual([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts index e6a188a20b5d..c4569ef9818d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_bucket_filters.ts @@ -9,14 +9,18 @@ import { Filter } from 'src/plugins/data/common'; import { ESFilter } from '../../../../../../../../src/core/types/elasticsearch'; import { ThresholdSignalHistory, ThresholdSignalHistoryRecord } from '../types'; +/* + * Returns a filter to exclude events that have already been included in a + * previous threshold signal. Uses the threshold signal history to achieve this. + */ export const getThresholdBucketFilters = async ({ - thresholdSignalHistory, + signalHistory, timestampOverride, }: { - thresholdSignalHistory: ThresholdSignalHistory; + signalHistory: ThresholdSignalHistory; timestampOverride: string | undefined; }): Promise => { - const filters = Object.values(thresholdSignalHistory).reduce( + const filters = Object.values(signalHistory).reduce( (acc: ESFilter[], bucket: ThresholdSignalHistoryRecord): ESFilter[] => { const filter = { bool: { @@ -24,6 +28,7 @@ export const getThresholdBucketFilters = async ({ { range: { [timestampOverride ?? '@timestamp']: { + // Timestamp of last event signaled on for this set of terms. lte: new Date(bucket.lastSignalTimestamp).toISOString(), }, }, @@ -32,6 +37,7 @@ export const getThresholdBucketFilters = async ({ }, } as ESFilter; + // Terms to filter events older than `lastSignalTimestamp`. bucket.terms.forEach((term) => { if (term.field != null) { (filter.bool!.filter as ESFilter[]).push({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts index b93d8423c025..276431c3bc92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema'; import { TimestampOverrideOrUndefined } from '../../../../../common/detection_engine/schemas/common/schemas'; import { AlertInstanceContext, @@ -16,7 +15,7 @@ import { Logger } from '../../../../../../../../src/core/server'; import { ThresholdSignalHistory } from '../types'; import { BuildRuleMessage } from '../rule_messages'; import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; -import { getThresholdTermsHash } from '../utils'; +import { buildThresholdSignalHistory } from './build_signal_history'; interface GetThresholdSignalHistoryParams { from: string; @@ -41,7 +40,7 @@ export const getThresholdSignalHistory = async ({ timestampOverride, buildRuleMessage, }: GetThresholdSignalHistoryParams): Promise<{ - thresholdSignalHistory: ThresholdSignalHistory; + signalHistory: ThresholdSignalHistory; searchErrors: string[]; }> => { const { searchResult, searchErrors } = await findPreviousThresholdSignals({ @@ -56,51 +55,10 @@ export const getThresholdSignalHistory = async ({ buildRuleMessage, }); - const thresholdSignalHistory = searchResult.hits.hits.reduce( - (acc, hit) => { - if (!hit._source) { - return acc; - } - - const terms = - hit._source.signal?.threshold_result?.terms != null - ? hit._source.signal.threshold_result.terms - : [ - // Pre-7.12 signals - { - field: - (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as { - field: string; - }).field ?? '', - value: ((hit._source.signal?.threshold_result as unknown) as { value: string }) - .value, - }, - ]; - - const hash = getThresholdTermsHash(terms); - const existing = acc[hash]; - const originalTime = - hit._source.signal?.original_time != null - ? new Date(hit._source.signal?.original_time).getTime() - : undefined; - - if (existing != null) { - if (originalTime && originalTime > existing.lastSignalTimestamp) { - acc[hash].lastSignalTimestamp = originalTime; - } - } else if (originalTime) { - acc[hash] = { - terms, - lastSignalTimestamp: originalTime, - }; - } - return acc; - }, - {} - ); - return { - thresholdSignalHistory, + signalHistory: buildThresholdSignalHistory({ + alerts: searchResult.hits.hits, + }), searchErrors, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 3751f6f6e98f..fc6b42c38549 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -362,3 +362,8 @@ export interface ThresholdQueryBucket extends TermAggregationBucket { value_as_string: string; }; } + +export interface ThresholdAlertState extends AlertTypeState { + initialized: boolean; + signalHistory: ThresholdSignalHistory; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index efd7200202b5..5993dd626729 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -63,10 +63,12 @@ import { WrappedRACAlert } from '../rule_types/types'; import { SearchTypes } from '../../../../common/detection_engine/types'; import { IRuleExecutionLogClient } from '../rule_execution_log/types'; import { + EQL_RULE_TYPE_ID, INDICATOR_RULE_TYPE_ID, ML_RULE_TYPE_ID, QUERY_RULE_TYPE_ID, SIGNALS_ID, + THRESHOLD_RULE_TYPE_ID, } from '../../../../common/constants'; interface SortExceptionsReturn { @@ -1013,10 +1015,10 @@ export const getField = (event: SimpleHit, field: string) * Maps legacy rule types to RAC rule type IDs. */ export const ruleTypeMappings = { - eql: SIGNALS_ID, + eql: EQL_RULE_TYPE_ID, machine_learning: ML_RULE_TYPE_ID, query: QUERY_RULE_TYPE_ID, saved_query: SIGNALS_ID, threat_match: INDICATOR_RULE_TYPE_ID, - threshold: SIGNALS_ID, + threshold: THRESHOLD_RULE_TYPE_ID, }; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f07137a118ab..14da8ca65096 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -53,6 +53,7 @@ import { createIndicatorMatchAlertType, createMlAlertType, createQueryAlertType, + createThresholdAlertType, } from './lib/detection_engine/rule_types'; import { initRoutes } from './routes'; import { isAlertExecutor } from './lib/detection_engine/signals/types'; @@ -264,9 +265,10 @@ export class Plugin implements IPlugin