From d280f1275715092d9811d1d97063d1223d285026 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Thu, 28 Oct 2021 07:21:51 -0700 Subject: [PATCH] [Security Solution] Security RAC migration follow up bug fixes (#116386) * Add compatibility aliases to alerts as data indices * Fix dupe mitigation, allow more fields in mapping * Remove legacy signals fields from new RAC alerts * Fix cypress test * Remove outdated comment * Reduce flakiness in time based test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../field_maps/technical_rule_field_map.ts | 5 +++++ .../rule_registry/common/field_map/types.ts | 2 +- .../resource_installer.ts | 1 + .../create_persistence_rule_type_wrapper.ts | 2 +- .../detection_alerts/alerts_details.spec.ts | 3 ++- .../create_security_rule_type_wrapper.ts | 6 ++++-- .../rule_types/factories/wrap_hits_factory.ts | 2 +- .../rule_types/field_maps/alerts.ts | 5 +++++ .../rule_types/field_maps/rules.ts | 6 +++--- .../signals/threat_mapping/utils.test.ts | 2 +- .../plugins/security_solution/server/plugin.ts | 16 ++++++++-------- 11 files changed, 32 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts index 1c59e56c0466..bd95e7e94252 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts @@ -50,6 +50,11 @@ export const technicalRuleFieldMap = { array: false, required: false, }, + [Fields.ALERT_RISK_SCORE]: { + type: 'float', + array: false, + required: false, + }, [Fields.ALERT_WORKFLOW_STATUS]: { type: 'keyword', array: false, diff --git a/x-pack/plugins/rule_registry/common/field_map/types.ts b/x-pack/plugins/rule_registry/common/field_map/types.ts index 3ff68315e93a..d4bdd34656ec 100644 --- a/x-pack/plugins/rule_registry/common/field_map/types.ts +++ b/x-pack/plugins/rule_registry/common/field_map/types.ts @@ -6,5 +6,5 @@ */ export interface FieldMap { - [key: string]: { type: string; required?: boolean; array?: boolean }; + [key: string]: { type: string; required?: boolean; array?: boolean; path?: string }; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts index 041dfdeed42e..3798506eeacd 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -315,6 +315,7 @@ export class ResourceInstaller { // @ts-expect-error rollover_alias: primaryNamespacedAlias, }, + 'index.mapping.total_fields.limit': 1100, }, mappings: { dynamic: false, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 7dea0f917247..e575b49d1776 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -30,7 +30,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper .getWriter({ namespace: options.spaceId }) .bulk({ body: alerts.flatMap((alert) => [ - { index: {} }, + { index: { _id: alert.id } }, { [VERSION]: ruleDataClient.kibanaVersion, ...commonRuleFields, diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts index 2cde29ec9da6..803ff4b4d0d8 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts @@ -46,9 +46,10 @@ describe('Alert details with unmapped fields', () => { }); }); + // This test needs to be updated to not look for the field in a specific row, as it prevents us from adding/removing fields it('Displays the unmapped field on the table', () => { const expectedUnmmappedField = { - row: 54, + row: 82, field: 'unmapped', text: 'This is the unmapped field', }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 5f70a5ec20bf..1b6d85540f91 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -37,6 +37,7 @@ import { bulkCreateFactory, wrapHitsFactory, wrapSequencesFactory } from './fact import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; +import aadFieldConversion from '../routes/index/signal_aad_mapping.json'; /* eslint-disable complexity */ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = @@ -225,8 +226,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = refresh ); + const legacySignalFields: string[] = Object.keys(aadFieldConversion); const wrapHits = wrapHitsFactory({ - ignoreFields, + ignoreFields: [...ignoreFields, ...legacySignalFields], mergeStrategy, completeRule, spaceId, @@ -234,7 +236,7 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = const wrapSequences = wrapSequencesFactory({ logger, - ignoreFields, + ignoreFields: [...ignoreFields, ...legacySignalFields], mergeStrategy, completeRule, spaceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index a66703e3a50b..81a4af31881f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -52,5 +52,5 @@ export const wrapHitsFactory = }; }); - return filterDuplicateSignals(completeRule.alertId, wrappedDocs, false); + return filterDuplicateSignals(completeRule.alertId, wrappedDocs, true); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts index 9cc5c63332a5..5a7ceb83baf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/alerts.ts @@ -163,6 +163,11 @@ export const alertsFieldMap: FieldMap = { array: false, required: true, }, + 'kibana.alert.original_event.severity': { + type: 'long', + array: false, + required: false, + }, 'kibana.alert.original_event.start': { type: 'date', array: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts index 87b55e092ec5..a067a36fe039 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/field_maps/rules.ts @@ -126,17 +126,17 @@ export const rulesFieldMap = { array: true, required: false, }, - 'kibana.alert.rule.threat_mapping.field': { + 'kibana.alert.rule.threat_mapping.entries.field': { type: 'keyword', array: true, required: false, }, - 'kibana.alert.rule.threat_mapping.value': { + 'kibana.alert.rule.threat_mapping.entries.value': { type: 'keyword', array: true, required: false, }, - 'kibana.alert.rule.threat_mapping.type': { + 'kibana.alert.rule.threat_mapping.entries.type': { type: 'keyword', array: true, required: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index f029b02127b0..ff4fbb58d749 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -723,7 +723,7 @@ describe('utils', () => { it('throws an error if the validator is called after the specified interval', async () => { const validator = buildExecutionIntervalValidator('1s'); - await new Promise((r) => setTimeout(r, 1001)); + await new Promise((r) => setTimeout(r, 2000)); expect(() => validator()).toThrowError( 'Current rule execution has exceeded its allotted interval (1s) and has been stopped.' ); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index b31ec3696fd4..307ccf4cfc97 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -7,7 +7,6 @@ import { Observable } from 'rxjs'; import LRU from 'lru-cache'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { SIGNALS_ID, QUERY_RULE_TYPE_ID, @@ -22,6 +21,8 @@ import { Logger, SavedObjectsClient } from '../../../../src/core/server'; import { UsageCounter } from '../../../../src/plugins/usage_collection/server'; import { ECS_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets'; +import { FieldMap } from '../../rule_registry/common/field_map'; +import { technicalRuleFieldMap } from '../../rule_registry/common/assets/field_maps/technical_rule_field_map'; import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; import { IRuleDataClient, Dataset } from '../../rule_registry/server'; import { ListPluginSetup } from '../../lists/server'; @@ -188,13 +189,9 @@ export class Plugin implements ISecuritySolutionPlugin { }; if (isRuleRegistryEnabled) { - // NOTE: this is not used yet - // TODO: convert the aliases to FieldMaps. Requires enhancing FieldMap to support alias path. - // Split aliases by component template since we need to alias some fields in technical field mappings, - // some fields in security solution specific component template. - const aliases: Record = {}; + const aliasesFieldMap: FieldMap = {}; Object.entries(aadFieldConversion).forEach(([key, value]) => { - aliases[key] = { + aliasesFieldMap[key] = { type: 'alias', path: value, }; @@ -208,7 +205,10 @@ export class Plugin implements ISecuritySolutionPlugin { componentTemplates: [ { name: 'mappings', - mappings: mappingFromFieldMap({ ...alertsFieldMap, ...rulesFieldMap }, false), + mappings: mappingFromFieldMap( + { ...technicalRuleFieldMap, ...alertsFieldMap, ...rulesFieldMap, ...aliasesFieldMap }, + false + ), }, ], secondaryAlias: config.signalsIndex,