From caf5fe3fb6740be66c5c9bbab5ddd779eb253d3d Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Fri, 10 Sep 2021 07:48:05 -0700 Subject: [PATCH] [Security Solution] Add host.os.name.caseless mapping and runtime field (#111455) * Add host.os.name.caseless field and runtime field * Tests * Only add backwards compatibility mappings to old indices by version * Always update aliases_version field even if there are no compat mappings * Add test for newest index version * More comments Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../get_signals_template.test.ts.snap | 609 ++++++++++++++++++ .../routes/index/create_index_route.ts | 27 +- .../routes/index/get_signals_template.test.ts | 12 +- .../routes/index/get_signals_template.ts | 80 ++- .../routes/index/other_mappings.json | 23 + .../basic/tests/query_signals.ts | 44 ++ 6 files changed, 759 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap index 3c065ab0ac10..1d4e84ea5dcc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap @@ -1,5 +1,609 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`get_signals_template backwards compatibility mappings for version 45 should match snapshot 1`] = ` +Object { + "_meta": Object { + "aliases_version": 1, + "version": 45, + }, + "properties": Object { + "kibana.alert.ancestors.depth": Object { + "path": "signal.ancestors.depth", + "type": "alias", + }, + "kibana.alert.ancestors.id": Object { + "path": "signal.ancestors.id", + "type": "alias", + }, + "kibana.alert.ancestors.index": Object { + "path": "signal.ancestors.index", + "type": "alias", + }, + "kibana.alert.ancestors.type": Object { + "path": "signal.ancestors.type", + "type": "alias", + }, + "kibana.alert.depth": Object { + "path": "signal.depth", + "type": "alias", + }, + "kibana.alert.original_event.action": Object { + "path": "signal.original_event.action", + "type": "alias", + }, + "kibana.alert.original_event.category": Object { + "path": "signal.original_event.category", + "type": "alias", + }, + "kibana.alert.original_event.code": Object { + "path": "signal.original_event.code", + "type": "alias", + }, + "kibana.alert.original_event.created": Object { + "path": "signal.original_event.created", + "type": "alias", + }, + "kibana.alert.original_event.dataset": Object { + "path": "signal.original_event.dataset", + "type": "alias", + }, + "kibana.alert.original_event.duration": Object { + "path": "signal.original_event.duration", + "type": "alias", + }, + "kibana.alert.original_event.end": Object { + "path": "signal.original_event.end", + "type": "alias", + }, + "kibana.alert.original_event.hash": Object { + "path": "signal.original_event.hash", + "type": "alias", + }, + "kibana.alert.original_event.id": Object { + "path": "signal.original_event.id", + "type": "alias", + }, + "kibana.alert.original_event.kind": Object { + "path": "signal.original_event.kind", + "type": "alias", + }, + "kibana.alert.original_event.module": Object { + "path": "signal.original_event.module", + "type": "alias", + }, + "kibana.alert.original_event.outcome": Object { + "path": "signal.original_event.outcome", + "type": "alias", + }, + "kibana.alert.original_event.provider": Object { + "path": "signal.original_event.provider", + "type": "alias", + }, + "kibana.alert.original_event.reason": Object { + "path": "signal.original_event.reason", + "type": "alias", + }, + "kibana.alert.original_event.risk_score": Object { + "path": "signal.original_event.risk_score", + "type": "alias", + }, + "kibana.alert.original_event.risk_score_norm": Object { + "path": "signal.original_event.risk_score_norm", + "type": "alias", + }, + "kibana.alert.original_event.sequence": Object { + "path": "signal.original_event.sequence", + "type": "alias", + }, + "kibana.alert.original_event.severity": Object { + "path": "signal.original_event.severity", + "type": "alias", + }, + "kibana.alert.original_event.start": Object { + "path": "signal.original_event.start", + "type": "alias", + }, + "kibana.alert.original_event.timezone": Object { + "path": "signal.original_event.timezone", + "type": "alias", + }, + "kibana.alert.original_event.type": Object { + "path": "signal.original_event.type", + "type": "alias", + }, + "kibana.alert.original_time": Object { + "path": "signal.original_time", + "type": "alias", + }, + "kibana.alert.reason": Object { + "path": "signal.reason", + "type": "alias", + }, + "kibana.alert.risk_score": Object { + "path": "signal.rule.risk_score", + "type": "alias", + }, + "kibana.alert.rule.author": Object { + "path": "signal.rule.author", + "type": "alias", + }, + "kibana.alert.rule.building_block_type": Object { + "path": "signal.rule.building_block_type", + "type": "alias", + }, + "kibana.alert.rule.created_at": Object { + "path": "signal.rule.created_at", + "type": "alias", + }, + "kibana.alert.rule.created_by": Object { + "path": "signal.rule.created_by", + "type": "alias", + }, + "kibana.alert.rule.description": Object { + "path": "signal.rule.description", + "type": "alias", + }, + "kibana.alert.rule.enabled": Object { + "path": "signal.rule.enabled", + "type": "alias", + }, + "kibana.alert.rule.false_positives": Object { + "path": "signal.rule.false_positives", + "type": "alias", + }, + "kibana.alert.rule.from": Object { + "path": "signal.rule.from", + "type": "alias", + }, + "kibana.alert.rule.immutable": Object { + "path": "signal.rule.immutable", + "type": "alias", + }, + "kibana.alert.rule.index": Object { + "path": "signal.rule.index", + "type": "alias", + }, + "kibana.alert.rule.interval": Object { + "path": "signal.rule.interval", + "type": "alias", + }, + "kibana.alert.rule.language": Object { + "path": "signal.rule.language", + "type": "alias", + }, + "kibana.alert.rule.license": Object { + "path": "signal.rule.license", + "type": "alias", + }, + "kibana.alert.rule.max_signals": Object { + "path": "signal.rule.max_signals", + "type": "alias", + }, + "kibana.alert.rule.name": Object { + "path": "signal.rule.name", + "type": "alias", + }, + "kibana.alert.rule.note": Object { + "path": "signal.rule.note", + "type": "alias", + }, + "kibana.alert.rule.query": Object { + "path": "signal.rule.query", + "type": "alias", + }, + "kibana.alert.rule.references": Object { + "path": "signal.rule.references", + "type": "alias", + }, + "kibana.alert.rule.risk_score_mapping.field": Object { + "path": "signal.rule.risk_score_mapping.field", + "type": "alias", + }, + "kibana.alert.rule.risk_score_mapping.operator": Object { + "path": "signal.rule.risk_score_mapping.operator", + "type": "alias", + }, + "kibana.alert.rule.risk_score_mapping.value": Object { + "path": "signal.rule.risk_score_mapping.value", + "type": "alias", + }, + "kibana.alert.rule.rule_id": Object { + "path": "signal.rule.rule_id", + "type": "alias", + }, + "kibana.alert.rule.rule_name_override": Object { + "path": "signal.rule.rule_name_override", + "type": "alias", + }, + "kibana.alert.rule.saved_id": Object { + "path": "signal.rule.saved_id", + "type": "alias", + }, + "kibana.alert.rule.severity_mapping.field": Object { + "path": "signal.rule.severity_mapping.field", + "type": "alias", + }, + "kibana.alert.rule.severity_mapping.operator": Object { + "path": "signal.rule.severity_mapping.operator", + "type": "alias", + }, + "kibana.alert.rule.severity_mapping.severity": Object { + "path": "signal.rule.severity_mapping.severity", + "type": "alias", + }, + "kibana.alert.rule.severity_mapping.value": Object { + "path": "signal.rule.severity_mapping.value", + "type": "alias", + }, + "kibana.alert.rule.tags": Object { + "path": "signal.rule.tags", + "type": "alias", + }, + "kibana.alert.rule.threat.framework": Object { + "path": "signal.rule.threat.framework", + "type": "alias", + }, + "kibana.alert.rule.threat.tactic.id": Object { + "path": "signal.rule.threat.tactic.id", + "type": "alias", + }, + "kibana.alert.rule.threat.tactic.name": Object { + "path": "signal.rule.threat.tactic.name", + "type": "alias", + }, + "kibana.alert.rule.threat.tactic.reference": Object { + "path": "signal.rule.threat.tactic.reference", + "type": "alias", + }, + "kibana.alert.rule.threat.technique.id": Object { + "path": "signal.rule.threat.technique.id", + "type": "alias", + }, + "kibana.alert.rule.threat.technique.name": Object { + "path": "signal.rule.threat.technique.name", + "type": "alias", + }, + "kibana.alert.rule.threat.technique.reference": Object { + "path": "signal.rule.threat.technique.reference", + "type": "alias", + }, + "kibana.alert.rule.threat.technique.subtechnique.id": Object { + "path": "signal.rule.threat.technique.subtechnique.id", + "type": "alias", + }, + "kibana.alert.rule.threat.technique.subtechnique.name": Object { + "path": "signal.rule.threat.technique.subtechnique.name", + "type": "alias", + }, + "kibana.alert.rule.threat.technique.subtechnique.reference": Object { + "path": "signal.rule.threat.technique.subtechnique.reference", + "type": "alias", + }, + "kibana.alert.rule.threat_index": Object { + "path": "signal.rule.threat_index", + "type": "alias", + }, + "kibana.alert.rule.threat_indicator_path": Object { + "path": "signal.rule.threat_indicator_path", + "type": "alias", + }, + "kibana.alert.rule.threat_language": Object { + "path": "signal.rule.threat_language", + "type": "alias", + }, + "kibana.alert.rule.threat_mapping.entries.field": Object { + "path": "signal.rule.threat_mapping.entries.field", + "type": "alias", + }, + "kibana.alert.rule.threat_mapping.entries.type": Object { + "path": "signal.rule.threat_mapping.entries.type", + "type": "alias", + }, + "kibana.alert.rule.threat_mapping.entries.value": Object { + "path": "signal.rule.threat_mapping.entries.value", + "type": "alias", + }, + "kibana.alert.rule.threat_query": Object { + "path": "signal.rule.threat_query", + "type": "alias", + }, + "kibana.alert.rule.threshold.field": Object { + "path": "signal.rule.threshold.field", + "type": "alias", + }, + "kibana.alert.rule.threshold.value": Object { + "path": "signal.rule.threshold.value", + "type": "alias", + }, + "kibana.alert.rule.timeline_id": Object { + "path": "signal.rule.timeline_id", + "type": "alias", + }, + "kibana.alert.rule.timeline_title": Object { + "path": "signal.rule.timeline_title", + "type": "alias", + }, + "kibana.alert.rule.to": Object { + "path": "signal.rule.to", + "type": "alias", + }, + "kibana.alert.rule.type": Object { + "path": "signal.rule.type", + "type": "alias", + }, + "kibana.alert.rule.updated_at": Object { + "path": "signal.rule.updated_at", + "type": "alias", + }, + "kibana.alert.rule.updated_by": Object { + "path": "signal.rule.updated_by", + "type": "alias", + }, + "kibana.alert.rule.uuid": Object { + "path": "signal.rule.id", + "type": "alias", + }, + "kibana.alert.rule.version": Object { + "path": "signal.rule.version", + "type": "alias", + }, + "kibana.alert.severity": Object { + "path": "signal.rule.severity", + "type": "alias", + }, + "kibana.alert.threshold_result.cardinality.field": Object { + "path": "signal.threshold_result.cardinality.field", + "type": "alias", + }, + "kibana.alert.threshold_result.cardinality.value": Object { + "path": "signal.threshold_result.cardinality.value", + "type": "alias", + }, + "kibana.alert.threshold_result.count": Object { + "path": "signal.threshold_result.count", + "type": "alias", + }, + "kibana.alert.threshold_result.from": Object { + "path": "signal.threshold_result.from", + "type": "alias", + }, + "kibana.alert.threshold_result.terms.field": Object { + "path": "signal.threshold_result.terms.field", + "type": "alias", + }, + "kibana.alert.threshold_result.terms.value": Object { + "path": "signal.threshold_result.terms.value", + "type": "alias", + }, + "kibana.alert.workflow_status": Object { + "path": "signal.status", + "type": "alias", + }, + "signal": Object { + "properties": Object { + "_meta": Object { + "properties": Object { + "version": Object { + "type": "long", + }, + }, + "type": "object", + }, + "ancestors": Object { + "properties": Object { + "depth": Object { + "type": "long", + }, + "id": Object { + "type": "keyword", + }, + "index": Object { + "type": "keyword", + }, + "rule": Object { + "type": "keyword", + }, + "type": Object { + "type": "keyword", + }, + }, + }, + "depth": Object { + "type": "integer", + }, + "group": Object { + "properties": Object { + "id": Object { + "type": "keyword", + }, + "index": Object { + "type": "integer", + }, + }, + "type": "object", + }, + "original_event": Object { + "properties": Object { + "reason": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + "reason": Object { + "type": "keyword", + }, + "rule": Object { + "properties": Object { + "author": Object { + "type": "keyword", + }, + "building_block_type": Object { + "type": "keyword", + }, + "license": Object { + "type": "keyword", + }, + "note": Object { + "type": "text", + }, + "risk_score_mapping": Object { + "properties": Object { + "field": Object { + "type": "keyword", + }, + "operator": Object { + "type": "keyword", + }, + "value": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + "rule_name_override": Object { + "type": "keyword", + }, + "severity_mapping": Object { + "properties": Object { + "field": Object { + "type": "keyword", + }, + "operator": Object { + "type": "keyword", + }, + "severity": Object { + "type": "keyword", + }, + "value": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + "threat": Object { + "properties": Object { + "technique": Object { + "properties": Object { + "subtechnique": Object { + "properties": Object { + "id": Object { + "type": "keyword", + }, + "name": Object { + "type": "keyword", + }, + "reference": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + }, + "type": "object", + }, + }, + "type": "object", + }, + "threat_index": Object { + "type": "keyword", + }, + "threat_indicator_path": Object { + "type": "keyword", + }, + "threat_language": Object { + "type": "keyword", + }, + "threat_mapping": Object { + "properties": Object { + "entries": Object { + "properties": Object { + "field": Object { + "type": "keyword", + }, + "type": Object { + "type": "keyword", + }, + "value": Object { + "type": "keyword", + }, + }, + "type": "object", + }, + }, + "type": "object", + }, + "threat_query": Object { + "type": "keyword", + }, + "threshold": Object { + "properties": Object { + "field": Object { + "type": "keyword", + }, + "value": Object { + "type": "float", + }, + }, + "type": "object", + }, + }, + "type": "object", + }, + "threshold_result": Object { + "properties": Object { + "cardinality": Object { + "properties": Object { + "field": Object { + "type": "keyword", + }, + "value": Object { + "type": "long", + }, + }, + }, + "count": Object { + "type": "long", + }, + "from": Object { + "type": "date", + }, + "terms": Object { + "properties": Object { + "field": Object { + "type": "keyword", + }, + "value": Object { + "type": "keyword", + }, + }, + }, + }, + }, + }, + "type": "object", + }, + }, + "runtime": Object { + "host.os.name.caseless": Object { + "script": Object { + "source": "if(doc['host.os.name'].size()!=0) emit(doc['host.os.name'].value.toLowerCase());", + }, + "type": "keyword", + }, + }, +} +`; + +exports[`get_signals_template backwards compatibility mappings for version 57 should match snapshot 1`] = ` +Object { + "_meta": Object { + "aliases_version": 1, + "version": 57, + }, +} +`; + exports[`get_signals_template it should match snapshot 1`] = ` Object { "index_patterns": Array [ @@ -1495,6 +2099,11 @@ Object { }, "name": Object { "fields": Object { + "caseless": Object { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword", + }, "text": Object { "norms": false, "type": "text", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index d65a1ad87b41..61635fdcef9f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -23,11 +23,9 @@ import type { import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; import { buildSiemResponse } from '../utils'; import { - createSignalsFieldAliases, getSignalsTemplate, SIGNALS_TEMPLATE_VERSION, - SIGNALS_FIELD_ALIASES_VERSION, - ALIAS_VERSION_FIELD, + createBackwardsCompatibilityMapping, } from './get_signals_template'; import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup'; import signalsPolicy from './signals_policy.json'; @@ -35,7 +33,6 @@ import { templateNeedsUpdate } from './check_template_version'; import { getIndexVersion } from './get_index_version'; import { isOutdated } from '../../migrations/helpers'; import { RuleDataPluginService } from '../../../../../../rule_registry/server'; -import signalExtraFields from './signal_extra_fields.json'; import { ConfigType } from '../../../../config'; import { parseExperimentalConfigValue } from '../../../../../common/experimental_features'; @@ -126,7 +123,7 @@ export const createDetectionIndex = async ( } if (indexExists) { - await addFieldAliasesToIndices({ esClient, index, spaceId }); + await addFieldAliasesToIndices({ esClient, index }); // The internal user is used here because Elasticsearch requires the PUT alias requestor to have 'manage' permissions // for BOTH the index AND alias name. However, through 7.14 admins only needed permissions for .siem-signals (the index) // and not .alerts-security.alerts (the alias). From the security solution perspective, all .siem-signals--* @@ -148,33 +145,17 @@ export const createDetectionIndex = async ( const addFieldAliasesToIndices = async ({ esClient, index, - spaceId, }: { esClient: ElasticsearchClient; index: string; - spaceId: string; }) => { const { body: indexMappings } = await esClient.indices.get({ index }); - // Make sure that all signal fields we add aliases for are guaranteed to exist in the mapping for ALL historical - // signals indices (either by adding them to signalExtraFields or ensuring they exist in the original signals - // mapping) or else this call will fail and not update ANY signals indices - const fieldAliases = createSignalsFieldAliases(); for (const [indexName, mapping] of Object.entries(indexMappings)) { const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version'); - const newMapping = { - properties: { - ...signalExtraFields, - ...fieldAliases, - // ...getRbacRequiredFields(spaceId), - }, - _meta: { - version: currentVersion, - [ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION, - }, - }; + const body = createBackwardsCompatibilityMapping(currentVersion ?? 0); await esClient.indices.putMapping({ index: indexName, - body: newMapping, + body, allow_no_indices: true, } as estypes.IndicesPutMappingRequest); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts index bb67dd1fca6d..70363cba34fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { getSignalsTemplate } from './get_signals_template'; +import { createBackwardsCompatibilityMapping, getSignalsTemplate } from './get_signals_template'; describe('get_signals_template', () => { test('it should set the lifecycle "name" and "rollover_alias" to be the name of the index passed in', () => { @@ -124,4 +124,14 @@ describe('get_signals_template', () => { ); expect(template).toMatchSnapshot(); }); + + test('backwards compatibility mappings for version 45 should match snapshot', () => { + const mapping = createBackwardsCompatibilityMapping(45); + expect(mapping).toMatchSnapshot(); + }); + + test('backwards compatibility mappings for version 57 should match snapshot', () => { + const mapping = createBackwardsCompatibilityMapping(57); + expect(mapping).toMatchSnapshot(); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index 3470f955dbdb..b7a0521e5c3c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -11,10 +11,12 @@ import { ALERT_RULE_PRODUCER, ALERT_RULE_TYPE_ID, } from '@kbn/rule-data-utils'; +import { merge } from 'lodash'; import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; import otherMapping from './other_mappings.json'; import aadFieldConversion from './signal_aad_mapping.json'; +import signalExtraFields from './signal_extra_fields.json'; /** @constant @@ -22,7 +24,9 @@ import aadFieldConversion from './signal_aad_mapping.json'; @description This value represents the template version assumed by app code. If this number is greater than the user's signals index version, the detections UI will attempt to update the signals template and roll over to - a new signals index. + a new signals index. + + Since we create a new index for new versions, this version on an existing index should never change. If making mappings changes in a patch release, this number should be incremented by 1. If making mappings changes in a minor release, this number should be @@ -34,12 +38,24 @@ export const SIGNALS_TEMPLATE_VERSION = 57; @constant @type {number} @description This value represents the version of the field aliases that map the new field names - used for alerts-as-data to the old signal.* field names. If any .siem-signals- indices - have an aliases_version less than this value, the detections UI will call create_index_route and - and go through the index update process. Increment this number if making changes to the field - aliases we use to make signals forwards-compatible. + used for alerts-as-data to the old signal.* field names and any other runtime fields that are added + to .siem-signals indices for compatibility reasons (e.g. host.os.name.caseless). + + This version number can change over time on existing indices as we add backwards compatibility fields. + + If any .siem-signals- indices have an aliases_version less than this value, the detections + UI will call create_index_route and and go through the index update process. Increment this number if + making changes to the field aliases we use to make signals forwards-compatible. */ export const SIGNALS_FIELD_ALIASES_VERSION = 1; + +/** + @constant + @type {number} + @description This value represents the minimum required index version (SIGNALS_TEMPLATE_VERSION) for EQL + rules to write signals correctly. If the write index has a `version` less than this value, the EQL rule + will throw an error on execution. +*/ export const MIN_EQL_RULE_INDEX_VERSION = 2; export const ALIAS_VERSION_FIELD = 'aliases_version'; @@ -68,13 +84,12 @@ export const getSignalsTemplate = (index: string, spaceId: string, aadIndexAlias }, mappings: { dynamic: false, - properties: { - ...ecsMapping.mappings.properties, - ...otherMapping.mappings.properties, - ...fieldAliases, - // ...getRbacRequiredFields(spaceId), - signal: signalsMapping.mappings.properties.signal, - }, + properties: merge( + ecsMapping.mappings.properties, + otherMapping.mappings.properties, + fieldAliases, + signalsMapping.mappings.properties + ), _meta: { version: SIGNALS_TEMPLATE_VERSION, [ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION, @@ -97,6 +112,47 @@ export const createSignalsFieldAliases = () => { return fieldAliases; }; +export const backwardsCompatibilityMappings = [ + { + minVersion: 0, + // Version 45 shipped with 7.14 + maxVersion: 45, + mapping: { + runtime: { + 'host.os.name.caseless': { + type: 'keyword', + script: { + source: + "if(doc['host.os.name'].size()!=0) emit(doc['host.os.name'].value.toLowerCase());", + }, + }, + }, + properties: { + // signalExtraFields contains the field mappings that have been added to the signals indices over time. + // We need to include these here because we can't add an alias for a field that isn't in the mapping, + // and we want to apply the aliases to all old signals indices at the same time. + ...signalExtraFields, + ...createSignalsFieldAliases(), + }, + }, + }, +]; + +export const createBackwardsCompatibilityMapping = (version: number) => { + const mappings = backwardsCompatibilityMappings + .filter((mapping) => version <= mapping.maxVersion && version >= mapping.minVersion) + .map((mapping) => mapping.mapping); + + const meta = { + _meta: { + version, + [ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION, + }, + }; + + return merge({}, ...mappings, meta); +}; + export const getRbacRequiredFields = (spaceId: string) => { return { [SPACE_IDS]: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/other_mappings.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/other_mappings.json index b61ad2e43ac0..5ad8f5238a97 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/other_mappings.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/other_mappings.json @@ -98,6 +98,29 @@ } } }, + "host": { + "properties": { + "os": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + }, + "caseless": { + "ignore_above": 1024, + "normalizer": "lowercase", + "type": "keyword" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, "interface": { "properties": { "alias": { diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts b/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts index 969315cb3f98..53225e4ea2ce 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts @@ -17,6 +17,7 @@ import { getSignalStatus, createSignalsIndex, deleteSignalsIndex } from '../../u // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); describe('query_signals_route and find_alerts_route', () => { describe('validation checks', () => { @@ -61,6 +62,49 @@ export default ({ getService }: FtrProviderContext) => { }); }); + describe('backwards compatibility', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/endpoint/resolver/signals'); + await createSignalsIndex(supertest); + }); + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/endpoint/resolver/signals'); + await deleteSignalsIndex(supertest); + }); + + it('should be able to filter old signals on host.os.name.caseless using runtime field', async () => { + const query = { + query: { + bool: { + should: [{ match_phrase: { 'host.os.name.caseless': 'windows' } }], + }, + }, + }; + const { body } = await supertest + .post(DETECTION_ENGINE_QUERY_SIGNALS_URL) + .set('kbn-xsrf', 'true') + .send(query) + .expect(200); + expect(body.hits.total.value).to.eql(3); + }); + + it('should be able to filter old signals using field aliases', async () => { + const query = { + query: { + bool: { + should: [{ match_phrase: { 'kibana.alert.workflow_status': 'open' } }], + }, + }, + }; + const { body } = await supertest + .post(DETECTION_ENGINE_QUERY_SIGNALS_URL) + .set('kbn-xsrf', 'true') + .send(query) + .expect(200); + expect(body.hits.total.value).to.eql(3); + }); + }); + describe('find_alerts_route', () => { describe('validation checks', () => { it('should not give errors when querying and the signals index does not exist yet', async () => {