[Security Solution] Add host.os.name.caseless mapping and runtime field (#111455)
* Add host.os.name.caseless field and runtime field * Tests * Only add backwards compatibility mappings to old indices by version * Always update aliases_version field even if there are no compat mappings * Add test for newest index version * More comments Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main
GitHub
parent
5fbc1d4c27
commit
caf5fe3fb6
|
|
@ -1,5 +1,609 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`get_signals_template backwards compatibility mappings for version 45 should match snapshot 1`] = `
|
||||
Object {
|
||||
"_meta": Object {
|
||||
"aliases_version": 1,
|
||||
"version": 45,
|
||||
},
|
||||
"properties": Object {
|
||||
"kibana.alert.ancestors.depth": Object {
|
||||
"path": "signal.ancestors.depth",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.ancestors.id": Object {
|
||||
"path": "signal.ancestors.id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.ancestors.index": Object {
|
||||
"path": "signal.ancestors.index",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.ancestors.type": Object {
|
||||
"path": "signal.ancestors.type",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.depth": Object {
|
||||
"path": "signal.depth",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.action": Object {
|
||||
"path": "signal.original_event.action",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.category": Object {
|
||||
"path": "signal.original_event.category",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.code": Object {
|
||||
"path": "signal.original_event.code",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.created": Object {
|
||||
"path": "signal.original_event.created",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.dataset": Object {
|
||||
"path": "signal.original_event.dataset",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.duration": Object {
|
||||
"path": "signal.original_event.duration",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.end": Object {
|
||||
"path": "signal.original_event.end",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.hash": Object {
|
||||
"path": "signal.original_event.hash",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.id": Object {
|
||||
"path": "signal.original_event.id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.kind": Object {
|
||||
"path": "signal.original_event.kind",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.module": Object {
|
||||
"path": "signal.original_event.module",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.outcome": Object {
|
||||
"path": "signal.original_event.outcome",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.provider": Object {
|
||||
"path": "signal.original_event.provider",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.reason": Object {
|
||||
"path": "signal.original_event.reason",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.risk_score": Object {
|
||||
"path": "signal.original_event.risk_score",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.risk_score_norm": Object {
|
||||
"path": "signal.original_event.risk_score_norm",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.sequence": Object {
|
||||
"path": "signal.original_event.sequence",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.severity": Object {
|
||||
"path": "signal.original_event.severity",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.start": Object {
|
||||
"path": "signal.original_event.start",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.timezone": Object {
|
||||
"path": "signal.original_event.timezone",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_event.type": Object {
|
||||
"path": "signal.original_event.type",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.original_time": Object {
|
||||
"path": "signal.original_time",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.reason": Object {
|
||||
"path": "signal.reason",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.risk_score": Object {
|
||||
"path": "signal.rule.risk_score",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.author": Object {
|
||||
"path": "signal.rule.author",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.building_block_type": Object {
|
||||
"path": "signal.rule.building_block_type",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.created_at": Object {
|
||||
"path": "signal.rule.created_at",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.created_by": Object {
|
||||
"path": "signal.rule.created_by",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.description": Object {
|
||||
"path": "signal.rule.description",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.enabled": Object {
|
||||
"path": "signal.rule.enabled",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.false_positives": Object {
|
||||
"path": "signal.rule.false_positives",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.from": Object {
|
||||
"path": "signal.rule.from",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.immutable": Object {
|
||||
"path": "signal.rule.immutable",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.index": Object {
|
||||
"path": "signal.rule.index",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.interval": Object {
|
||||
"path": "signal.rule.interval",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.language": Object {
|
||||
"path": "signal.rule.language",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.license": Object {
|
||||
"path": "signal.rule.license",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.max_signals": Object {
|
||||
"path": "signal.rule.max_signals",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.name": Object {
|
||||
"path": "signal.rule.name",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.note": Object {
|
||||
"path": "signal.rule.note",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.query": Object {
|
||||
"path": "signal.rule.query",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.references": Object {
|
||||
"path": "signal.rule.references",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.risk_score_mapping.field": Object {
|
||||
"path": "signal.rule.risk_score_mapping.field",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.risk_score_mapping.operator": Object {
|
||||
"path": "signal.rule.risk_score_mapping.operator",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.risk_score_mapping.value": Object {
|
||||
"path": "signal.rule.risk_score_mapping.value",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.rule_id": Object {
|
||||
"path": "signal.rule.rule_id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.rule_name_override": Object {
|
||||
"path": "signal.rule.rule_name_override",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.saved_id": Object {
|
||||
"path": "signal.rule.saved_id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.severity_mapping.field": Object {
|
||||
"path": "signal.rule.severity_mapping.field",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.severity_mapping.operator": Object {
|
||||
"path": "signal.rule.severity_mapping.operator",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.severity_mapping.severity": Object {
|
||||
"path": "signal.rule.severity_mapping.severity",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.severity_mapping.value": Object {
|
||||
"path": "signal.rule.severity_mapping.value",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.tags": Object {
|
||||
"path": "signal.rule.tags",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.framework": Object {
|
||||
"path": "signal.rule.threat.framework",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.tactic.id": Object {
|
||||
"path": "signal.rule.threat.tactic.id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.tactic.name": Object {
|
||||
"path": "signal.rule.threat.tactic.name",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.tactic.reference": Object {
|
||||
"path": "signal.rule.threat.tactic.reference",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.technique.id": Object {
|
||||
"path": "signal.rule.threat.technique.id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.technique.name": Object {
|
||||
"path": "signal.rule.threat.technique.name",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.technique.reference": Object {
|
||||
"path": "signal.rule.threat.technique.reference",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.technique.subtechnique.id": Object {
|
||||
"path": "signal.rule.threat.technique.subtechnique.id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.technique.subtechnique.name": Object {
|
||||
"path": "signal.rule.threat.technique.subtechnique.name",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat.technique.subtechnique.reference": Object {
|
||||
"path": "signal.rule.threat.technique.subtechnique.reference",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_index": Object {
|
||||
"path": "signal.rule.threat_index",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_indicator_path": Object {
|
||||
"path": "signal.rule.threat_indicator_path",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_language": Object {
|
||||
"path": "signal.rule.threat_language",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_mapping.entries.field": Object {
|
||||
"path": "signal.rule.threat_mapping.entries.field",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_mapping.entries.type": Object {
|
||||
"path": "signal.rule.threat_mapping.entries.type",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_mapping.entries.value": Object {
|
||||
"path": "signal.rule.threat_mapping.entries.value",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threat_query": Object {
|
||||
"path": "signal.rule.threat_query",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threshold.field": Object {
|
||||
"path": "signal.rule.threshold.field",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.threshold.value": Object {
|
||||
"path": "signal.rule.threshold.value",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.timeline_id": Object {
|
||||
"path": "signal.rule.timeline_id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.timeline_title": Object {
|
||||
"path": "signal.rule.timeline_title",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.to": Object {
|
||||
"path": "signal.rule.to",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.type": Object {
|
||||
"path": "signal.rule.type",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.updated_at": Object {
|
||||
"path": "signal.rule.updated_at",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.updated_by": Object {
|
||||
"path": "signal.rule.updated_by",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.uuid": Object {
|
||||
"path": "signal.rule.id",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.rule.version": Object {
|
||||
"path": "signal.rule.version",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.severity": Object {
|
||||
"path": "signal.rule.severity",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.threshold_result.cardinality.field": Object {
|
||||
"path": "signal.threshold_result.cardinality.field",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.threshold_result.cardinality.value": Object {
|
||||
"path": "signal.threshold_result.cardinality.value",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.threshold_result.count": Object {
|
||||
"path": "signal.threshold_result.count",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.threshold_result.from": Object {
|
||||
"path": "signal.threshold_result.from",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.threshold_result.terms.field": Object {
|
||||
"path": "signal.threshold_result.terms.field",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.threshold_result.terms.value": Object {
|
||||
"path": "signal.threshold_result.terms.value",
|
||||
"type": "alias",
|
||||
},
|
||||
"kibana.alert.workflow_status": Object {
|
||||
"path": "signal.status",
|
||||
"type": "alias",
|
||||
},
|
||||
"signal": Object {
|
||||
"properties": Object {
|
||||
"_meta": Object {
|
||||
"properties": Object {
|
||||
"version": Object {
|
||||
"type": "long",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"ancestors": Object {
|
||||
"properties": Object {
|
||||
"depth": Object {
|
||||
"type": "long",
|
||||
},
|
||||
"id": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"index": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"rule": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"type": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
},
|
||||
"depth": Object {
|
||||
"type": "integer",
|
||||
},
|
||||
"group": Object {
|
||||
"properties": Object {
|
||||
"id": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"index": Object {
|
||||
"type": "integer",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"original_event": Object {
|
||||
"properties": Object {
|
||||
"reason": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"reason": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"rule": Object {
|
||||
"properties": Object {
|
||||
"author": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"building_block_type": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"license": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"note": Object {
|
||||
"type": "text",
|
||||
},
|
||||
"risk_score_mapping": Object {
|
||||
"properties": Object {
|
||||
"field": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"operator": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"value": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"rule_name_override": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"severity_mapping": Object {
|
||||
"properties": Object {
|
||||
"field": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"operator": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"severity": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"value": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"threat": Object {
|
||||
"properties": Object {
|
||||
"technique": Object {
|
||||
"properties": Object {
|
||||
"subtechnique": Object {
|
||||
"properties": Object {
|
||||
"id": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"name": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"reference": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"threat_index": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"threat_indicator_path": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"threat_language": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"threat_mapping": Object {
|
||||
"properties": Object {
|
||||
"entries": Object {
|
||||
"properties": Object {
|
||||
"field": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"type": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"value": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"threat_query": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"threshold": Object {
|
||||
"properties": Object {
|
||||
"field": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"value": Object {
|
||||
"type": "float",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
"threshold_result": Object {
|
||||
"properties": Object {
|
||||
"cardinality": Object {
|
||||
"properties": Object {
|
||||
"field": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"value": Object {
|
||||
"type": "long",
|
||||
},
|
||||
},
|
||||
},
|
||||
"count": Object {
|
||||
"type": "long",
|
||||
},
|
||||
"from": Object {
|
||||
"type": "date",
|
||||
},
|
||||
"terms": Object {
|
||||
"properties": Object {
|
||||
"field": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
"value": Object {
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"type": "object",
|
||||
},
|
||||
},
|
||||
"runtime": Object {
|
||||
"host.os.name.caseless": Object {
|
||||
"script": Object {
|
||||
"source": "if(doc['host.os.name'].size()!=0) emit(doc['host.os.name'].value.toLowerCase());",
|
||||
},
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`get_signals_template backwards compatibility mappings for version 57 should match snapshot 1`] = `
|
||||
Object {
|
||||
"_meta": Object {
|
||||
"aliases_version": 1,
|
||||
"version": 57,
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`get_signals_template it should match snapshot 1`] = `
|
||||
Object {
|
||||
"index_patterns": Array [
|
||||
|
|
@ -1495,6 +2099,11 @@ Object {
|
|||
},
|
||||
"name": Object {
|
||||
"fields": Object {
|
||||
"caseless": Object {
|
||||
"ignore_above": 1024,
|
||||
"normalizer": "lowercase",
|
||||
"type": "keyword",
|
||||
},
|
||||
"text": Object {
|
||||
"norms": false,
|
||||
"type": "text",
|
||||
|
|
|
|||
|
|
@ -23,11 +23,9 @@ import type {
|
|||
import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants';
|
||||
import { buildSiemResponse } from '../utils';
|
||||
import {
|
||||
createSignalsFieldAliases,
|
||||
getSignalsTemplate,
|
||||
SIGNALS_TEMPLATE_VERSION,
|
||||
SIGNALS_FIELD_ALIASES_VERSION,
|
||||
ALIAS_VERSION_FIELD,
|
||||
createBackwardsCompatibilityMapping,
|
||||
} from './get_signals_template';
|
||||
import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup';
|
||||
import signalsPolicy from './signals_policy.json';
|
||||
|
|
@ -35,7 +33,6 @@ import { templateNeedsUpdate } from './check_template_version';
|
|||
import { getIndexVersion } from './get_index_version';
|
||||
import { isOutdated } from '../../migrations/helpers';
|
||||
import { RuleDataPluginService } from '../../../../../../rule_registry/server';
|
||||
import signalExtraFields from './signal_extra_fields.json';
|
||||
import { ConfigType } from '../../../../config';
|
||||
import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
|
||||
|
||||
|
|
@ -126,7 +123,7 @@ export const createDetectionIndex = async (
|
|||
}
|
||||
|
||||
if (indexExists) {
|
||||
await addFieldAliasesToIndices({ esClient, index, spaceId });
|
||||
await addFieldAliasesToIndices({ esClient, index });
|
||||
// The internal user is used here because Elasticsearch requires the PUT alias requestor to have 'manage' permissions
|
||||
// for BOTH the index AND alias name. However, through 7.14 admins only needed permissions for .siem-signals (the index)
|
||||
// and not .alerts-security.alerts (the alias). From the security solution perspective, all .siem-signals-<space id>-*
|
||||
|
|
@ -148,33 +145,17 @@ export const createDetectionIndex = async (
|
|||
const addFieldAliasesToIndices = async ({
|
||||
esClient,
|
||||
index,
|
||||
spaceId,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
index: string;
|
||||
spaceId: string;
|
||||
}) => {
|
||||
const { body: indexMappings } = await esClient.indices.get({ index });
|
||||
// Make sure that all signal fields we add aliases for are guaranteed to exist in the mapping for ALL historical
|
||||
// signals indices (either by adding them to signalExtraFields or ensuring they exist in the original signals
|
||||
// mapping) or else this call will fail and not update ANY signals indices
|
||||
const fieldAliases = createSignalsFieldAliases();
|
||||
for (const [indexName, mapping] of Object.entries(indexMappings)) {
|
||||
const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version');
|
||||
const newMapping = {
|
||||
properties: {
|
||||
...signalExtraFields,
|
||||
...fieldAliases,
|
||||
// ...getRbacRequiredFields(spaceId),
|
||||
},
|
||||
_meta: {
|
||||
version: currentVersion,
|
||||
[ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION,
|
||||
},
|
||||
};
|
||||
const body = createBackwardsCompatibilityMapping(currentVersion ?? 0);
|
||||
await esClient.indices.putMapping({
|
||||
index: indexName,
|
||||
body: newMapping,
|
||||
body,
|
||||
allow_no_indices: true,
|
||||
} as estypes.IndicesPutMappingRequest);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { getSignalsTemplate } from './get_signals_template';
|
||||
import { createBackwardsCompatibilityMapping, getSignalsTemplate } from './get_signals_template';
|
||||
|
||||
describe('get_signals_template', () => {
|
||||
test('it should set the lifecycle "name" and "rollover_alias" to be the name of the index passed in', () => {
|
||||
|
|
@ -124,4 +124,14 @@ describe('get_signals_template', () => {
|
|||
);
|
||||
expect(template).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('backwards compatibility mappings for version 45 should match snapshot', () => {
|
||||
const mapping = createBackwardsCompatibilityMapping(45);
|
||||
expect(mapping).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('backwards compatibility mappings for version 57 should match snapshot', () => {
|
||||
const mapping = createBackwardsCompatibilityMapping(57);
|
||||
expect(mapping).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ import {
|
|||
ALERT_RULE_PRODUCER,
|
||||
ALERT_RULE_TYPE_ID,
|
||||
} from '@kbn/rule-data-utils';
|
||||
import { merge } from 'lodash';
|
||||
import signalsMapping from './signals_mapping.json';
|
||||
import ecsMapping from './ecs_mapping.json';
|
||||
import otherMapping from './other_mappings.json';
|
||||
import aadFieldConversion from './signal_aad_mapping.json';
|
||||
import signalExtraFields from './signal_extra_fields.json';
|
||||
|
||||
/**
|
||||
@constant
|
||||
|
|
@ -22,7 +24,9 @@ import aadFieldConversion from './signal_aad_mapping.json';
|
|||
@description This value represents the template version assumed by app code.
|
||||
If this number is greater than the user's signals index version, the
|
||||
detections UI will attempt to update the signals template and roll over to
|
||||
a new signals index.
|
||||
a new signals index.
|
||||
|
||||
Since we create a new index for new versions, this version on an existing index should never change.
|
||||
|
||||
If making mappings changes in a patch release, this number should be incremented by 1.
|
||||
If making mappings changes in a minor release, this number should be
|
||||
|
|
@ -34,12 +38,24 @@ export const SIGNALS_TEMPLATE_VERSION = 57;
|
|||
@constant
|
||||
@type {number}
|
||||
@description This value represents the version of the field aliases that map the new field names
|
||||
used for alerts-as-data to the old signal.* field names. If any .siem-signals-<space id> indices
|
||||
have an aliases_version less than this value, the detections UI will call create_index_route and
|
||||
and go through the index update process. Increment this number if making changes to the field
|
||||
aliases we use to make signals forwards-compatible.
|
||||
used for alerts-as-data to the old signal.* field names and any other runtime fields that are added
|
||||
to .siem-signals indices for compatibility reasons (e.g. host.os.name.caseless).
|
||||
|
||||
This version number can change over time on existing indices as we add backwards compatibility fields.
|
||||
|
||||
If any .siem-signals-<space id> indices have an aliases_version less than this value, the detections
|
||||
UI will call create_index_route and and go through the index update process. Increment this number if
|
||||
making changes to the field aliases we use to make signals forwards-compatible.
|
||||
*/
|
||||
export const SIGNALS_FIELD_ALIASES_VERSION = 1;
|
||||
|
||||
/**
|
||||
@constant
|
||||
@type {number}
|
||||
@description This value represents the minimum required index version (SIGNALS_TEMPLATE_VERSION) for EQL
|
||||
rules to write signals correctly. If the write index has a `version` less than this value, the EQL rule
|
||||
will throw an error on execution.
|
||||
*/
|
||||
export const MIN_EQL_RULE_INDEX_VERSION = 2;
|
||||
export const ALIAS_VERSION_FIELD = 'aliases_version';
|
||||
|
||||
|
|
@ -68,13 +84,12 @@ export const getSignalsTemplate = (index: string, spaceId: string, aadIndexAlias
|
|||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
...ecsMapping.mappings.properties,
|
||||
...otherMapping.mappings.properties,
|
||||
...fieldAliases,
|
||||
// ...getRbacRequiredFields(spaceId),
|
||||
signal: signalsMapping.mappings.properties.signal,
|
||||
},
|
||||
properties: merge(
|
||||
ecsMapping.mappings.properties,
|
||||
otherMapping.mappings.properties,
|
||||
fieldAliases,
|
||||
signalsMapping.mappings.properties
|
||||
),
|
||||
_meta: {
|
||||
version: SIGNALS_TEMPLATE_VERSION,
|
||||
[ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION,
|
||||
|
|
@ -97,6 +112,47 @@ export const createSignalsFieldAliases = () => {
|
|||
return fieldAliases;
|
||||
};
|
||||
|
||||
export const backwardsCompatibilityMappings = [
|
||||
{
|
||||
minVersion: 0,
|
||||
// Version 45 shipped with 7.14
|
||||
maxVersion: 45,
|
||||
mapping: {
|
||||
runtime: {
|
||||
'host.os.name.caseless': {
|
||||
type: 'keyword',
|
||||
script: {
|
||||
source:
|
||||
"if(doc['host.os.name'].size()!=0) emit(doc['host.os.name'].value.toLowerCase());",
|
||||
},
|
||||
},
|
||||
},
|
||||
properties: {
|
||||
// signalExtraFields contains the field mappings that have been added to the signals indices over time.
|
||||
// We need to include these here because we can't add an alias for a field that isn't in the mapping,
|
||||
// and we want to apply the aliases to all old signals indices at the same time.
|
||||
...signalExtraFields,
|
||||
...createSignalsFieldAliases(),
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export const createBackwardsCompatibilityMapping = (version: number) => {
|
||||
const mappings = backwardsCompatibilityMappings
|
||||
.filter((mapping) => version <= mapping.maxVersion && version >= mapping.minVersion)
|
||||
.map((mapping) => mapping.mapping);
|
||||
|
||||
const meta = {
|
||||
_meta: {
|
||||
version,
|
||||
[ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION,
|
||||
},
|
||||
};
|
||||
|
||||
return merge({}, ...mappings, meta);
|
||||
};
|
||||
|
||||
export const getRbacRequiredFields = (spaceId: string) => {
|
||||
return {
|
||||
[SPACE_IDS]: {
|
||||
|
|
|
|||
|
|
@ -98,6 +98,29 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"properties": {
|
||||
"os": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"fields": {
|
||||
"text": {
|
||||
"norms": false,
|
||||
"type": "text"
|
||||
},
|
||||
"caseless": {
|
||||
"ignore_above": 1024,
|
||||
"normalizer": "lowercase",
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"interface": {
|
||||
"properties": {
|
||||
"alias": {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import { getSignalStatus, createSignalsIndex, deleteSignalsIndex } from '../../u
|
|||
// eslint-disable-next-line import/no-default-export
|
||||
export default ({ getService }: FtrProviderContext) => {
|
||||
const supertest = getService('supertest');
|
||||
const esArchiver = getService('esArchiver');
|
||||
|
||||
describe('query_signals_route and find_alerts_route', () => {
|
||||
describe('validation checks', () => {
|
||||
|
|
@ -61,6 +62,49 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('backwards compatibility', () => {
|
||||
before(async () => {
|
||||
await esArchiver.load('x-pack/test/functional/es_archives/endpoint/resolver/signals');
|
||||
await createSignalsIndex(supertest);
|
||||
});
|
||||
after(async () => {
|
||||
await esArchiver.unload('x-pack/test/functional/es_archives/endpoint/resolver/signals');
|
||||
await deleteSignalsIndex(supertest);
|
||||
});
|
||||
|
||||
it('should be able to filter old signals on host.os.name.caseless using runtime field', async () => {
|
||||
const query = {
|
||||
query: {
|
||||
bool: {
|
||||
should: [{ match_phrase: { 'host.os.name.caseless': 'windows' } }],
|
||||
},
|
||||
},
|
||||
};
|
||||
const { body } = await supertest
|
||||
.post(DETECTION_ENGINE_QUERY_SIGNALS_URL)
|
||||
.set('kbn-xsrf', 'true')
|
||||
.send(query)
|
||||
.expect(200);
|
||||
expect(body.hits.total.value).to.eql(3);
|
||||
});
|
||||
|
||||
it('should be able to filter old signals using field aliases', async () => {
|
||||
const query = {
|
||||
query: {
|
||||
bool: {
|
||||
should: [{ match_phrase: { 'kibana.alert.workflow_status': 'open' } }],
|
||||
},
|
||||
},
|
||||
};
|
||||
const { body } = await supertest
|
||||
.post(DETECTION_ENGINE_QUERY_SIGNALS_URL)
|
||||
.set('kbn-xsrf', 'true')
|
||||
.send(query)
|
||||
.expect(200);
|
||||
expect(body.hits.total.value).to.eql(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('find_alerts_route', () => {
|
||||
describe('validation checks', () => {
|
||||
it('should not give errors when querying and the signals index does not exist yet', async () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue