From f91c795e30b7221cdce36e235470a4cda82f70b2 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Fri, 17 Apr 2020 13:23:30 +0200 Subject: [PATCH] [EPM] Handle multi fields in index template generation (#63112) * Add unit test stub for multi fields. * Add multi field handling to mapping generation. * Start documenting index template generation. * Add unit tests. * Remove stub for fields.yml documentation Co-authored-by: Elastic Machine --- docs/ingest_manager/index-templates.asciidoc | 7 + docs/ingest_manager/index.asciidoc | 7 + .../__snapshots__/template.test.ts.snap | 176 +++++++++--------- .../elasticsearch/template/template.test.ts | 98 ++++++++++ .../epm/elasticsearch/template/template.ts | 69 +++++-- 5 files changed, 256 insertions(+), 101 deletions(-) create mode 100644 docs/ingest_manager/index-templates.asciidoc diff --git a/docs/ingest_manager/index-templates.asciidoc b/docs/ingest_manager/index-templates.asciidoc new file mode 100644 index 000000000000..e19af63c3116 --- /dev/null +++ b/docs/ingest_manager/index-templates.asciidoc @@ -0,0 +1,7 @@ +# Elasticsearch Index Templates + +## Generation + +* Index templates are generated from `YAML` files contained in the package. +* There is one index template per dataset. +* For the generation of an index template, all `yml` files contained in the package subdirectory `dataset/DATASET_NAME/fields/` are used. diff --git a/docs/ingest_manager/index.asciidoc b/docs/ingest_manager/index.asciidoc index 22afa88c919e..866935d1fa58 100644 --- a/docs/ingest_manager/index.asciidoc +++ b/docs/ingest_manager/index.asciidoc @@ -199,3 +199,10 @@ The new ingest pipeline is expected to still work with the data coming from olde In case of a breaking change in the data structure, the new ingest pipeline is also expected to deal with this change. In case there are breaking changes which cannot be dealt with in an ingest pipeline, a new package has to be created. Each package lists its minimal required agent version. In case there are agents enrolled with an older version, the user is notified to upgrade these agents as otherwise the new configs cannot be rolled out. + +=== Generated assets + +When a package is installed or upgraded, certain Kibana and Elasticsearch assets are generated from . These follow the naming conventions explained above (see "indexing strategy") and contain configuration for the elastic stack that makes ingesting and displaying data work with as little user interaction as possible. + +* link:index-templates.asciidoc[Elasticsearch Index Templates] +* Kibana Index Patterns diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index 0e239c24dd9c..166983fbccc3 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -47,12 +47,12 @@ exports[`tests loading base.yml: base.yml 1`] = ` "user": { "properties": { "auid": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "euid": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" } } }, @@ -73,12 +73,12 @@ exports[`tests loading base.yml: base.yml 1`] = ` "nested": { "properties": { "bar": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "baz": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" } } }, @@ -142,8 +142,8 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` "coredns": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "query": { "properties": { @@ -151,28 +151,28 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` "type": "long" }, "class": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "name": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "type": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" } } }, "response": { "properties": { "code": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "flags": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "size": { "type": "long" @@ -509,12 +509,12 @@ exports[`tests loading system.yml: system.yml 1`] = ` "diskio": { "properties": { "name": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "serial_number": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "read": { "properties": { @@ -643,16 +643,16 @@ exports[`tests loading system.yml: system.yml 1`] = ` "type": "long" }, "device_name": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "type": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "mount_point": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "files": { "type": "long" @@ -867,8 +867,8 @@ exports[`tests loading system.yml: system.yml 1`] = ` "network": { "properties": { "name": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "out": { "properties": { @@ -946,12 +946,12 @@ exports[`tests loading system.yml: system.yml 1`] = ` "process": { "properties": { "state": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "cmdline": { - "type": "keyword", - "ignore_above": 2048 + "ignore_above": 2048, + "type": "keyword" }, "env": { "type": "object" @@ -1040,22 +1040,22 @@ exports[`tests loading system.yml: system.yml 1`] = ` "cgroup": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "path": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "cpu": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "path": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "cfs": { "properties": { @@ -1118,12 +1118,12 @@ exports[`tests loading system.yml: system.yml 1`] = ` "cpuacct": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "path": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "total": { "properties": { @@ -1158,12 +1158,12 @@ exports[`tests loading system.yml: system.yml 1`] = ` "memory": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "path": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "mem": { "properties": { @@ -1382,12 +1382,12 @@ exports[`tests loading system.yml: system.yml 1`] = ` "blkio": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "path": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "total": { "properties": { @@ -1436,20 +1436,20 @@ exports[`tests loading system.yml: system.yml 1`] = ` "raid": { "properties": { "name": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "status": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "level": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "sync_action": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "disks": { "properties": { @@ -1507,24 +1507,24 @@ exports[`tests loading system.yml: system.yml 1`] = ` "type": "long" }, "host": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "etld_plus_one": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "host_error": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" } } }, "process": { "properties": { "cmdline": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1622,42 +1622,42 @@ exports[`tests loading system.yml: system.yml 1`] = ` "users": { "properties": { "id": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "seat": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "path": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "type": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "service": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "remote": { "type": "boolean" }, "state": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "scope": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" }, "leader": { "type": "long" }, "remote_host": { - "type": "keyword", - "ignore_above": 1024 + "ignore_above": 1024, + "type": "keyword" } } } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts index f4e13748641e..cde2e5b420f3 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts @@ -63,3 +63,101 @@ test('tests loading system.yml', () => { expect(template).toMatchSnapshot(path.basename(ymlPath)); }); + +test('tests processing text field with multi fields', () => { + const textWithMultiFieldsLiteralYml = ` +- name: textWithMultiFields + type: text + multi_fields: + - name: raw + type: keyword + - name: indexed + type: text +`; + const textWithMultiFieldsMapping = { + properties: { + textWithMultiFields: { + type: 'text', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping)); +}); + +test('tests processing keyword field with multi fields', () => { + const keywordWithMultiFieldsLiteralYml = ` +- name: keywordWithMultiFields + type: keyword + multi_fields: + - name: raw + type: keyword + - name: indexed + type: text +`; + + const keywordWithMultiFieldsMapping = { + properties: { + keywordWithMultiFields: { + ignore_above: 1024, + type: 'keyword', + fields: { + raw: { + ignore_above: 1024, + type: 'keyword', + }, + indexed: { + type: 'text', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping)); +}); + +test('tests processing keyword field with multi fields with analyzed text field', () => { + const keywordWithAnalyzedMultiFieldsLiteralYml = ` + - name: keywordWithAnalyzedMultiField + type: keyword + multi_fields: + - name: analyzed + type: text + analyzer: autocomplete + search_analyzer: standard + `; + + const keywordWithAnalyzedMultiFieldsMapping = { + properties: { + keywordWithAnalyzedMultiField: { + ignore_above: 1024, + type: 'keyword', + fields: { + analyzed: { + analyzer: 'autocomplete', + search_analyzer: 'standard', + type: 'text', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping)); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 832e4772beb0..e876c8a2efb4 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Field } from '../../fields/field'; +import { Field, Fields } from '../../fields/field'; import { Dataset, IndexTemplate } from '../../../../types'; import { getDatasetAssetBaseName } from '../index'; @@ -15,6 +15,14 @@ interface Mappings { properties: any; } +interface Mapping { + [key: string]: any; +} + +interface MultiFields { + [key: string]: object; +} + const DEFAULT_SCALING_FACTOR = 1000; const DEFAULT_IGNORE_ABOVE = 1024; @@ -67,23 +75,19 @@ export function generateMappings(fields: Field[]): Mappings { fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR; break; case 'text': - fieldProps.type = 'text'; - if (field.analyzer) { - fieldProps.analyzer = field.analyzer; - } - if (field.search_analyzer) { - fieldProps.search_analyzer = field.search_analyzer; + const textMapping = generateTextMapping(field); + fieldProps = { ...fieldProps, ...textMapping, type: 'text' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); } break; case 'keyword': - fieldProps.type = 'keyword'; - if (field.ignore_above) { - fieldProps.ignore_above = field.ignore_above; - } else { - fieldProps.ignore_above = DEFAULT_IGNORE_ABOVE; + const keywordMapping = generateKeywordMapping(field); + fieldProps = { ...fieldProps, ...keywordMapping, type: 'keyword' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); } break; - // TODO move handling of multi_fields here? case 'object': // TODO improve fieldProps.type = 'object'; @@ -113,6 +117,45 @@ export function generateMappings(fields: Field[]): Mappings { return { properties: props }; } +function generateMultiFields(fields: Fields): MultiFields { + const multiFields: MultiFields = {}; + if (fields) { + fields.forEach((f: Field) => { + const type = f.type; + switch (type) { + case 'text': + multiFields[f.name] = { ...generateTextMapping(f), type: f.type }; + break; + case 'keyword': + multiFields[f.name] = { ...generateKeywordMapping(f), type: f.type }; + break; + } + }); + } + return multiFields; +} + +function generateKeywordMapping(field: Field): Mapping { + const mapping: Mapping = { + ignore_above: DEFAULT_IGNORE_ABOVE, + }; + if (field.ignore_above) { + mapping.ignore_above = field.ignore_above; + } + return mapping; +} + +function generateTextMapping(field: Field): Mapping { + const mapping: Mapping = {}; + if (field.analyzer) { + mapping.analyzer = field.analyzer; + } + if (field.search_analyzer) { + mapping.search_analyzer = field.search_analyzer; + } + return mapping; +} + function getDefaultProperties(field: Field): Properties { const properties: Properties = {};