maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[EPM] Handle multi fields in index template generation (#63112)

Browse source 
* Add unit test stub for multi fields.

* Add multi field handling to mapping generation.

* Start documenting index template generation.

* Add unit tests.

* Remove stub for fields.yml documentation

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Sonja Krause-Harder 2020-04-17 13:23:30 +02:00 committed by GitHub
parent 90f5ce0ef3
commit f91c795e30
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 256 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
docs/ingest_manager/index-templates.asciidoc Normal file
View file

@ -0,0 +1,7 @@
# Elasticsearch Index Templates
## Generation
* Index templates are generated from `YAML` files contained in the package.
* There is one index template per dataset.
* For the generation of an index template, all `yml` files contained in the package subdirectory `dataset/DATASET_NAME/fields/` are used.

7
docs/ingest_manager/index.asciidoc
View file

@ -199,3 +199,10 @@ The new ingest pipeline is expected to still work with the data coming from olde
In case of a breaking change in the data structure, the new ingest pipeline is also expected to deal with this change. In case there are breaking changes which cannot be dealt with in an ingest pipeline, a new package has to be created.
Each package lists its minimal required agent version. In case there are agents enrolled with an older version, the user is notified to upgrade these agents as otherwise the new configs cannot be rolled out.
=== Generated assets
When a package is installed or upgraded, certain Kibana and Elasticsearch assets are generated from . These follow the naming conventions explained above (see "indexing strategy") and contain configuration for the elastic stack that makes ingesting and displaying data work with as little user interaction as possible.
* link:index-templates.asciidoc[Elasticsearch Index Templates]
* Kibana Index Patterns

176
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap generated
View file

@ -47,12 +47,12 @@ exports[`tests loading base.yml: base.yml 1`] = `
"user": {
"properties": {
"auid": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"euid": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
}
}
},
@ -73,12 +73,12 @@ exports[`tests loading base.yml: base.yml 1`] = `
"nested": {
"properties": {
"bar": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"baz": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
}
}
},
@ -142,8 +142,8 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = `
"coredns": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"query": {
"properties": {
@ -151,28 +151,28 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = `
"type": "long"
},
"class": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
}
}
},
"response": {
"properties": {
"code": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"flags": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"size": {
"type": "long"
@ -509,12 +509,12 @@ exports[`tests loading system.yml: system.yml 1`] = `
"diskio": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"serial_number": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"read": {
"properties": {
@ -643,16 +643,16 @@ exports[`tests loading system.yml: system.yml 1`] = `
"type": "long"
},
"device_name": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"mount_point": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"files": {
"type": "long"
@ -867,8 +867,8 @@ exports[`tests loading system.yml: system.yml 1`] = `
"network": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"out": {
"properties": {
@ -946,12 +946,12 @@ exports[`tests loading system.yml: system.yml 1`] = `
"process": {
"properties": {
"state": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"cmdline": {
"type": "keyword",
"ignore_above": 2048
"ignore_above": 2048,
"type": "keyword"
},
"env": {
"type": "object"
@ -1040,22 +1040,22 @@ exports[`tests loading system.yml: system.yml 1`] = `
"cgroup": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"cpu": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"cfs": {
"properties": {
@ -1118,12 +1118,12 @@ exports[`tests loading system.yml: system.yml 1`] = `
"cpuacct": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"total": {
"properties": {
@ -1158,12 +1158,12 @@ exports[`tests loading system.yml: system.yml 1`] = `
"memory": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"mem": {
"properties": {
@ -1382,12 +1382,12 @@ exports[`tests loading system.yml: system.yml 1`] = `
"blkio": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"total": {
"properties": {
@ -1436,20 +1436,20 @@ exports[`tests loading system.yml: system.yml 1`] = `
"raid": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"level": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"sync_action": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"disks": {
"properties": {
@ -1507,24 +1507,24 @@ exports[`tests loading system.yml: system.yml 1`] = `
"type": "long"
},
"host": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"etld_plus_one": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"host_error": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
}
}
},
"process": {
"properties": {
"cmdline": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
}
}
},
@ -1622,42 +1622,42 @@ exports[`tests loading system.yml: system.yml 1`] = `
"users": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"seat": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"service": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"remote": {
"type": "boolean"
},
"state": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"scope": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
},
"leader": {
"type": "long"
},
"remote_host": {
"type": "keyword",
"ignore_above": 1024
"ignore_above": 1024,
"type": "keyword"
}
}
}

98
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
View file

@ -63,3 +63,101 @@ test('tests loading system.yml', () => {
expect(template).toMatchSnapshot(path.basename(ymlPath));
});
test('tests processing text field with multi fields', () => {
const textWithMultiFieldsLiteralYml = `
- name: textWithMultiFields
type: text
multi_fields:
- name: raw
type: keyword
- name: indexed
type: text
`;
const textWithMultiFieldsMapping = {
properties: {
textWithMultiFields: {
type: 'text',
fields: {
raw: {
ignore_above: 1024,
type: 'keyword',
},
indexed: {
type: 'text',
},
},
},
},
};
const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping));
});
test('tests processing keyword field with multi fields', () => {
const keywordWithMultiFieldsLiteralYml = `
- name: keywordWithMultiFields
type: keyword
multi_fields:
- name: raw
type: keyword
- name: indexed
type: text
`;
const keywordWithMultiFieldsMapping = {
properties: {
keywordWithMultiFields: {
ignore_above: 1024,
type: 'keyword',
fields: {
raw: {
ignore_above: 1024,
type: 'keyword',
},
indexed: {
type: 'text',
},
},
},
},
};
const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping));
});
test('tests processing keyword field with multi fields with analyzed text field', () => {
const keywordWithAnalyzedMultiFieldsLiteralYml = `
- name: keywordWithAnalyzedMultiField
type: keyword
multi_fields:
- name: analyzed
type: text
analyzer: autocomplete
search_analyzer: standard
`;
const keywordWithAnalyzedMultiFieldsMapping = {
properties: {
keywordWithAnalyzedMultiField: {
ignore_above: 1024,
type: 'keyword',
fields: {
analyzed: {
analyzer: 'autocomplete',
search_analyzer: 'standard',
type: 'text',
},
},
},
},
};
const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping));
});

69
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { Field } from '../../fields/field';
import { Field, Fields } from '../../fields/field';
import { Dataset, IndexTemplate } from '../../../../types';
import { getDatasetAssetBaseName } from '../index';
@ -15,6 +15,14 @@ interface Mappings {
properties: any;
}
interface Mapping {
[key: string]: any;
}
interface MultiFields {
[key: string]: object;
}
const DEFAULT_SCALING_FACTOR = 1000;
const DEFAULT_IGNORE_ABOVE = 1024;
@ -67,23 +75,19 @@ export function generateMappings(fields: Field[]): Mappings {
fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR;
break;
case 'text':
fieldProps.type = 'text';
if (field.analyzer) {
fieldProps.analyzer = field.analyzer;
}
if (field.search_analyzer) {
fieldProps.search_analyzer = field.search_analyzer;
const textMapping = generateTextMapping(field);
fieldProps = { ...fieldProps, ...textMapping, type: 'text' };
if (field.multi_fields) {
fieldProps.fields = generateMultiFields(field.multi_fields);
}
break;
case 'keyword':
fieldProps.type = 'keyword';
if (field.ignore_above) {
fieldProps.ignore_above = field.ignore_above;
} else {
fieldProps.ignore_above = DEFAULT_IGNORE_ABOVE;
const keywordMapping = generateKeywordMapping(field);
fieldProps = { ...fieldProps, ...keywordMapping, type: 'keyword' };
if (field.multi_fields) {
fieldProps.fields = generateMultiFields(field.multi_fields);
}
break;
// TODO move handling of multi_fields here?
case 'object':
// TODO improve
fieldProps.type = 'object';
@ -113,6 +117,45 @@ export function generateMappings(fields: Field[]): Mappings {
return { properties: props };
}
function generateMultiFields(fields: Fields): MultiFields {
const multiFields: MultiFields = {};
if (fields) {
fields.forEach((f: Field) => {
const type = f.type;
switch (type) {
case 'text':
multiFields[f.name] = { ...generateTextMapping(f), type: f.type };
break;
case 'keyword':
multiFields[f.name] = { ...generateKeywordMapping(f), type: f.type };
break;
}
});
}
return multiFields;
}
function generateKeywordMapping(field: Field): Mapping {
const mapping: Mapping = {
ignore_above: DEFAULT_IGNORE_ABOVE,
};
if (field.ignore_above) {
mapping.ignore_above = field.ignore_above;
}
return mapping;
}
function generateTextMapping(field: Field): Mapping {
const mapping: Mapping = {};
if (field.analyzer) {
mapping.analyzer = field.analyzer;
}
if (field.search_analyzer) {
mapping.search_analyzer = field.search_analyzer;
}
return mapping;
}
function getDefaultProperties(field: Field): Properties {
const properties: Properties = {};