maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[EPM] Adding support for nested fields (#64829)

Browse source 
* Allowing nested types to be merged with group

* Adding nested to case and handling other fields

* Cleaing up if logic

* Removing unneeded if statement

* Adding nested type to switch and more tests

* Keeping functions immutable
This commit is contained in:
Jonathan Buttner 2020-04-30 13:03:08 -04:00 committed by GitHub
parent 9112b6c1f1
commit a7291fa8c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 339 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

106
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
View file

@ -93,7 +93,7 @@ test('tests processing text field with multi fields', () => {
const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping));
expect(mappings).toEqual(textWithMultiFieldsMapping);
});
test('tests processing keyword field with multi fields', () => {
@ -127,7 +127,7 @@ test('tests processing keyword field with multi fields', () => {
const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping));
expect(mappings).toEqual(keywordWithMultiFieldsMapping);
});
test('tests processing keyword field with multi fields with analyzed text field', () => {
@ -159,7 +159,7 @@ test('tests processing keyword field with multi fields with analyzed text field'
const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping));
expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
});
test('tests processing object field with no other attributes', () => {
@ -177,7 +177,7 @@ test('tests processing object field with no other attributes', () => {
const fields: Field[] = safeLoad(objectFieldLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldMapping));
expect(mappings).toEqual(objectFieldMapping);
});
test('tests processing object field with enabled set to false', () => {
@ -197,7 +197,7 @@ test('tests processing object field with enabled set to false', () => {
const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldEnabledFalseMapping));
expect(mappings).toEqual(objectFieldEnabledFalseMapping);
});
test('tests processing object field with dynamic set to false', () => {
@ -217,7 +217,7 @@ test('tests processing object field with dynamic set to false', () => {
const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicFalseMapping));
expect(mappings).toEqual(objectFieldDynamicFalseMapping);
});
test('tests processing object field with dynamic set to true', () => {
@ -237,7 +237,7 @@ test('tests processing object field with dynamic set to true', () => {
const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicTrueMapping));
expect(mappings).toEqual(objectFieldDynamicTrueMapping);
});
test('tests processing object field with dynamic set to strict', () => {
@ -257,7 +257,7 @@ test('tests processing object field with dynamic set to strict', () => {
const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicStrictMapping));
expect(mappings).toEqual(objectFieldDynamicStrictMapping);
});
test('tests processing object field with property', () => {
@ -282,7 +282,7 @@ test('tests processing object field with property', () => {
const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyMapping));
expect(mappings).toEqual(objectFieldWithPropertyMapping);
});
test('tests processing object field with property, reverse order', () => {
@ -291,10 +291,12 @@ test('tests processing object field with property, reverse order', () => {
type: keyword
- name: a
type: object
dynamic: false
`;
const objectFieldWithPropertyReversedMapping = {
properties: {
a: {
dynamic: false,
properties: {
b: {
ignore_above: 1024,
@ -307,7 +309,91 @@ test('tests processing object field with property, reverse order', () => {
const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyReversedMapping));
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});
test('tests processing nested field with property', () => {
const nestedYaml = `
- name: a.b
type: keyword
- name: a
type: nested
dynamic: false
`;
const expectedMapping = {
properties: {
a: {
dynamic: false,
type: 'nested',
properties: {
b: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});
test('tests processing nested field with property, nested field first', () => {
const nestedYaml = `
- name: a
type: nested
include_in_parent: true
- name: a.b
type: keyword
`;
const expectedMapping = {
properties: {
a: {
include_in_parent: true,
type: 'nested',
properties: {
b: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});
test('tests processing nested leaf field with properties', () => {
const nestedYaml = `
- name: a
type: object
dynamic: false
- name: a.b
type: nested
enabled: false
`;
const expectedMapping = {
properties: {
a: {
dynamic: false,
properties: {
b: {
enabled: false,
type: 'nested',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});
test('tests constant_keyword field type handling', () => {

43
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
View file

@ -71,7 +71,14 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
switch (type) {
case 'group':
fieldProps = generateMappings(field.fields!);
fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) };
break;
case 'group-nested':
fieldProps = {
...generateMappings(field.fields!),
...generateNestedProps(field),
type: 'nested',
};
break;
case 'integer':
fieldProps.type = 'long';
@ -95,13 +102,10 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
}
break;
case 'object':
fieldProps.type = 'object';
if (field.hasOwnProperty('enabled')) {
fieldProps.enabled = field.enabled;
}
if (field.hasOwnProperty('dynamic')) {
fieldProps.dynamic = field.dynamic;
}
fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
break;
case 'nested':
fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' };
break;
case 'array':
// this assumes array fields were validated in an earlier step
@ -128,6 +132,29 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
return { properties: props };
}
function generateDynamicAndEnabled(field: Field) {
const props: Properties = {};
if (field.hasOwnProperty('enabled')) {
props.enabled = field.enabled;
}
if (field.hasOwnProperty('dynamic')) {
props.dynamic = field.dynamic;
}
return props;
}
function generateNestedProps(field: Field) {
const props = generateDynamicAndEnabled(field);
if (field.hasOwnProperty('include_in_parent')) {
props.include_in_parent = field.include_in_parent;
}
if (field.hasOwnProperty('include_in_root')) {
props.include_in_root = field.include_in_root;
}
return props;
}
function generateMultiFields(fields: Fields): MultiFields {
const multiFields: MultiFields = {};
if (fields) {

162
x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts
View file

@ -210,4 +210,166 @@ describe('processFields', () => {
JSON.stringify(objectFieldWithPropertyExpanded)
);
});
test('correctly handles properties of object type fields where object comes second', () => {
const nested = [
{
name: 'a.b',
type: 'keyword',
},
{
name: 'a',
type: 'object',
dynamic: true,
},
];
const nestedExpanded = [
{
name: 'a',
type: 'group',
dynamic: true,
fields: [
{
name: 'b',
type: 'keyword',
},
],
},
];
expect(processFields(nested)).toEqual(nestedExpanded);
});
test('correctly handles properties of nested type fields', () => {
const nested = [
{
name: 'a',
type: 'nested',
dynamic: true,
},
{
name: 'a.b',
type: 'keyword',
},
];
const nestedExpanded = [
{
name: 'a',
type: 'group-nested',
dynamic: true,
fields: [
{
name: 'b',
type: 'keyword',
},
],
},
];
expect(processFields(nested)).toEqual(nestedExpanded);
});
test('correctly handles properties of nested type where nested top level comes second', () => {
const nested = [
{
name: 'a.b',
type: 'keyword',
},
{
name: 'a',
type: 'nested',
dynamic: true,
},
];
const nestedExpanded = [
{
name: 'a',
type: 'group-nested',
dynamic: true,
fields: [
{
name: 'b',
type: 'keyword',
},
],
},
];
expect(processFields(nested)).toEqual(nestedExpanded);
});
test('ignores redefinitions of an object field', () => {
const object = [
{
name: 'a',
type: 'object',
dynamic: true,
},
{
name: 'a',
type: 'object',
dynamic: false,
},
];
const objectExpected = [
{
name: 'a',
type: 'object',
// should preserve the field that was parsed first which had dynamic: true
dynamic: true,
},
];
expect(processFields(object)).toEqual(objectExpected);
});
test('ignores redefinitions of a nested field', () => {
const nested = [
{
name: 'a',
type: 'nested',
dynamic: true,
},
{
name: 'a',
type: 'nested',
dynamic: false,
},
];
const nestedExpected = [
{
name: 'a',
type: 'nested',
// should preserve the field that was parsed first which had dynamic: true
dynamic: true,
},
];
expect(processFields(nested)).toEqual(nestedExpected);
});
test('ignores redefinitions of a nested and object field', () => {
const nested = [
{
name: 'a',
type: 'nested',
dynamic: true,
},
{
name: 'a',
type: 'object',
dynamic: false,
},
];
const nestedExpected = [
{
name: 'a',
type: 'nested',
// should preserve the field that was parsed first which had dynamic: true
dynamic: true,
},
];
expect(processFields(nested)).toEqual(nestedExpected);
});
});

54
x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts
View file

@ -28,6 +28,8 @@ export interface Field {
object_type?: string;
scaling_factor?: number;
dynamic?: 'strict' | boolean;
include_in_parent?: boolean;
include_in_root?: boolean;
// Kibana specific
analyzed?: boolean;
@ -108,18 +110,54 @@ function dedupFields(fields: Fields): Fields {
return f.name === field.name;
});
if (found) {
// remove name, type, and fields from `field` variable so we avoid merging them into `found`
const { name, type, fields: nestedFields, ...importantFieldProps } = field;
/**
* There are a couple scenarios this if is trying to account for:
* Example 1
* - name: a.b
* - name: a
* In this scenario found will be `group` and field could be either `object` or `nested`
* Example 2
* - name: a
* - name: a.b
* In this scenario found could be `object` or `nested` and field will be group
*/
if (
(found.type === 'group' || found.type === 'object') &&
field.type === 'group' &&
field.fields
// only merge if found is a group and field is object, nested, or group.
// Or if found is object, or nested, and field is a group.
// This is to avoid merging two objects, or nested, or object with a nested.
(found.type === 'group' &&
(field.type === 'object' || field.type === 'nested' || field.type === 'group')) ||
((found.type === 'object' || found.type === 'nested') && field.type === 'group')
) {
if (!found.fields) {
found.fields = [];
// if the new field has properties let's dedup and concat them with the already existing found variable in
// the array
if (field.fields) {
// if the found type was object or nested it won't have a fields array so let's initialize it
if (!found.fields) {
found.fields = [];
}
found.fields = dedupFields(found.fields.concat(field.fields));
}
found.type = 'group';
found.fields = dedupFields(found.fields.concat(field.fields));
// if found already had fields or got new ones from the new field coming in we need to assign the right
// type to it
if (found.fields) {
// If this field is supposed to be `nested` and we have fields, we need to preserve the fact that it is
// supposed to be `nested` for when the template is actually generated
if (found.type === 'nested' || field.type === 'nested') {
found.type = 'group-nested';
} else {
// found was either `group` already or `object` so just set it to `group`
found.type = 'group';
}
}
// we need to merge in other properties (like `dynamic`) that might exist
Object.assign(found, importantFieldProps);
// if `field.type` wasn't group object or nested, then there's a conflict in types, so lets ignore it
} else {
// only 'group' fields can be merged in this way
// only `group`, `object`, and `nested` fields can be merged in this way
// XXX: don't abort on error for now
// see discussion in https://github.com/elastic/kibana/pull/59894
// throw new Error(