[EPM] Adding support for nested fields (#64829)
* Allowing nested types to be merged with group * Adding nested to case and handling other fields * Cleaing up if logic * Removing unneeded if statement * Adding nested type to switch and more tests * Keeping functions immutable
This commit is contained in:
parent
9112b6c1f1
commit
a7291fa8c8
|
@ -93,7 +93,7 @@ test('tests processing text field with multi fields', () => {
|
|||
const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping));
|
||||
expect(mappings).toEqual(textWithMultiFieldsMapping);
|
||||
});
|
||||
|
||||
test('tests processing keyword field with multi fields', () => {
|
||||
|
@ -127,7 +127,7 @@ test('tests processing keyword field with multi fields', () => {
|
|||
const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping));
|
||||
expect(mappings).toEqual(keywordWithMultiFieldsMapping);
|
||||
});
|
||||
|
||||
test('tests processing keyword field with multi fields with analyzed text field', () => {
|
||||
|
@ -159,7 +159,7 @@ test('tests processing keyword field with multi fields with analyzed text field'
|
|||
const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping));
|
||||
expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with no other attributes', () => {
|
||||
|
@ -177,7 +177,7 @@ test('tests processing object field with no other attributes', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldMapping));
|
||||
expect(mappings).toEqual(objectFieldMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with enabled set to false', () => {
|
||||
|
@ -197,7 +197,7 @@ test('tests processing object field with enabled set to false', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldEnabledFalseMapping));
|
||||
expect(mappings).toEqual(objectFieldEnabledFalseMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with dynamic set to false', () => {
|
||||
|
@ -217,7 +217,7 @@ test('tests processing object field with dynamic set to false', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicFalseMapping));
|
||||
expect(mappings).toEqual(objectFieldDynamicFalseMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with dynamic set to true', () => {
|
||||
|
@ -237,7 +237,7 @@ test('tests processing object field with dynamic set to true', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicTrueMapping));
|
||||
expect(mappings).toEqual(objectFieldDynamicTrueMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with dynamic set to strict', () => {
|
||||
|
@ -257,7 +257,7 @@ test('tests processing object field with dynamic set to strict', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicStrictMapping));
|
||||
expect(mappings).toEqual(objectFieldDynamicStrictMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with property', () => {
|
||||
|
@ -282,7 +282,7 @@ test('tests processing object field with property', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyMapping));
|
||||
expect(mappings).toEqual(objectFieldWithPropertyMapping);
|
||||
});
|
||||
|
||||
test('tests processing object field with property, reverse order', () => {
|
||||
|
@ -291,10 +291,12 @@ test('tests processing object field with property, reverse order', () => {
|
|||
type: keyword
|
||||
- name: a
|
||||
type: object
|
||||
dynamic: false
|
||||
`;
|
||||
const objectFieldWithPropertyReversedMapping = {
|
||||
properties: {
|
||||
a: {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
b: {
|
||||
ignore_above: 1024,
|
||||
|
@ -307,7 +309,91 @@ test('tests processing object field with property, reverse order', () => {
|
|||
const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyReversedMapping));
|
||||
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
|
||||
});
|
||||
|
||||
test('tests processing nested field with property', () => {
|
||||
const nestedYaml = `
|
||||
- name: a.b
|
||||
type: keyword
|
||||
- name: a
|
||||
type: nested
|
||||
dynamic: false
|
||||
`;
|
||||
const expectedMapping = {
|
||||
properties: {
|
||||
a: {
|
||||
dynamic: false,
|
||||
type: 'nested',
|
||||
properties: {
|
||||
b: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const fields: Field[] = safeLoad(nestedYaml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
test('tests processing nested field with property, nested field first', () => {
|
||||
const nestedYaml = `
|
||||
- name: a
|
||||
type: nested
|
||||
include_in_parent: true
|
||||
- name: a.b
|
||||
type: keyword
|
||||
`;
|
||||
const expectedMapping = {
|
||||
properties: {
|
||||
a: {
|
||||
include_in_parent: true,
|
||||
type: 'nested',
|
||||
properties: {
|
||||
b: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const fields: Field[] = safeLoad(nestedYaml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
test('tests processing nested leaf field with properties', () => {
|
||||
const nestedYaml = `
|
||||
- name: a
|
||||
type: object
|
||||
dynamic: false
|
||||
- name: a.b
|
||||
type: nested
|
||||
enabled: false
|
||||
`;
|
||||
const expectedMapping = {
|
||||
properties: {
|
||||
a: {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
b: {
|
||||
enabled: false,
|
||||
type: 'nested',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const fields: Field[] = safeLoad(nestedYaml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
test('tests constant_keyword field type handling', () => {
|
||||
|
|
|
@ -71,7 +71,14 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
|
|||
|
||||
switch (type) {
|
||||
case 'group':
|
||||
fieldProps = generateMappings(field.fields!);
|
||||
fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) };
|
||||
break;
|
||||
case 'group-nested':
|
||||
fieldProps = {
|
||||
...generateMappings(field.fields!),
|
||||
...generateNestedProps(field),
|
||||
type: 'nested',
|
||||
};
|
||||
break;
|
||||
case 'integer':
|
||||
fieldProps.type = 'long';
|
||||
|
@ -95,13 +102,10 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
|
|||
}
|
||||
break;
|
||||
case 'object':
|
||||
fieldProps.type = 'object';
|
||||
if (field.hasOwnProperty('enabled')) {
|
||||
fieldProps.enabled = field.enabled;
|
||||
}
|
||||
if (field.hasOwnProperty('dynamic')) {
|
||||
fieldProps.dynamic = field.dynamic;
|
||||
}
|
||||
fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
|
||||
break;
|
||||
case 'nested':
|
||||
fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' };
|
||||
break;
|
||||
case 'array':
|
||||
// this assumes array fields were validated in an earlier step
|
||||
|
@ -128,6 +132,29 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
|
|||
return { properties: props };
|
||||
}
|
||||
|
||||
function generateDynamicAndEnabled(field: Field) {
|
||||
const props: Properties = {};
|
||||
if (field.hasOwnProperty('enabled')) {
|
||||
props.enabled = field.enabled;
|
||||
}
|
||||
if (field.hasOwnProperty('dynamic')) {
|
||||
props.dynamic = field.dynamic;
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
function generateNestedProps(field: Field) {
|
||||
const props = generateDynamicAndEnabled(field);
|
||||
|
||||
if (field.hasOwnProperty('include_in_parent')) {
|
||||
props.include_in_parent = field.include_in_parent;
|
||||
}
|
||||
if (field.hasOwnProperty('include_in_root')) {
|
||||
props.include_in_root = field.include_in_root;
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
function generateMultiFields(fields: Fields): MultiFields {
|
||||
const multiFields: MultiFields = {};
|
||||
if (fields) {
|
||||
|
|
|
@ -210,4 +210,166 @@ describe('processFields', () => {
|
|||
JSON.stringify(objectFieldWithPropertyExpanded)
|
||||
);
|
||||
});
|
||||
|
||||
test('correctly handles properties of object type fields where object comes second', () => {
|
||||
const nested = [
|
||||
{
|
||||
name: 'a.b',
|
||||
type: 'keyword',
|
||||
},
|
||||
{
|
||||
name: 'a',
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
];
|
||||
|
||||
const nestedExpanded = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'group',
|
||||
dynamic: true,
|
||||
fields: [
|
||||
{
|
||||
name: 'b',
|
||||
type: 'keyword',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(processFields(nested)).toEqual(nestedExpanded);
|
||||
});
|
||||
|
||||
test('correctly handles properties of nested type fields', () => {
|
||||
const nested = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
dynamic: true,
|
||||
},
|
||||
{
|
||||
name: 'a.b',
|
||||
type: 'keyword',
|
||||
},
|
||||
];
|
||||
|
||||
const nestedExpanded = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'group-nested',
|
||||
dynamic: true,
|
||||
fields: [
|
||||
{
|
||||
name: 'b',
|
||||
type: 'keyword',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(processFields(nested)).toEqual(nestedExpanded);
|
||||
});
|
||||
|
||||
test('correctly handles properties of nested type where nested top level comes second', () => {
|
||||
const nested = [
|
||||
{
|
||||
name: 'a.b',
|
||||
type: 'keyword',
|
||||
},
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
dynamic: true,
|
||||
},
|
||||
];
|
||||
|
||||
const nestedExpanded = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'group-nested',
|
||||
dynamic: true,
|
||||
fields: [
|
||||
{
|
||||
name: 'b',
|
||||
type: 'keyword',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(processFields(nested)).toEqual(nestedExpanded);
|
||||
});
|
||||
|
||||
test('ignores redefinitions of an object field', () => {
|
||||
const object = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
{
|
||||
name: 'a',
|
||||
type: 'object',
|
||||
dynamic: false,
|
||||
},
|
||||
];
|
||||
|
||||
const objectExpected = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'object',
|
||||
// should preserve the field that was parsed first which had dynamic: true
|
||||
dynamic: true,
|
||||
},
|
||||
];
|
||||
expect(processFields(object)).toEqual(objectExpected);
|
||||
});
|
||||
|
||||
test('ignores redefinitions of a nested field', () => {
|
||||
const nested = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
dynamic: true,
|
||||
},
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
dynamic: false,
|
||||
},
|
||||
];
|
||||
|
||||
const nestedExpected = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
// should preserve the field that was parsed first which had dynamic: true
|
||||
dynamic: true,
|
||||
},
|
||||
];
|
||||
expect(processFields(nested)).toEqual(nestedExpected);
|
||||
});
|
||||
|
||||
test('ignores redefinitions of a nested and object field', () => {
|
||||
const nested = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
dynamic: true,
|
||||
},
|
||||
{
|
||||
name: 'a',
|
||||
type: 'object',
|
||||
dynamic: false,
|
||||
},
|
||||
];
|
||||
|
||||
const nestedExpected = [
|
||||
{
|
||||
name: 'a',
|
||||
type: 'nested',
|
||||
// should preserve the field that was parsed first which had dynamic: true
|
||||
dynamic: true,
|
||||
},
|
||||
];
|
||||
expect(processFields(nested)).toEqual(nestedExpected);
|
||||
});
|
||||
});
|
||||
|
|
|
@ -28,6 +28,8 @@ export interface Field {
|
|||
object_type?: string;
|
||||
scaling_factor?: number;
|
||||
dynamic?: 'strict' | boolean;
|
||||
include_in_parent?: boolean;
|
||||
include_in_root?: boolean;
|
||||
|
||||
// Kibana specific
|
||||
analyzed?: boolean;
|
||||
|
@ -108,18 +110,54 @@ function dedupFields(fields: Fields): Fields {
|
|||
return f.name === field.name;
|
||||
});
|
||||
if (found) {
|
||||
// remove name, type, and fields from `field` variable so we avoid merging them into `found`
|
||||
const { name, type, fields: nestedFields, ...importantFieldProps } = field;
|
||||
/**
|
||||
* There are a couple scenarios this if is trying to account for:
|
||||
* Example 1
|
||||
* - name: a.b
|
||||
* - name: a
|
||||
* In this scenario found will be `group` and field could be either `object` or `nested`
|
||||
* Example 2
|
||||
* - name: a
|
||||
* - name: a.b
|
||||
* In this scenario found could be `object` or `nested` and field will be group
|
||||
*/
|
||||
if (
|
||||
(found.type === 'group' || found.type === 'object') &&
|
||||
field.type === 'group' &&
|
||||
field.fields
|
||||
// only merge if found is a group and field is object, nested, or group.
|
||||
// Or if found is object, or nested, and field is a group.
|
||||
// This is to avoid merging two objects, or nested, or object with a nested.
|
||||
(found.type === 'group' &&
|
||||
(field.type === 'object' || field.type === 'nested' || field.type === 'group')) ||
|
||||
((found.type === 'object' || found.type === 'nested') && field.type === 'group')
|
||||
) {
|
||||
if (!found.fields) {
|
||||
found.fields = [];
|
||||
// if the new field has properties let's dedup and concat them with the already existing found variable in
|
||||
// the array
|
||||
if (field.fields) {
|
||||
// if the found type was object or nested it won't have a fields array so let's initialize it
|
||||
if (!found.fields) {
|
||||
found.fields = [];
|
||||
}
|
||||
found.fields = dedupFields(found.fields.concat(field.fields));
|
||||
}
|
||||
found.type = 'group';
|
||||
found.fields = dedupFields(found.fields.concat(field.fields));
|
||||
|
||||
// if found already had fields or got new ones from the new field coming in we need to assign the right
|
||||
// type to it
|
||||
if (found.fields) {
|
||||
// If this field is supposed to be `nested` and we have fields, we need to preserve the fact that it is
|
||||
// supposed to be `nested` for when the template is actually generated
|
||||
if (found.type === 'nested' || field.type === 'nested') {
|
||||
found.type = 'group-nested';
|
||||
} else {
|
||||
// found was either `group` already or `object` so just set it to `group`
|
||||
found.type = 'group';
|
||||
}
|
||||
}
|
||||
// we need to merge in other properties (like `dynamic`) that might exist
|
||||
Object.assign(found, importantFieldProps);
|
||||
// if `field.type` wasn't group object or nested, then there's a conflict in types, so lets ignore it
|
||||
} else {
|
||||
// only 'group' fields can be merged in this way
|
||||
// only `group`, `object`, and `nested` fields can be merged in this way
|
||||
// XXX: don't abort on error for now
|
||||
// see discussion in https://github.com/elastic/kibana/pull/59894
|
||||
// throw new Error(
|
||||
|
|
Loading…
Reference in a new issue