maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[EPM] Conditionally generate ES index pattern name based on dataset_is_prefix (#89870)

Browse source 
* Explicitly generate ES index pattern name.

* Adjust tests.

* Adjust and reenable tests.

* Set template priority based on dataset_is_prefix

* Refactor indexPatternName -> templateIndexPattern

* Add unit tests.

* Use more realistic index pattern in test.

* Fix unit test.

* Add unit test for installTemplate().

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Sonja Krause-Harder 2021-02-08 18:22:30 +01:00 committed by GitHub
parent bda7b2816f
commit c306a444f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 268 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/fleet/common/types/models/epm.ts
View file

@ -221,6 +221,7 @@ export interface RegistryDataStream {
path: string;
ingest_pipeline: string;
elasticsearch?: RegistryElasticsearch;
dataset_is_prefix?: boolean;
}
export interface RegistryElasticsearch {

110
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RegistryDataStream } from '../../../../types';
import { Field } from '../../fields/field';
import { elasticsearchServiceMock } from 'src/core/server/mocks';
import { installTemplate } from './install';
test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser;
const fields: Field[] = [];
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
} as RegistryDataStream;
const pkg = {
name: 'package',
version: '0.0.1',
};
const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixUnset = 200;
await installTemplate({
callCluster,
fields,
dataStream: dataStreamDatasetIsPrefixUnset,
packageVersion: pkg.version,
packageName: pkg.name,
});
// @ts-ignore
const sentTemplate = callCluster.mock.calls[0][1].body;
expect(sentTemplate).toBeDefined();
expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
});
test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => {
const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser;
const fields: Field[] = [];
const dataStreamDatasetIsPrefixFalse = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
dataset_is_prefix: false,
} as RegistryDataStream;
const pkg = {
name: 'package',
version: '0.0.1',
};
const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixFalse = 200;
await installTemplate({
callCluster,
fields,
dataStream: dataStreamDatasetIsPrefixFalse,
packageVersion: pkg.version,
packageName: pkg.name,
});
// @ts-ignore
const sentTemplate = callCluster.mock.calls[0][1].body;
expect(sentTemplate).toBeDefined();
expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse);
expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
});
test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => {
const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser;
const fields: Field[] = [];
const dataStreamDatasetIsPrefixTrue = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
dataset_is_prefix: true,
} as RegistryDataStream;
const pkg = {
name: 'package',
version: '0.0.1',
};
const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*';
const templatePriorityDatasetIsPrefixTrue = 150;
await installTemplate({
callCluster,
fields,
dataStream: dataStreamDatasetIsPrefixTrue,
packageVersion: pkg.version,
packageName: pkg.name,
});
// @ts-ignore
const sentTemplate = callCluster.mock.calls[0][1].body;
expect(sentTemplate).toBeDefined();
expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue);
expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
});

14
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
View file

@ -17,7 +17,13 @@ import {
import { CallESAsCurrentUser } from '../../../../types';
import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
import { generateMappings, generateTemplateName, getTemplate } from './template';
import {
generateMappings,
generateTemplateName,
generateTemplateIndexPattern,
getTemplate,
getTemplatePriority,
} from './template';
import { getAsset, getPathParts } from '../../archive';
import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install';
@ -293,6 +299,9 @@ export async function installTemplate({
}): Promise<TemplateRef> {
const mappings = generateMappings(processFields(fields));
const templateName = generateTemplateName(dataStream);
const templateIndexPattern = generateTemplateIndexPattern(dataStream);
const templatePriority = getTemplatePriority(dataStream);
let pipelineName;
if (dataStream.ingest_pipeline) {
pipelineName = getPipelineNameForInstallation({
@ -310,11 +319,12 @@ export async function installTemplate({
const template = getTemplate({
type: dataStream.type,
templateName,
templateIndexPattern,
mappings,
pipelineName,
packageName,
composedOfTemplates,
templatePriority,
ilmPolicy: dataStream.ilm_policy,
hidden: dataStream.hidden,
});

99
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
View file

@ -8,8 +8,14 @@
import { readFileSync } from 'fs';
import { safeLoad } from 'js-yaml';
import path from 'path';
import { RegistryDataStream } from '../../../../types';
import { Field, processFields } from '../../fields/field';
import { generateMappings, getTemplate } from './template';
import {
generateMappings,
getTemplate,
getTemplatePriority,
generateTemplateIndexPattern,
} from './template';
// Add our own serialiser to just do JSON.stringify
expect.addSnapshotSerializer({
@ -23,16 +29,17 @@ expect.addSnapshotSerializer({
});
test('get template', () => {
const templateName = 'logs-nginx-access-abcd';
const templateIndexPattern = 'logs-nginx.access-abcd-*';
const template = getTemplate({
type: 'logs',
templateName,
templateIndexPattern,
packageName: 'nginx',
mappings: { properties: {} },
composedOfTemplates: [],
templatePriority: 200,
});
expect(template.index_patterns).toStrictEqual([`${templateName}-*`]);
expect(template.index_patterns).toStrictEqual([templateIndexPattern]);
});
test('adds composed_of correctly', () => {
@ -40,10 +47,11 @@ test('adds composed_of correctly', () => {
const template = getTemplate({
type: 'logs',
templateName: 'name',
templateIndexPattern: 'name-*',
packageName: 'nginx',
mappings: { properties: {} },
composedOfTemplates,
templatePriority: 200,
});
expect(template.composed_of).toStrictEqual(composedOfTemplates);
});
@ -53,35 +61,36 @@ test('adds empty composed_of correctly', () => {
const template = getTemplate({
type: 'logs',
templateName: 'name',
templateIndexPattern: 'name-*',
packageName: 'nginx',
mappings: { properties: {} },
composedOfTemplates,
templatePriority: 200,
});
expect(template.composed_of).toStrictEqual(composedOfTemplates);
});
test('adds hidden field correctly', () => {
const templateWithHiddenName = 'logs-nginx-access-abcd';
const templateIndexPattern = 'logs-nginx.access-abcd-*';
const templateWithHidden = getTemplate({
type: 'logs',
templateName: templateWithHiddenName,
templateIndexPattern,
packageName: 'nginx',
mappings: { properties: {} },
composedOfTemplates: [],
templatePriority: 200,
hidden: true,
});
expect(templateWithHidden.data_stream.hidden).toEqual(true);
const templateWithoutHiddenName = 'logs-nginx-access-efgh';
const templateWithoutHidden = getTemplate({
type: 'logs',
templateName: templateWithoutHiddenName,
templateIndexPattern,
packageName: 'nginx',
mappings: { properties: {} },
composedOfTemplates: [],
templatePriority: 200,
});
expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
});
@ -95,10 +104,11 @@ test('tests loading base.yml', () => {
const mappings = generateMappings(processedFields);
const template = getTemplate({
type: 'logs',
templateName: 'foo',
templateIndexPattern: 'foo-*',
packageName: 'nginx',
mappings,
composedOfTemplates: [],
templatePriority: 200,
});
expect(template).toMatchSnapshot(path.basename(ymlPath));
@ -113,10 +123,11 @@ test('tests loading coredns.logs.yml', () => {
const mappings = generateMappings(processedFields);
const template = getTemplate({
type: 'logs',
templateName: 'foo',
templateIndexPattern: 'foo-*',
packageName: 'coredns',
mappings,
composedOfTemplates: [],
templatePriority: 200,
});
expect(template).toMatchSnapshot(path.basename(ymlPath));
@ -131,10 +142,11 @@ test('tests loading system.yml', () => {
const mappings = generateMappings(processedFields);
const template = getTemplate({
type: 'metrics',
templateName: 'whatsthis',
templateIndexPattern: 'whatsthis-*',
packageName: 'system',
mappings,
composedOfTemplates: [],
templatePriority: 200,
});
expect(template).toMatchSnapshot(path.basename(ymlPath));
@ -520,3 +532,62 @@ test('tests constant_keyword field type handling', () => {
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping));
});
test('tests priority and index pattern for data stream without dataset_is_prefix', () => {
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
} as RegistryDataStream;
const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixUnset = 200;
const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixUnset);
const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixUnset);
expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixUnset);
expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixUnset);
});
test('tests priority and index pattern for data stream with dataset_is_prefix set to false', () => {
const dataStreamDatasetIsPrefixFalse = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
dataset_is_prefix: false,
} as RegistryDataStream;
const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixFalse = 200;
const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixFalse);
const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixFalse);
expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixFalse);
expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixFalse);
});
test('tests priority and index pattern for data stream with dataset_is_prefix set to true', () => {
const dataStreamDatasetIsPrefixTrue = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
dataset_is_prefix: true,
} as RegistryDataStream;
const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*';
const templatePriorityDatasetIsPrefixTrue = 150;
const templateIndexPattern = generateTemplateIndexPattern(dataStreamDatasetIsPrefixTrue);
const templatePriority = getTemplatePriority(dataStreamDatasetIsPrefixTrue);
expect(templateIndexPattern).toEqual(templateIndexPatternDatasetIsPrefixTrue);
expect(templatePriority).toEqual(templatePriorityDatasetIsPrefixTrue);
});

55
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file

@ -33,6 +33,10 @@ export interface CurrentDataStream {
const DEFAULT_SCALING_FACTOR = 1000;
const DEFAULT_IGNORE_ABOVE = 1024;
// see discussion in https://github.com/elastic/kibana/issues/88307
const DEFAULT_TEMPLATE_PRIORITY = 200;
const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150;
/**
* getTemplate retrieves the default template but overwrites the index pattern with the given value.
*
@ -40,29 +44,32 @@ const DEFAULT_IGNORE_ABOVE = 1024;
*/
export function getTemplate({
type,
templateName,
templateIndexPattern,
mappings,
pipelineName,
packageName,
composedOfTemplates,
templatePriority,
ilmPolicy,
hidden,
}: {
type: string;
templateName: string;
templateIndexPattern: string;
mappings: IndexTemplateMappings;
pipelineName?: string | undefined;
packageName: string;
composedOfTemplates: string[];
templatePriority: number;
ilmPolicy?: string | undefined;
hidden?: boolean;
}): IndexTemplate {
const template = getBaseTemplate(
type,
templateName,
templateIndexPattern,
mappings,
packageName,
composedOfTemplates,
templatePriority,
ilmPolicy,
hidden
);
@ -242,6 +249,35 @@ export function generateTemplateName(dataStream: RegistryDataStream): string {
return getRegistryDataStreamAssetBaseName(dataStream);
}
export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string {
// undefined or explicitly set to false
// See also https://github.com/elastic/package-spec/pull/102
if (!dataStream.dataset_is_prefix) {
return getRegistryDataStreamAssetBaseName(dataStream) + '-*';
} else {
return getRegistryDataStreamAssetBaseName(dataStream) + '.*-*';
}
}
// Template priorities are discussed in https://github.com/elastic/kibana/issues/88307
// See also https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html
//
// Built-in templates like logs-*-* and metrics-*-* have priority 100
//
// EPM generated templates for data streams have priority 200 (DEFAULT_TEMPLATE_PRIORITY)
//
// EPM generated templates for data streams with dataset_is_prefix: true have priority 150 (DATASET_IS_PREFIX_TEMPLATE_PRIORITY)
export function getTemplatePriority(dataStream: RegistryDataStream): number {
// undefined or explicitly set to false
// See also https://github.com/elastic/package-spec/pull/102
if (!dataStream.dataset_is_prefix) {
return DEFAULT_TEMPLATE_PRIORITY;
} else {
return DATASET_IS_PREFIX_TEMPLATE_PRIORITY;
}
}
/**
* Returns a map of the data stream path fields to elasticsearch index pattern.
* @param dataStreams an array of RegistryDataStream objects
@ -255,17 +291,18 @@ export function generateESIndexPatterns(
const patterns: Record<string, string> = {};
for (const dataStream of dataStreams) {
patterns[dataStream.path] = generateTemplateName(dataStream) + '-*';
patterns[dataStream.path] = generateTemplateIndexPattern(dataStream);
}
return patterns;
}
function getBaseTemplate(
type: string,
templateName: string,
templateIndexPattern: string,
mappings: IndexTemplateMappings,
packageName: string,
composedOfTemplates: string[],
templatePriority: number,
ilmPolicy?: string | undefined,
hidden?: boolean
): IndexTemplate {
@ -279,13 +316,9 @@ function getBaseTemplate(
};
return {
// This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*)
// if this number is lower than the ES value (which is 100) this template will never be applied when a data stream
// is created. I'm using 200 here to give some room for users to create their own template and fit it between the
// default and the one the ingest manager uses.
priority: 200,
priority: templatePriority,
// To be completed with the correct index patterns
index_patterns: [`${templateName}-*`],
index_patterns: [templateIndexPattern],
template: {
settings: {
index: {

2
x-pack/test/fleet_api_integration/apis/epm/index.js
View file

@ -11,7 +11,7 @@ export default function loadTests({ loadTestFile }) {
loadTestFile(require.resolve('./setup'));
loadTestFile(require.resolve('./get'));
loadTestFile(require.resolve('./file'));
//loadTestFile(require.resolve('./template'));
loadTestFile(require.resolve('./template'));
loadTestFile(require.resolve('./ilm'));
loadTestFile(require.resolve('./install_by_upload'));
loadTestFile(require.resolve('./install_overrides'));

21
x-pack/test/fleet_api_integration/apis/epm/template.ts
View file

@ -10,8 +10,8 @@ import { FtrProviderContext } from '../../../api_integration/ftr_provider_contex
import { getTemplate } from '../../../../plugins/fleet/server/services/epm/elasticsearch/template/template';
export default function ({ getService }: FtrProviderContext) {
const indexPattern = 'foo';
const templateName = 'bar';
const templateIndexPattern = 'bar-*';
const es = getService('es');
const mappings = {
properties: {
@ -25,27 +25,36 @@ export default function ({ getService }: FtrProviderContext) {
it('can be loaded', async () => {
const template = getTemplate({
type: 'logs',
templateName,
templateIndexPattern,
mappings,
packageName: 'system',
composedOfTemplates: [],
templatePriority: 200,
});
// This test is not an API integration test with Kibana
// We want to test here if the template is valid and for this we need a running ES instance.
// If the ES instance takes the template, we assume it is a valid template.
const { body: response1 } = await es.indices.putTemplate({
name: templateName,
const { body: response1 } = await es.transport.request({
method: 'PUT',
path: `/_index_template/${templateName}`,
body: template,
});
// Checks if template loading worked as expected
expect(response1).to.eql({ acknowledged: true });
const { body: response2 } = await es.indices.getTemplate({ name: templateName });
const { body: response2 } = await es.transport.request({
method: 'GET',
path: `/_index_template/${templateName}`,
});
// Checks if the content of the template that was loaded is as expected
// We already know based on the above test that the template was valid
// but we check here also if we wrote the index pattern inside the template as expected
expect(response2[templateName].index_patterns).to.eql([`${indexPattern}-*`]);
expect(response2.index_templates[0].index_template.index_patterns).to.eql([
templateIndexPattern,
]);
});
});
}