Remove the concept of a template resource from the index pattern API

This commit is contained in:
Matthew Bargar 2016-01-04 15:52:57 -05:00
parent bc0406157d
commit 3fae90fa0b
14 changed files with 220 additions and 275 deletions

View file

@ -0,0 +1,31 @@
const {templateToPattern, patternToTemplate} = require('../convert_pattern_and_template_name');
const expect = require('expect.js');
describe('convertPatternAndTemplateName', function () {
describe('templateToPattern', function () {
it('should convert an index template\'s name to its matching index pattern\'s title', function () {
expect(templateToPattern('kibana-logstash-*')).to.be('logstash-*');
});
it('should throw an error if the template name isn\'t a valid kibana namespaced name', function () {
expect(templateToPattern).withArgs('logstash-*').to.throwException('not a valid kibana namespaced template name');
expect(templateToPattern).withArgs('').to.throwException(/not a valid kibana namespaced template name/);
});
});
describe('patternToTemplate', function () {
it('should convert an index pattern\'s title to its matching index template\'s name', function () {
expect(patternToTemplate('logstash-*')).to.be('kibana-logstash-*');
});
it('should throw an error if the pattern is empty', function () {
expect(patternToTemplate).withArgs('').to.throwException(/pattern must not be empty/);
});
});
});

View file

@ -0,0 +1,51 @@
const createMappingFromPatternField = require('../create_mapping_from_pattern_field');
const expect = require('expect.js');
const _ = require('lodash');
let testField;
describe('createMappingFromPatternField', function () {
beforeEach(function () {
testField = {
'name': 'ip',
'type': 'ip',
'count': 2,
'scripted': false
};
});
it('should throw an error if the argument is empty', function () {
expect(createMappingFromPatternField).to.throwException(/argument must not be empty/);
});
it('should not modify the original argument', function () {
const testFieldClone = _.cloneDeep(testField);
const mapping = createMappingFromPatternField(testField);
expect(mapping).to.not.be(testField);
expect(_.isEqual(testField, testFieldClone)).to.be.ok();
});
it('should remove kibana properties that are not valid for ES field mappings', function () {
const mapping = createMappingFromPatternField(testField);
expect(mapping).to.not.have.property('count');
expect(mapping).to.not.have.property('scripted');
expect(mapping).to.not.have.property('indexed');
expect(mapping).to.not.have.property('analyzed');
});
it('should set doc_values and indexed status based on the relevant kibana properties if they exist', function () {
testField.indexed = true;
testField.analyzed = false;
testField.doc_values = true;
let mapping = createMappingFromPatternField(testField);
expect(mapping).to.have.property('doc_values', true);
expect(mapping).to.have.property('index', 'not_analyzed');
testField.analyzed = true;
mapping = createMappingFromPatternField(testField);
expect(mapping).to.have.property('index', 'analyzed');
});
});

View file

@ -0,0 +1,22 @@
// To avoid index template naming collisions the index pattern creation API
// namespaces template names by prepending 'kibana-' to the matching pattern's title.
// e.g. a pattern with title `logstash-*` will have a matching template named `kibana-logstash-*`.
// This module provides utility functions for easily converting between template and pattern names.
module.exports = {
templateToPattern: (templateName) => {
if (templateName.indexOf('kibana-') === -1) {
throw new Error('not a valid kibana namespaced template name');
}
return templateName.slice(templateName.indexOf('-') + 1);
},
patternToTemplate: (patternName) => {
if (patternName === '') {
throw new Error('pattern must not be empty');
}
return `kibana-${patternName.toLowerCase()}`;
}
};

View file

@ -0,0 +1,28 @@
const _ = require('lodash');
// Creates an ES field mapping from a single field object in a kibana index pattern
module.exports = function createMappingFromPatternField(field) {
if (_.isEmpty(field)) {
throw new Error('argument must not be empty');
}
const mapping = _.cloneDeep(field);
delete mapping.count;
delete mapping.scripted;
delete mapping.indexed;
delete mapping.analyzed;
if (field.indexed === false) {
mapping.index = 'no';
}
else {
if (field.analyzed === false) {
mapping.index = 'not_analyzed';
}
else if (field.analyzed === true) {
mapping.index = 'analyzed';
}
}
return mapping;
};

View file

@ -11,20 +11,17 @@ const indexPatternResourceObject = createResourceObjectSchema(
fields: Joi.array().items(
Joi.object({
name: Joi.string().required(),
type: Joi.string().required(),
count: Joi.number().integer(),
scripted: Joi.boolean(),
doc_values: Joi.boolean(),
analyzed: Joi.boolean(),
indexed: Joi.boolean(),
type: Joi.string(),
script: Joi.string(),
lang: Joi.string()
})
),
).required(),
field_format_map: Joi.object()
}),
Joi.object({
template: relationshipObjectSchema
})
);
@ -33,44 +30,6 @@ module.exports = {
Joi.alternatives().try(
indexPatternResourceObject,
Joi.array().items(indexPatternResourceObject)
).required(),
Joi.array().items(
createResourceObjectSchema(
Joi.object({
template: Joi.string().required(),
order: Joi.number().integer(),
mappings: Joi.object()
}).unknown()
)
)
),
// No attributes are required for an update
// Templates can't be updated in an index_pattern PUT
put: createApiDocumentSchema(
createResourceObjectSchema(
Joi.object({
title: Joi.string(),
time_field_name: Joi.string(),
interval_name: Joi.string(),
fields: Joi.array().items(
Joi.object({
name: Joi.string().required(),
count: Joi.number().integer(),
scripted: Joi.boolean(),
doc_values: Joi.boolean(),
analyzed: Joi.boolean(),
indexed: Joi.boolean(),
type: Joi.string(),
script: Joi.string(),
lang: Joi.string()
})
),
field_format_map: Joi.object()
}),
Joi.object({
template: relationshipObjectSchema
})
).required()
)
};

View file

@ -1,11 +1,10 @@
const { convertToSnakeCase } = require('../../../lib/case_conversion');
const _ = require('lodash');
const createApiDocument = require('../../../lib/api_document_builders/create_api_document');
const createRelationshipObject = require('../../../lib/api_document_builders/create_relationship_object');
const createResourceObject = require('../../../lib/api_document_builders/create_resource_object');
module.exports = function getIndexPattern(patternId, boundCallWithRequest, shouldIncludeTemplate) {
module.exports = function getIndexPattern(patternId, boundCallWithRequest) {
const params = {
index: '.kibana',
type: 'index-pattern',
@ -21,28 +20,7 @@ module.exports = function getIndexPattern(patternId, boundCallWithRequest, shoul
result._source.fieldFormatMap = JSON.parse(result._source.fieldFormatMap);
}
let relationshipsObject;
if (result._source.templateId) {
relationshipsObject = {
template: createRelationshipObject('index_templates', result._source.templateId)
};
delete result._source.templateId;
}
const snakeAttributes = convertToSnakeCase(result._source);
return createResourceObject('index_patterns', result._id, snakeAttributes, relationshipsObject);
})
.then((patternResource) => {
if (!shouldIncludeTemplate) {
return createApiDocument(patternResource);
}
const templateId = _.get(patternResource, 'relationships.template.data.id');
return boundCallWithRequest('indices.getTemplate', {name: templateId})
.then((template) => {
return createApiDocument(patternResource, [
createResourceObject('index_templates', templateId, template[templateId])
]);
});
return createApiDocument(createResourceObject('index_patterns', result._id, snakeAttributes));
});
};

View file

@ -1,11 +1,10 @@
const { convertToSnakeCase } = require('../../../lib/case_conversion');
const _ = require('lodash');
const createApiDocument = require('../../../lib/api_document_builders/create_api_document');
const createRelationshipObject = require('../../../lib/api_document_builders/create_relationship_object');
const createResourceObject = require('../../../lib/api_document_builders/create_resource_object');
const Promise = require('bluebird');
module.exports = function getIndexPatterns(boundCallWithRequest, shouldIncludeTemplate) {
module.exports = function getIndexPatterns(boundCallWithRequest) {
const params = {
index: '.kibana',
type: 'index-pattern',
@ -27,40 +26,11 @@ module.exports = function getIndexPatterns(boundCallWithRequest, shouldIncludeTe
patternHit._source.fieldFormatMap = JSON.parse(patternHit._source.fieldFormatMap);
}
let relationshipsObject;
if (patternHit._source.templateId) {
relationshipsObject = {
template: createRelationshipObject('index_templates', patternHit._source.templateId)
};
delete patternHit._source.templateId;
}
const snakeAttributes = convertToSnakeCase(patternHit._source);
return createResourceObject('index_patterns', patternHit._id, snakeAttributes, relationshipsObject);
return createResourceObject('index_patterns', patternHit._id, snakeAttributes);
});
})
.then((patterns) => {
if (!shouldIncludeTemplate) {
return createApiDocument(patterns);
}
const templateIdSet = new Set();
patterns.forEach(pattern => {
const templateId = _.get(pattern, 'relationships.template.data.id');
if (templateId) {
templateIdSet.add(templateId);
}
});
const commaDelimitedTemplateIds = Array.from(templateIdSet).join(',');
return boundCallWithRequest('indices.getTemplate', {name: commaDelimitedTemplateIds})
.then((templates) => {
return _.map(templates, (template, templateId) => {
return createResourceObject('index_templates', templateId, template);
});
})
.then((templates) => {
return createApiDocument(patterns, templates);
});
return createApiDocument(patterns);
});
};

View file

@ -1,58 +1,33 @@
const _ = require('lodash');
const Promise = require('bluebird');
const handleESError = require('../../../lib/handle_es_error');
const getIndexPattern = require('./get_index_pattern');
const Boom = require('boom');
const {templateToPattern, patternToTemplate} = require('../../../lib/convert_pattern_and_template_name');
module.exports = function registerDelete(server) {
server.route({
path: '/api/kibana/index_patterns/{id}',
method: 'DELETE',
handler: function (req, reply) {
const boundCallWithRequest = _.partial(server.plugins.elasticsearch.callWithRequest, req);
const shouldIncludeTemplate = req.query.include === 'template';
const patternId = req.params.id;
const callWithRequest = server.plugins.elasticsearch.callWithRequest;
const deletePatternParams = {
index: '.kibana',
type: 'index-pattern',
id: patternId
id: req.params.id
};
let result;
if (shouldIncludeTemplate) {
result = getIndexPattern(patternId, boundCallWithRequest)
.then((patternResource) => {
const templateId = _.get(patternResource, 'data.relationships.template.data.id');
if (!templateId) {
return;
}
return boundCallWithRequest(
'indices.deleteTemplate',
{name: templateId}
)
.catch((error) => {
if (!error.status || error.status !== 404) {
throw error;
}
});
})
.then(() => {
return boundCallWithRequest('delete', deletePatternParams);
});
}
else {
result = boundCallWithRequest('delete', deletePatternParams);
}
result.then(
function () {
Promise.all([
callWithRequest(req, 'delete', deletePatternParams),
callWithRequest(req, 'indices.deleteTemplate', {name: patternToTemplate(req.params.id)})
.catch((error) => {
if (!error.status || error.status !== 404) {
throw error;
}
})
])
.then(function (pattern) {
reply('success');
},
function (error) {
}, function (error) {
reply(handleESError(error));
}
);
});
}
});
};

View file

@ -10,9 +10,8 @@ module.exports = function registerGet(server) {
method: 'GET',
handler: function (req, reply) {
const boundCallWithRequest = _.partial(server.plugins.elasticsearch.callWithRequest, req);
const shouldIncludeTemplate = req.query.include === 'template';
getIndexPatterns(boundCallWithRequest, shouldIncludeTemplate)
getIndexPatterns(boundCallWithRequest)
.then(
function (patterns) {
reply(patterns);
@ -29,10 +28,9 @@ module.exports = function registerGet(server) {
method: 'GET',
handler: function (req, reply) {
const boundCallWithRequest = _.partial(server.plugins.elasticsearch.callWithRequest, req);
const shouldIncludeTemplate = req.query.include === 'template';
const patternId = req.params.id;
getIndexPattern(patternId, boundCallWithRequest, shouldIncludeTemplate)
getIndexPattern(patternId, boundCallWithRequest)
.then(
function (pattern) {
reply(pattern);

View file

@ -1,9 +1,12 @@
const Boom = require('boom');
const _ = require('lodash');
const {templateToPattern, patternToTemplate} = require('../../../lib/convert_pattern_and_template_name');
const indexPatternSchema = require('../../../lib/schemas/resources/index_pattern_schema');
const handleESError = require('../../../lib/handle_es_error');
const addMappingInfoToPatternFields = require('../../../lib/add_mapping_info_to_pattern_fields');
const { convertToCamelCase } = require('../../../lib/case_conversion');
const createMappingFromPatternField = require('../../../lib/create_mapping_from_pattern_field');
const castMappingType = require('../../../lib/cast_mapping_type');
module.exports = function registerPost(server) {
server.route({
@ -17,15 +20,28 @@ module.exports = function registerPost(server) {
handler: function (req, reply) {
const callWithRequest = server.plugins.elasticsearch.callWithRequest;
const requestDocument = _.cloneDeep(req.payload);
const included = requestDocument.included;
const indexPatternId = requestDocument.data.id;
const indexPattern = convertToCamelCase(requestDocument.data.attributes);
const templateResource = _.isEmpty(included) ? null : included[0];
if (!_.isEmpty(templateResource)) {
addMappingInfoToPatternFields(indexPattern, templateResource.attributes);
indexPattern.templateId = templateResource.id;
}
_.forEach(indexPattern.fields, function (field) {
_.defaults(field, {
indexed: true,
analyzed: false,
doc_values: true,
scripted: false,
count: 0
});
});
const mappings = _(indexPattern.fields)
.indexBy('name')
.mapValues(createMappingFromPatternField)
.value();
_.forEach(indexPattern.fields, function (field) {
field.type = castMappingType(field.type);
});
indexPattern.fields = JSON.stringify(indexPattern.fields);
indexPattern.fieldFormatMap = JSON.stringify(indexPattern.fieldFormatMap);
@ -38,20 +54,24 @@ module.exports = function registerPost(server) {
callWithRequest(req, 'create', patternCreateParams)
.then((patternResponse) => {
if (_.isEmpty(included)) {
return patternResponse;
}
return callWithRequest(req, 'indices.exists', {index: indexPatternId})
.then((matchingIndices) => {
if (matchingIndices) {
throw Boom.conflict('Cannot create an index template if existing indices already match index pattern');
throw Boom.conflict('Cannot create an index pattern via this API if existing indices already match the pattern');
}
const templateParams = {
order: 0,
create: true,
name: templateResource.id,
body: templateResource.attributes
name: patternToTemplate(indexPatternId),
body: {
template: indexPatternId,
mappings: {
_default_: {
properties: mappings
}
}
}
};
return callWithRequest(req, 'indices.putTemplate', templateParams);

View file

@ -13,7 +13,7 @@ define(function (require) {
return scenarioManager.reload('emptyKibana')
.then(function () {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.send(createTestData().indexPattern)
.expect(201);
});
});
@ -31,7 +31,7 @@ define(function (require) {
});
bdd.it('should return 200 for successful deletion of pattern and template', function () {
return request.del('/kibana/index_patterns/logstash-*?include=template')
return request.del('/kibana/index_patterns/logstash-*')
.expect(200)
.then(function () {
return request.get('/kibana/index_patterns/logstash-*').expect(404);
@ -44,22 +44,8 @@ define(function (require) {
});
});
bdd.it('should not delete the template if the include parameter is not sent', function () {
return request.del('/kibana/index_patterns/logstash-*')
.expect(200)
.then(function () {
return request.get('/kibana/index_patterns/logstash-*').expect(404);
})
.then(function () {
return scenarioManager.client.indices.getTemplate({name: 'kibana-logstash-*'})
.then(function (res) {
expect(res['kibana-logstash-*']).to.be.ok();
});
});
});
bdd.it('should return 404 for a non-existent id', function () {
return request.del('/kibana/index_patterns/doesnotexist?include=template')
return request.del('/kibana/index_patterns/doesnotexist')
.expect(404);
});

View file

@ -19,21 +19,17 @@ define(function (require) {
bdd.before(function () {
return scenarioManager.reload('emptyKibana').then(function () {
return Promise.all([
request.post('/kibana/index_patterns').send(createTestData().indexPatternWithTemplate),
request.post('/kibana/index_patterns').send(createTestData().indexPattern),
request.post('/kibana/index_patterns').send(
_(createTestData().indexPatternWithTemplate)
_(createTestData().indexPattern)
.set('data.attributes.title', 'foo')
.set('data.id', 'foo')
.set('included[0].id', 'kibana-foo')
.set('included[0].attributes.template', 'foo')
.value()
),
request.post('/kibana/index_patterns').send(
_(createTestData().indexPatternWithTemplate)
_(createTestData().indexPattern)
.set('data.attributes.title', 'bar*')
.set('data.id', 'bar*')
.set('included[0].id', 'kibana-bar*')
.set('included[0].attributes.template', 'bar*')
.value()
)
]).then(function () {
@ -46,9 +42,9 @@ define(function (require) {
bdd.after(function () {
return Promise.all([
request.del('/kibana/index_patterns/logstash-*?include=template'),
request.del('/kibana/index_patterns/foo?include=template'),
request.del('/kibana/index_patterns/bar*?include=template')
request.del('/kibana/index_patterns/logstash-*'),
request.del('/kibana/index_patterns/foo'),
request.del('/kibana/index_patterns/bar*')
]);
});
@ -63,16 +59,6 @@ define(function (require) {
});
});
bdd.it('should include related index templates if the include query string param is set', function () {
return request.get('/kibana/index_patterns?include=template')
.expect(200)
.then(function (res) {
expect(res.body.included).to.be.an('array');
expect(res.body.included.length).to.be(3);
Joi.assert(res.body, indexPatternSchema.post);
});
});
bdd.it('should use snake_case in the response body', function () {
return request.get('/kibana/index_patterns')
.expect(200)
@ -95,16 +81,6 @@ define(function (require) {
});
});
bdd.it('should include related index template if the include query string param is set', function () {
return request.get('/kibana/index_patterns/logstash-*?include=template')
.expect(200)
.then(function (res) {
expect(res.body.data.attributes.title).to.be('logstash-*');
expect(res.body.included[0].id).to.be('kibana-logstash-*');
Joi.assert(res.body, indexPatternSchema.post);
});
});
bdd.it('should use snake_case in the response body', function () {
return request.get('/kibana/index_patterns/logstash-*')
.expect(200)

View file

@ -12,7 +12,7 @@ define(function (require) {
});
bdd.afterEach(function () {
return request.del('/kibana/index_patterns/logstash-*?include=template');
return request.del('/kibana/index_patterns/logstash-*');
});
bdd.it('should return 400 for an invalid payload', function invalidPayload() {
@ -24,61 +24,42 @@ define(function (require) {
.expect(400),
request.post('/kibana/index_patterns')
.send(_.set(createTestData().indexPatternWithTemplate, 'data.attributes.title', false))
.send(_.set(createTestData().indexPattern, 'data.attributes.title', false))
.expect(400),
request.post('/kibana/index_patterns')
.send(_.set(createTestData().indexPatternWithTemplate, 'data.attributes.fields', {}))
.send(_.set(createTestData().indexPattern, 'data.attributes.fields', {}))
.expect(400),
// Fields must have a name
// Fields must have a name and type
request.post('/kibana/index_patterns')
.send(_.set(createTestData().indexPatternWithTemplate, 'data.attributes.fields', [{count: 0}]))
.send(_.set(createTestData().indexPattern, 'data.attributes.fields', [{count: 0}]))
.expect(400)
]);
});
bdd.it('should return 201 when a pattern is successfully created', function createPattern() {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.send(createTestData().indexPattern)
.expect(201);
});
bdd.it('should create an index template if a template is included', function createTemplate() {
bdd.it('should create an index template if a fields array is included', function createTemplate() {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.send(createTestData().indexPattern)
.expect(201)
.then(function () {
return scenarioManager.client.indices.getTemplate({name: 'kibana-logstash-*'});
});
});
bdd.it('should normalize field mappings and add them to the index pattern if a template is included', function () {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.expect(201)
.then(function () {
return request.get('/kibana/index_patterns/logstash-*')
.expect(200)
.then(function (res) {
_.forEach(res.body.data.attributes.fields, function (field) {
expect(field).to.have.keys('type', 'indexed', 'analyzed', 'doc_values');
if (field.name === 'bytes') {
expect(field).to.have.property('type', 'number');
}
});
});
});
});
bdd.it('should return 409 conflict when a pattern with the given ID already exists', function patternConflict() {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.send(createTestData().indexPattern)
.expect(201)
.then(function () {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.send(createTestData().indexPattern)
.expect(409);
});
});
@ -91,7 +72,7 @@ define(function (require) {
}
}).then(function () {
return request.post('/kibana/index_patterns')
.send(createTestData().indexPatternWithTemplate)
.send(createTestData().indexPattern)
.expect(409);
})
.then(function () {
@ -101,29 +82,18 @@ define(function (require) {
});
});
bdd.it('should return 409 conflict when a template is included with a pattern that matches existing indices',
bdd.it('should return 409 conflict when the pattern matches existing indices',
function existingIndicesConflict() {
var pattern = createTestData().indexPatternWithTemplate;
pattern.data.id = pattern.data.attributes.title = pattern.included[0].attributes.template = '.kib*';
var pattern = createTestData().indexPattern;
pattern.data.id = pattern.data.attributes.title = '.kib*';
return request.post('/kibana/index_patterns')
.send(pattern)
.expect(409);
});
bdd.it('should return 201 created successfully if a pattern matches existing indices but no template is included',
function existingIndicesNoTemplate() {
var pattern = createTestData().indexPatternWithTemplate;
pattern.data.id = pattern.data.attributes.title = '.kib*';
delete pattern.included;
return request.post('/kibana/index_patterns')
.send(pattern)
.expect(201);
});
bdd.it('should enforce snake_case in the request body', function () {
var pattern = createTestData().indexPatternWithTemplate;
var pattern = createTestData().indexPattern;
pattern.data.attributes = _.mapKeys(pattern.data.attributes, function (value, key) {
return _.camelCase(key);
});

View file

@ -1,6 +1,6 @@
module.exports = function createTestData() {
return {
indexPatternWithTemplate: {
indexPattern: {
'data': {
'type': 'index_patterns',
'id': 'logstash-*',
@ -9,46 +9,27 @@ module.exports = function createTestData() {
'time_field_name': '@timestamp',
'fields': [{
'name': 'ip',
'type': 'ip',
'count': 2,
'scripted': false
}, {
'name': '@timestamp',
'type': 'date',
'count': 0,
'scripted': false
}, {
'name': 'agent',
'type': 'string',
'count': 0,
'scripted': false
}, {
'name': 'bytes',
'type': 'long',
'count': 2,
'scripted': false
}]
},
'relationships': {
'template': {
'data': {'type': 'index_templates', 'id': 'kibana-logstash-*'}
}
}
},
'included': [{
'type': 'index_templates',
'id': 'kibana-logstash-*',
'attributes': {
'template': 'logstash-*',
'order': 0,
'mappings': {
'_default_': {
'properties': {
'ip': {'type': 'ip', 'index': 'not_analyzed', 'doc_values': true},
'@timestamp': {'type': 'date', 'index': 'not_analyzed', 'doc_values': true},
'agent': {'type': 'string', 'index': 'analyzed', 'doc_values': false},
'bytes': {'type': 'long', 'index': 'not_analyzed', 'doc_values': true}
}
}
}
}
}]
}
}
};
};