maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #7700 from Bargs/painless

Browse source 
Set language for scripted field
This commit is contained in:
Matt Bargar 2016-08-29 12:12:02 -04:00 committed by GitHub
commit 5bbe02e66a
19 changed files with 280 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/core_plugins/kibana/index.js
View file

@ -1,6 +1,7 @@
import ingest from './server/routes/api/ingest';
import search from './server/routes/api/search';
import settings from './server/routes/api/settings';
import scripts from './server/routes/api/scripts';
module.exports = function (kibana) {
return new kibana.Plugin({
@ -84,6 +85,7 @@ module.exports = function (kibana) {
ingest(server);
search(server);
settings(server);
scripts(server);
}
});

2
src/core_plugins/kibana/public/management/sections/indices/_scripted_fields.js
View file

@ -25,6 +25,7 @@ uiModules.get('apps/management')
$scope.perPage = 25;
$scope.columns = [
{ title: 'name' },
{ title: 'lang' },
{ title: 'script' },
{ title: 'format' },
{ title: 'controls', sortable: false }
@ -46,6 +47,7 @@ uiModules.get('apps/management')
return [
_.escape(field.name),
_.escape(field.lang),
_.escape(field.script),
_.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']),
{

5
src/core_plugins/kibana/server/routes/api/scripts/index.js Normal file
View file

@ -0,0 +1,5 @@
import { registerLanguages } from './register_languages';
export default function (server) {
registerLanguages(server);
}

27
src/core_plugins/kibana/server/routes/api/scripts/register_languages.js Normal file
View file

@ -0,0 +1,27 @@
import _ from 'lodash';
import handleESError from '../../../lib/handle_es_error';
export function registerLanguages(server) {
server.route({
path: '/api/kibana/scripts/languages',
method: 'GET',
handler: function (request, reply) {
const callWithRequest = server.plugins.elasticsearch.callWithRequest;
return callWithRequest(request, 'cluster.getSettings', {
include_defaults: true,
filter_path: '**.script.engine.*.inline'
})
.then((esResponse) => {
const langs = _.get(esResponse, 'defaults.script.engine', {});
const inlineLangs = _.pick(langs, (lang) => lang.inline === 'true');
const supportedLangs = _.omit(inlineLangs, 'mustache');
return _.keys(supportedLangs);
})
.then(reply)
.catch((error) => {
reply(handleESError(error));
});
}
});
}

1
src/fixtures/logstash_fields.js
View file

@ -22,6 +22,7 @@ function stubbedLogstashFields() {
{ name: 'custom_user_field', type: 'conflict', indexed: false, analyzed: false, sortable: false, filterable: true },
{ name: 'script string', type: 'string', scripted: true, script: '\'i am a string\'', lang: 'expression' },
{ name: 'script number', type: 'number', scripted: true, script: '1234', lang: 'expression' },
{ name: 'script date', type: 'date', scripted: true, script: '1234', lang: 'painless' },
{ name: 'script murmur3', type: 'murmur3', scripted: true, script: '1234', lang: 'expression'},
].map(function (field) {
field.count = field.count || 0;

2
src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js
View file

@ -35,7 +35,7 @@ describe('AggConfig Filters', function () {
interval = interval || 'auto';
if (interval === 'custom') interval = agg.params.customInterval;
duration = duration || moment.duration(15, 'minutes');
field = _.sample(indexPattern.fields.byType.date);
field = _.sample(_.reject(indexPattern.fields.byType.date, 'scripted'));
vis = new Vis(indexPattern, {
type: 'histogram',
aggs: [

8
src/ui/public/documentation_links/documentation_links.js
View file

@ -14,5 +14,13 @@ export default {
elasticsearchOutputAnchorParameters: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/elasticsearch-output.html#_parameters`,
startup: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/_step_5_starting_filebeat.html`,
exportedFields: `${baseUrl}guide/en/beats/filebeat/${urlVersion}/exported-fields.html`
},
scriptedFields: {
scriptFields: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/search-request-script-fields.html`,
scriptAggs: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/search-aggregations.html#_values_source`,
painless: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless.html`,
painlessApi: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless.html#painless-api`,
painlessSyntax: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-painless-syntax.html`,
luceneExpressions: `${baseUrl}guide/en/elasticsearch/reference/${urlVersion}/modules-scripting-expression.html`
}
};

59
src/ui/public/field_editor/__tests__/field_editor.js
View file

@ -4,6 +4,8 @@ import expect from 'expect.js';
import IndexPatternsFieldProvider from 'ui/index_patterns/_field';
import RegistryFieldFormatsProvider from 'ui/registry/field_formats';
import FixturesStubbedLogstashIndexPatternProvider from 'fixtures/stubbed_logstash_index_pattern';
import _ from 'lodash';
describe('FieldEditor directive', function () {
let Field;
@ -14,8 +16,15 @@ describe('FieldEditor directive', function () {
let $scope;
let $el;
let $httpBackend;
beforeEach(ngMock.module('kibana'));
beforeEach(ngMock.inject(function ($compile, $injector, Private) {
$httpBackend = $injector.get('$httpBackend');
$httpBackend
.when('GET', '/api/kibana/scripts/languages')
.respond(['expression', 'painless']);
$rootScope = $injector.get('$rootScope');
Field = Private(IndexPatternsFieldProvider);
StringFormat = Private(RegistryFieldFormatsProvider).getType('string');
@ -127,6 +136,56 @@ describe('FieldEditor directive', function () {
});
});
describe('scripted fields', function () {
let editor;
let field;
beforeEach(function () {
$rootScope.field = $rootScope.indexPattern.fields.byName['script string'];
compile();
editor = $scope.editor;
field = editor.field;
});
it('has a scripted flag set to true', function () {
expect(field.scripted).to.be(true);
});
it('contains a lang param', function () {
expect(field).to.have.property('lang');
expect(field.lang).to.be('expression');
});
it('provides lang options based on what is enabled for inline use in ES', function () {
$httpBackend.flush();
expect(_.isEqual(editor.scriptingLangs, ['expression', 'painless'])).to.be.ok();
});
it('provides curated type options based on language', function () {
$rootScope.$apply();
expect(editor.fieldTypes).to.have.length(1);
expect(editor.fieldTypes[0]).to.be('number');
editor.field.lang = 'painless';
$rootScope.$apply();
expect(editor.fieldTypes).to.have.length(4);
expect(_.isEqual(editor.fieldTypes, ['number', 'string', 'date', 'boolean'])).to.be.ok();
});
it('updates formatter options based on field type', function () {
field.lang = 'painless';
$rootScope.$apply();
expect(editor.field.type).to.be('string');
const stringFormats = editor.fieldFormatTypes;
field.type = 'date';
$rootScope.$apply();
expect(editor.fieldFormatTypes).to.not.be(stringFormats);
});
});
});
});

84
src/ui/public/field_editor/field_editor.html
View file

@ -1,4 +1,11 @@
<form ng-submit="editor.save()" name="form">
<div ng-if="editor.scriptingLangs.length === 0" class="hintbox">
<p>
<i class="fa fa-danger text-danger"></i>
<strong>Scripting disabled:</strong>
All inline scripting has been disabled in Elasticsearch. You must enable inline scripting for at least one language in order to use scripted fields in Kibana.
</p>
</div>
<div ng-if="editor.creating" class="form-group">
<label>Name</label>
<input
@ -17,9 +24,27 @@
</p>
</div>
<div ng-if="editor.field.scripted" class="form-group">
<label>Language</label>
<select
ng-model="editor.field.lang"
ng-options="lang as lang for lang in editor.scriptingLangs"
required
class="form-control">
<option value="">-- Select Language --</option>
</select>
</div>
<div class="form-group">
<label>Type</label>
<select
ng-if="editor.field.scripted"
ng-model="editor.field.type"
ng-options="type as type for type in editor.fieldTypes"
class="form-control">
</select>
<input
ng-if="!editor.field.scripted"
ng-model="editor.field.type"
readonly
class="form-control">
@ -86,15 +111,68 @@
<div ng-if="editor.field.scripted">
<div class="form-group">
<label>Script</label>
<textarea required class="form-control text-monospace" ng-model="editor.field.script"></textarea>
<textarea required class="field-editor_script-input form-control text-monospace" ng-model="editor.field.script"></textarea>
</div>
<div class="form-group">
<div ng-bind-html="editor.scriptingWarning" class="hintbox"></div>
<div class="hintbox">
<h4>
<i class="fa fa-warning text-warning"></i> Proceed with caution
</h4>
<p>
Please familiarize yourself with <a target="_window" ng-href="{{ editor.docLinks.scriptFields }}">script fields <i class="fa-link fa"></i></a> and with <a target="_window" ng-href="">scripts in aggregations <i class="fa-link fa"></i></a> before using scripted fields.
</p>
<p>
Scripted fields can be used to display and aggregate calculated values. As such, they can be very slow, and if done incorrectly, can cause Kibana to be unusable. There's no safety net here. If you make a typo, unexpected exceptions will be thrown all over the place!
</p>
</div>
</div>
<div class="form-group">
<div ng-bind-html="editor.scriptingInfo" class="hintbox"></div>
<div class="hintbox">
<h4>
<i class="fa fa-question-circle text-info"></i> Scripting Help
</h4>
<p>
By default, Kibana scripted fields use <a target="_window" ng-href="{{editor.docLinks.painless}}">Painless <i class="fa-link fa"></i></a>, a simple and secure scripting language designed specifically for use with Elasticsearch. To access values in the document use the following format:
</p>
<p><code>doc['some_field'].value</code></p>
<p>
Painless is powerful but easy to use. It provides access to many <a target="_window" ng-href="{{editor.docLinks.painlessApi}}">native Java APIs <i class="fa-link fa"></i></a>. Read up on its <a target="_window" ng-href="{{editor.docLinks.painlessSyntax}}">syntax <i class="fa-link fa"></i></a> and you'll be up to speed in no time!
</p>
<p>
Coming from an older version of Kibana? The <a target="_window" ng-href="{{editor.docLinks.luceneExpressions}}">Lucene Expressions <i class="fa-link fa"></i></a> you know and love are still available. Lucene expressions are a lot like JavaScript, but limited to basic arithmetic, bitwise and comparison operations.
</p>
<p>
There are a few limitations when using Lucene Expressions:
</p>
<ul>
<li> Only numeric, boolean, date, and geo_point fields may be accessed </li>
<li> Stored fields are not available </li>
<li> If a field is sparse (only some documents contain a value), documents missing the field will have a value of 0 </li>
</ul>
<p>
Here are all the operations available to lucene expressions:
</p>
<ul>
<li> Arithmetic operators: + - * / % </li>
<li> Bitwise operators: | & ^ ~ << >> >>> </li>
<li> Boolean operators (including the ternary operator): && || ! ?: </li>
<li> Comparison operators: < <= == >= > </li>
<li> Common mathematic functions: abs ceil exp floor ln log10 logn max min sqrt pow </li>
<li> Trigonometric library functions: acosh acos asinh asin atanh atan atan2 cosh cos sinh sin tanh tan </li>
<li> Distance functions: haversin </li>
<li> Miscellaneous functions: min, max </li>
</ul>
</div>
</div>
</div>

50
src/ui/public/field_editor/field_editor.js
View file

@ -6,15 +6,22 @@ import RegistryFieldFormatsProvider from 'ui/registry/field_formats';
import IndexPatternsFieldProvider from 'ui/index_patterns/_field';
import uiModules from 'ui/modules';
import fieldEditorTemplate from 'ui/field_editor/field_editor.html';
import chrome from 'ui/chrome';
import IndexPatternsCastMappingTypeProvider from 'ui/index_patterns/_cast_mapping_type';
import { scriptedFields as docLinks } from '../documentation_links/documentation_links';
import './field_editor.less';
uiModules
.get('kibana', ['colorpicker.module'])
.directive('fieldEditor', function (Private, $sce) {
let fieldFormats = Private(RegistryFieldFormatsProvider);
let Field = Private(IndexPatternsFieldProvider);
let scriptingInfo = $sce.trustAsHtml(require('ui/field_editor/scripting_info.html'));
let scriptingWarning = $sce.trustAsHtml(require('ui/field_editor/scripting_warning.html'));
const fieldTypesByLang = {
painless: ['number', 'string', 'date', 'boolean'],
expression: ['number'],
default: _.keys(Private(IndexPatternsCastMappingTypeProvider).types.byType)
};
return {
restrict: 'E',
@ -24,12 +31,17 @@ uiModules
getField: '&field'
},
controllerAs: 'editor',
controller: function ($scope, Notifier, kbnUrl) {
controller: function ($scope, Notifier, kbnUrl, $http, $q) {
let self = this;
let notify = new Notifier({ location: 'Field Editor' });
self.scriptingInfo = scriptingInfo;
self.scriptingWarning = scriptingWarning;
self.docLinks = docLinks;
getScriptingLangs().then((langs) => {
self.scriptingLangs = langs;
if (!_.includes(self.scriptingLangs, self.field.lang)) {
self.field.lang = undefined;
}
});
self.indexPattern = $scope.getIndexPattern();
self.field = shadowCopy($scope.getField());
@ -39,7 +51,6 @@ uiModules
self.creating = !self.indexPattern.fields.byName[self.field.name];
self.selectedFormatId = _.get(self.indexPattern, ['fieldFormatMap', self.field.name, 'type', 'id']);
self.defFormatType = initDefaultFormat();
self.fieldFormatTypes = [self.defFormatType].concat(fieldFormats.byFieldType[self.field.type] || []);
self.cancel = redirectAway;
self.save = function () {
@ -91,6 +102,23 @@ uiModules
self.field.format = new FieldFormat(self.formatParams);
}, true);
$scope.$watch('editor.field.type', function (newValue) {
self.defFormatType = initDefaultFormat();
self.fieldFormatTypes = [self.defFormatType].concat(fieldFormats.byFieldType[newValue] || []);
if (_.isUndefined(_.find(self.fieldFormatTypes, {id: self.selectedFormatId}))) {
delete self.selectedFormatId;
}
});
$scope.$watch('editor.field.lang', function (newValue) {
self.fieldTypes = _.get(fieldTypesByLang, newValue, fieldTypesByLang.default);
if (!_.contains(self.fieldTypes, self.field.type)) {
self.field.type = _.first(self.fieldTypes);
}
});
// copy the defined properties of the field to a plain object
// which is mutable, and capture the changed seperately.
function shadowCopy(field) {
@ -129,6 +157,14 @@ uiModules
else return fieldFormats.getDefaultType(self.field.type);
}
function getScriptingLangs() {
return $http.get(chrome.addBasePath('/api/kibana/scripts/languages'))
.then((res) => res.data)
.catch(() => {
return notify.error('Error getting available scripting languages from Elasticsearch');
});
}
function initDefaultFormat() {
let def = Object.create(fieldFormats.getDefaultType(self.field.type));

3
src/ui/public/field_editor/field_editor.less Normal file
View file

@ -0,0 +1,3 @@
textarea.field-editor_script-input {
height: 100px;
}

32
src/ui/public/field_editor/scripting_info.html
View file

@ -1,32 +0,0 @@
<h4>
<i class="fa fa-question-circle text-info"></i> Scripting Help
</h4>
<p>
By default, Elasticsearch scripts use <a target="_window" href="http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html#_lucene_expressions_scripts">Lucene Expressions <i class="fa-link fa"></i></a>, which is a lot like JavaScript, but limited to basic arithmetic, bitwise and comparison operations. We'll let you do some reading on <a target="_window" href="http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html#_lucene_expressions_scripts">Lucene Expressions<i class="fa-link fa"></i></a> To access values in the document use the following format:
</p>
<p><code>doc['some_field'].value</code></p>
<p>
There are a few limitations when using Lucene Expressions:
</p>
<ul>
<li>Only numeric fields may be accessed</li>
<li> Stored fields are not available </li>
<li> If a field is sparse (only some documents contain a value), documents missing the field will have a value of 0 </li>
</ul>
<p>
Here are all the operations available to scripted fields:
</p>
<ul>
<li> Arithmetic operators: + - * / % </li>
<li> Bitwise operators: | & ^ ~ << >> >>> </li>
<li> Boolean operators (including the ternary operator): && || ! ?: </li>
<li> Comparison operators: < <= == >= > </li>
<li> Common mathematic functions: abs ceil exp floor ln log10 logn max min sqrt pow </li>
<li> Trigonometric library functions: acosh acos asinh asin atanh atan atan2 cosh cos sinh sin tanh tan </li>
<li> Distance functions: haversin </li>
<li> Miscellaneous functions: min, max </li>
</ul>

11
src/ui/public/field_editor/scripting_warning.html
View file

@ -1,11 +0,0 @@
<h4>
<i class="fa fa-warning text-warning"></i> Proceed with caution
</h4>
<p>
Please familiarize yourself with <a target="_window" href="http://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-script-fields.html#search-request-script-fields">script fields <i class="fa-link fa"></i></a> and with <a target="_window" href="http://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-script">scripts in aggregations <i class="fa-link fa"></i></a> before using scripted fields.
</p>
<p>
Scripted fields can be used to display and aggregate calculated values. As such, they can be very slow, and if done incorrectly, can cause Kibana to be unusable. There's no safety net here. If you make a typo, unexpected exceptions will be thrown all over the place!
</p>

5
src/ui/public/index_patterns/__tests__/_get_computed_fields.js
View file

@ -32,8 +32,11 @@ describe('get computed fields', function () {
it('should request date fields as docvalue_fields', function () {
expect(fn().docvalueFields).to.contain('@timestamp');
expect(fn().docvalueFields).to.not.include.keys('bytes');
expect(fn().docvalueFields).to.not.contain('bytes');
});
it('should not request scripted date fields as docvalue_fields', function () {
expect(fn().docvalueFields).to.not.contain('script date');
});
});

2
src/ui/public/index_patterns/_field.js
View file

@ -50,7 +50,7 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope
// scripted objs
obj.fact('scripted', scripted);
obj.writ('script', scripted ? spec.script : null);
obj.writ('lang', scripted ? (spec.lang || 'expression') : null);
obj.writ('lang', scripted ? (spec.lang || 'painless') : null);
// mapping info
obj.fact('indexed', indexed);

2
src/ui/public/index_patterns/_get_computed_fields.js
View file

@ -6,7 +6,7 @@ export default function () {
let scriptFields = {};
let docvalueFields = [];
docvalueFields = _.pluck(self.fields.byType.date, 'name');
docvalueFields = _.map(_.reject(self.fields.byType.date, 'scripted'), 'name');
_.each(self.getScriptedFields(), function (field) {
scriptFields[field.name] = {

3
test/intern_api.js
View file

@ -1,7 +1,8 @@
define({
suites: [
'test/unit/api/ingest/index',
'test/unit/api/search/index'
'test/unit/api/search/index',
'test/unit/api/scripts/index'
],
excludeInstrumentation: /(fixtures|node_modules)\//,
loaderOptions: {

27
test/unit/api/scripts/_languages.js Normal file
View file

@ -0,0 +1,27 @@
define(function (require) {
var expect = require('intern/dojo/node!expect.js');
return function (bdd, request) {
bdd.describe('Languages API', function getLanguages() {
bdd.it('should return 200 with an array of languages', function () {
return request.get('/kibana/scripts/languages')
.expect(200)
.then(function (response) {
expect(response.body).to.be.an('array');
});
});
bdd.it('should only return langs enabled for inline scripting', function () {
return request.get('/kibana/scripts/languages')
.expect(200)
.then(function (response) {
expect(response.body).to.contain('expression');
expect(response.body).to.contain('painless');
expect(response.body).to.not.contain('groovy');
});
});
});
};
});

13
test/unit/api/scripts/index.js Normal file
View file

@ -0,0 +1,13 @@
define(function (require) {
var bdd = require('intern!bdd');
var serverConfig = require('intern/dojo/node!../../../server_config');
var request = require('intern/dojo/node!supertest-as-promised');
var url = require('intern/dojo/node!url');
var languages = require('./_languages');
bdd.describe('scripts API', function () {
request = request(url.format(serverConfig.servers.kibana) + '/api');
languages(bdd, request);
});
});