Filter non-aggregatable fields from vis editor UI

Using the field_capabilities API added in the previous commit, this commit enhances
the client side index pattern object with information about the
searchable and aggregatable status of each field in the index pattern.
We then use this information to filter out non-aggregatable fields from
the vis editor so that users won't accidentally select them and get
nasty errors. An example of a non-aggregatable field would be a `text`
field without fielddata enabled (which is the default).

I also added the searchable and aggregatable flags to the index pattern
page so users can see the status of their fields. I removed the `indexed`
column because it was mostly redundant with `searchable` and I needed
the horizontal space.

The addition of the searchable and aggregatable properties for index
pattern fields would require users to manually refresh their field list
when upgrading to 5.0. This commit also adds a check for those properties and
if they're missing it automatically refreshes the field list for the
user in a seamless manner.

src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js

@@ -23,8 +23,9 @@ uiModules.get('apps/management')
         { title: 'name' },
         { title: 'type' },
         { title: 'format' },
+        { title: 'searchable', info: 'These fields can be used in the filter bar' },
+        { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
         { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
-        { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' },
         { title: 'controls', sortable: false }
       ];
 
@@ -57,12 +58,16 @@ uiModules.get('apps/management')
             },
             _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']),
             {
-              markup: field.analyzed ? yesTemplate : noTemplate,
-              value: field.analyzed
+              markup: field.searchable ? yesTemplate : noTemplate,
+              value: field.searchable
             },
             {
-              markup: field.indexed ? yesTemplate : noTemplate,
-              value: field.indexed
+              markup: field.aggregatable ? yesTemplate : noTemplate,
+              value: field.aggregatable
+            },
+            {
+              markup: field.analyzed ? yesTemplate : noTemplate,
+              value: field.analyzed
             },
             {
               markup: controlsHtml,

src/core_plugins/kibana/public/visualize/editor/agg_params.js

@@ -138,7 +138,7 @@ uiModules
       }
 
       function getIndexedFields(param) {
-        let fields = $scope.agg.vis.indexPattern.fields.raw;
+        let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable');
         const fieldTypes = param.filterFieldTypes;
 
         if (fieldTypes) {

src/ui/public/index_patterns/__tests__/_index_pattern.js

@@ -43,6 +43,9 @@ describe('index pattern', function () {
     sinon.stub(mapper, 'getFieldsForIndexPattern', function () {
       return Promise.resolve(_.filter(mockLogstashFields, { scripted: false }));
     });
+    sinon.stub(mapper, 'clearCache', function () {
+      return Promise.resolve();
+    });
 
     // stub mappingSetup
     mappingSetup = Private(UtilsMappingSetupProvider);
@@ -151,15 +154,34 @@ describe('index pattern', function () {
     const indexPatternId = 'test-pattern';
     let indexPattern;
     let fieldLength;
-    let truncatedFields;
+    let customFields;
 
     beforeEach(function () {
       fieldLength = mockLogstashFields.length;
-      truncatedFields = mockLogstashFields.slice(3);
+      customFields = [{
+        analyzed: true,
+        count: 30,
+        filterable: true,
+        indexed: true,
+        name: 'foo',
+        scripted: false,
+        sortable: true,
+        type: 'number',
+        aggregatable: true,
+        searchable: false
+      },
+      {
+        name: 'script number',
+        type: 'number',
+        scripted: true,
+        script: '1234',
+        lang: 'expression'
+      }];
+
       return create(indexPatternId, {
         _source: {
           customFormats: '{}',
-          fields: JSON.stringify(truncatedFields)
+          fields: JSON.stringify(customFields)
         }
       }).then(function (pattern) {
         indexPattern = pattern;
@@ -168,8 +190,8 @@ describe('index pattern', function () {
 
     it('should fetch fields from the doc source', function () {
       // ensure that we don't have all the fields
-      expect(truncatedFields.length).to.not.equal(mockLogstashFields.length);
-      expect(indexPattern.fields).to.have.length(truncatedFields.length);
+      expect(customFields.length).to.not.equal(mockLogstashFields.length);
+      expect(indexPattern.fields).to.have.length(customFields.length);
 
       // ensure that all fields will be included in the returned docSource
       setDocsourcePayload(docSourceResponse(indexPatternId));
@@ -201,9 +223,8 @@ describe('index pattern', function () {
         // called to append scripted fields to the response from mapper.getFieldsForIndexPattern
         expect(scriptedFieldsSpy.callCount).to.equal(1);
 
-        const scripted = _.where(mockLogstashFields, { scripted: true });
-        const expected = _.filter(indexPattern.fields, { scripted: true });
-        expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name'));
+        const expected = _.filter(indexPattern.fields, {scripted: true});
+        expect(_.pluck(expected, 'name')).to.eql(['script number']);
       });
     });
   });

src/ui/public/index_patterns/_enhance_fields_with_capabilities.js

@@ -0,0 +1,16 @@
+import chrome from 'ui/chrome';
+import _ from 'lodash';
+
+export default function ($http) {
+
+  return function (fields, indices) {
+    return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`))
+    .then((res) => {
+      const stats = _.get(res, 'data.fields', {});
+
+      return _.map(fields, (field) => {
+        return _.assign(field, stats[field.name]);
+      });
+    });
+  };
+}

src/ui/public/index_patterns/_field.js

@@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope
     obj.fact('analyzed', !!spec.analyzed);
     obj.fact('doc_values', !!spec.doc_values);
 
+    // stats
+    obj.fact('searchable', !!spec.searchable);
+    obj.fact('aggregatable', !!spec.aggregatable);
+
     // usage flags, read-only and won't be saved
     obj.comp('format', format);
     obj.comp('sortable', sortable);

src/ui/public/index_patterns/_index_pattern.js

@@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
     // give index pattern all of the values in _source
     _.assign(indexPattern, response._source);
 
-    indexFields(indexPattern);
+    const promise = indexFields(indexPattern);
 
     // any time index pattern in ES is updated, update index pattern object
     docSources
     .get(indexPattern)
     .onUpdate()
     .then(response => updateFromElasticSearch(indexPattern, response), notify.fatal);
+
+    return promise;
+  }
+
+  function containsFieldCapabilities(fields) {
+    return _.any(fields, (field) => {
+      return _.has(field, 'aggregatable') && _.has(field, 'searchable');
+    });
   }
 
   function indexFields(indexPattern) {
+    let promise = Promise.resolve();
+
     if (!indexPattern.id) {
-      return;
+      return promise;
     }
-    if (!indexPattern.fields) {
-      return indexPattern.refreshFields();
+
+    if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) {
+      promise = indexPattern.refreshFields();
     }
-    initFields(indexPattern);
+    return promise.then(() => {initFields(indexPattern);});
   }
 
   function setId(indexPattern, id) {

src/ui/public/index_patterns/_mapper.js

@@ -1,12 +1,14 @@
 import { IndexPatternMissingIndices } from 'ui/errors';
 import _ from 'lodash';
 import moment from 'moment';
+import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities';
 import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields';
 import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals';
 import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard';
 import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache';
 export default function MapperService(Private, Promise, es, config, kbnIndex) {
 
+  let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider);
   let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider);
   let intervals = Private(IndexPatternsIntervalsProvider);
   let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider);
@@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
         });
       }
 
-      let promise = Promise.resolve(id);
+      let indexList = id;
+      let promise = Promise.resolve();
       if (indexPattern.intervalName) {
         promise = self.getIndicesForIndexPattern(indexPattern)
         .then(function (existing) {
           if (existing.matches.length === 0) throw new IndexPatternMissingIndices();
-          return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
+          indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
         });
       }
 
-      return promise.then(function (indexList) {
+      return promise.then(function () {
         return es.indices.getFieldMapping({
           index: indexList,
           fields: '*',
@@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
       })
       .catch(handleMissingIndexPattern)
       .then(transformMappingIntoFields)
+      .then(fields => enhanceFieldsWithCapabilities(fields, indexList))
       .then(function (fields) {
         fieldCache.set(id, fields);
         return fieldCache.get(id);