Merge pull request #5736 from epixa/5733-pre-fieldstats-optin

Search non-expandable, wildcard index patterns

src/plugins/kibana/public/settings/sections/indices/_create.html

@@ -73,6 +73,36 @@
               </small>
             </div>
 
+          <div class="form-group" ng-if="canExpandIndices()">
+            <label>
+              <input ng-model="index.notExpandable" type="checkbox">
+              Do not expand index pattern when searching <small>(Not recommended)</small>
+            </label>
+
+            <div ng-if="index.notExpandable" class="alert alert-info">
+              This index pattern will be queried directly rather than being
+              expanded into more performant searches against individual indices.
+
+              Elasticsearch will receive a query against <em>{{index.name}}</em>
+              and will have to search through all matching indices regardless
+              of whether they have data that matches the current time range.
+            </div>
+
+            <p class="help-block">
+              By default, searches against any time-based index pattern that
+              contains a wildcard will automatically be expanded to query only
+              the indices that contain data within the currently selected time
+              range.
+            </p>
+
+            <p class="help-block">
+              Searching against the index pattern <em>logstash-*</em> will
+              actually query elasticsearch for the specific matching indices
+              (e.g. <em>logstash-2015.12.21</em>) that fall within the current
+              time range.
+            </p>
+          </div>
+
           <section>
             <div class="alert alert-danger" ng-repeat="err in index.patternErrors">
               {{err}}

src/plugins/kibana/public/settings/sections/indices/_create.js

@@ -24,6 +24,7 @@ define(function (require) {
 
       isTimeBased: true,
       nameIsPattern: false,
+      notExpandable: false,
       sampleCount: 5,
       nameIntervalOptions: intervals,
 
@@ -33,6 +34,12 @@ define(function (require) {
     index.nameInterval = _.find(index.nameIntervalOptions, { name: 'daily' });
     index.timeField = null;
 
+    $scope.canExpandIndices = function () {
+      // to maximize performance in the digest cycle, move from the least
+      // expensive operation to most
+      return index.isTimeBased && !index.nameIsPattern && _.includes(index.name, '*');
+    };
+
     $scope.refreshFieldList = function () {
       fetchFieldList().then(updateFieldList);
     };
@@ -50,6 +57,10 @@ define(function (require) {
           }
         }
 
+        if (index.notExpandable && $scope.canExpandIndices()) {
+          indexPattern.notExpandable = true;
+        }
+
         // fetch the fields
         return indexPattern.create()
         .then(function (id) {

src/plugins/kibana/public/settings/sections/indices/_edit.html

@@ -22,6 +22,10 @@
         <div ng-if="indexPattern.timeFieldName && indexPattern.intervalName" class="alert alert-info">
           This index uses a <strong>Time-based index pattern</strong> which repeats <span ng-bind="::indexPattern.getInterval().display"></span>
         </div>
+        <div ng-if="!indexPattern.canExpandIndices()" class="alert alert-info">
+          This index pattern is set to be queried directly rather than being
+          expanded into more performant searches against individual indices.
+        </div>
         <div ng-if="conflictFields.length" class="alert alert-warning">
           <strong>Mapping conflict!</strong> {{conflictFields.length > 1 ? conflictFields.length : 'A'}} field{{conflictFields.length > 1 ? 's' : ''}} {{conflictFields.length > 1 ? 'are' : 'is'}} defined as several types (string, integer, etc) across the indices that match this pattern. You may still be able to use these conflict fields in parts of Kibana, but they will be unavailable for functions that require Kibana to know their type. Correcting this issue will require reindexing your data.
         </div>

src/ui/public/index_patterns/__tests__/_index_pattern.js

@@ -294,6 +294,84 @@ describe('index pattern', function () {
     });
   });
 
+  describe('#toDetailedIndexList', function () {
+    require('testUtils/noDigestPromises').activateForSuite();
+    context('when index pattern is an interval', function () {
+      var interval;
+      beforeEach(function () {
+        interval = 'result:getInterval';
+        sinon.stub(indexPattern, 'getInterval').returns(interval);
+      });
+
+      it('invokes interval toDetailedIndexList with given start/stop times', async function () {
+        await indexPattern.toDetailedIndexList(1, 2);
+        var id = indexPattern.id;
+        expect(intervals.toIndexList.calledWith(id, interval, 1, 2)).to.be(true);
+      });
+      it('is fulfilled by the result of interval toDetailedIndexList', async function () {
+        var indexList = await indexPattern.toDetailedIndexList();
+        expect(indexList[0].index).to.equal('foo');
+        expect(indexList[1].index).to.equal('bar');
+      });
+
+      context('with sort order', function () {
+        it('passes the sort order to the intervals module', function () {
+          return indexPattern.toDetailedIndexList(1, 2, 'SORT_DIRECTION')
+          .then(function () {
+            expect(intervals.toIndexList.callCount).to.be(1);
+            expect(intervals.toIndexList.getCall(0).args[4]).to.be('SORT_DIRECTION');
+          });
+        });
+      });
+    });
+
+    context('when index pattern is a time-base wildcard', function () {
+      beforeEach(function () {
+        sinon.stub(indexPattern, 'getInterval').returns(false);
+        sinon.stub(indexPattern, 'hasTimeField').returns(true);
+        sinon.stub(indexPattern, 'isWildcard').returns(true);
+      });
+
+      it('invokes calculateIndices with given start/stop times and sortOrder', async function () {
+        await indexPattern.toDetailedIndexList(1, 2, 'sortOrder');
+        var id = indexPattern.id;
+        var field = indexPattern.timeFieldName;
+        expect(calculateIndices.calledWith(id, field, 1, 2, 'sortOrder')).to.be(true);
+      });
+
+      it('is fulfilled by the result of calculateIndices', async function () {
+        var indexList = await indexPattern.toDetailedIndexList();
+        expect(indexList[0].index).to.equal('foo');
+        expect(indexList[1].index).to.equal('bar');
+      });
+    });
+
+    context('when index pattern is a time-base wildcard that is configured not to expand', function () {
+      beforeEach(function () {
+        sinon.stub(indexPattern, 'getInterval').returns(false);
+        sinon.stub(indexPattern, 'hasTimeField').returns(true);
+        sinon.stub(indexPattern, 'isWildcard').returns(true);
+        sinon.stub(indexPattern, 'canExpandIndices').returns(false);
+      });
+
+      it('is fulfilled by id', async function () {
+        var indexList = await indexPattern.toDetailedIndexList();
+        expect(indexList.index).to.equal(indexPattern.id);
+      });
+    });
+
+    context('when index pattern is neither an interval nor a time-based wildcard', function () {
+      beforeEach(function () {
+        sinon.stub(indexPattern, 'getInterval').returns(false);
+      });
+
+      it('is fulfilled by id', async function () {
+        var indexList = await indexPattern.toDetailedIndexList();
+        expect(indexList.index).to.equal(indexPattern.id);
+      });
+    });
+  });
+
   describe('#toIndexList', function () {
     context('when index pattern is an interval', function () {
       require('testUtils/noDigestPromises').activateForSuite();
@@ -348,6 +426,21 @@ describe('index pattern', function () {
       });
     });
 
+    context('when index pattern is a time-base wildcard that is configured not to expand', function () {
+      require('testUtils/noDigestPromises').activateForSuite();
+      beforeEach(function () {
+        sinon.stub(indexPattern, 'getInterval').returns(false);
+        sinon.stub(indexPattern, 'hasTimeField').returns(true);
+        sinon.stub(indexPattern, 'isWildcard').returns(true);
+        sinon.stub(indexPattern, 'canExpandIndices').returns(false);
+      });
+
+      it('is fulfilled by id', async function () {
+        var indexList = await indexPattern.toIndexList();
+        expect(indexList).to.equal(indexPattern.id);
+      });
+    });
+
     context('when index pattern is neither an interval nor a time-based wildcard', function () {
       beforeEach(function () {
         sinon.stub(indexPattern, 'getInterval').returns(false);
@@ -365,6 +458,21 @@ describe('index pattern', function () {
     });
   });
 
+  describe('#canExpandIndices()', function () {
+    it('returns true if notExpandable is false', function () {
+      indexPattern.notExpandable = false;
+      expect(indexPattern.canExpandIndices()).to.be(true);
+    });
+    it('returns true if notExpandable is not defined', function () {
+      delete indexPattern.notExpandable;
+      expect(indexPattern.canExpandIndices()).to.be(true);
+    });
+    it('returns false if notExpandable is true', function () {
+      indexPattern.notExpandable = true;
+      expect(indexPattern.canExpandIndices()).to.be(false);
+    });
+  });
+
   describe('#hasTimeField()', function () {
     beforeEach(function () {
       // for the sake of these tests, it doesn't much matter what type of field

src/ui/public/index_patterns/_index_pattern.js

@@ -25,6 +25,7 @@ define(function (require) {
     var mapping = mappingSetup.expandShorthand({
       title: 'string',
       timeFieldName: 'string',
+      notExpandable: 'boolean',
       intervalName: 'string',
       fields: 'json',
       fieldFormatMap: {
@@ -196,7 +197,7 @@ define(function (require) {
           return intervals.toIndexList(self.id, interval, start, stop, sortDirection);
         }
 
-        if (self.isWildcard() && self.hasTimeField()) {
+        if (self.isWildcard() && self.hasTimeField() && self.canExpandIndices()) {
           return calculateIndices(self.id, self.timeFieldName, start, stop, sortDirection);
         }
 
@@ -207,6 +208,10 @@ define(function (require) {
         };
       });
 
+      self.canExpandIndices = function () {
+        return !this.notExpandable;
+      };
+
       self.hasTimeField = function () {
         return !!(this.timeFieldName && this.fields.byName[this.timeFieldName]);
       };

src/ui/public/utils/__tests__/mapping_setup.js

@@ -0,0 +1,41 @@
+const _ = require('lodash');
+const ngMock = require('ngMock');
+const expect = require('expect.js');
+
+let mappingSetup;
+
+describe('ui/utils/mapping_setup', function () {
+  beforeEach(ngMock.module('kibana'));
+  beforeEach(ngMock.inject(function (Private) {
+    mappingSetup = Private(require('ui/utils/mapping_setup'));
+  }));
+
+  describe('#expandShorthand()', function () {
+    it('allows shortcuts for field types by just setting the value to the type name', function () {
+      const mapping = mappingSetup.expandShorthand({ foo: 'boolean' });
+      expect(mapping.foo.type).to.be('boolean');
+    });
+
+    it('can set type as an option', function () {
+      const mapping = mappingSetup.expandShorthand({ foo: {type: 'integer'} });
+      expect(mapping.foo.type).to.be('integer');
+    });
+
+    context('when type is json', function () {
+      it('returned object is type string', function () {
+        const mapping = mappingSetup.expandShorthand({ foo: 'json' });
+        expect(mapping.foo.type).to.be('string');
+      });
+
+      it('returned object has _serialize function', function () {
+        const mapping = mappingSetup.expandShorthand({ foo: 'json' });
+        expect(_.isFunction(mapping.foo._serialize)).to.be(true);
+      });
+
+      it('returned object has _deserialize function', function () {
+        const mapping = mappingSetup.expandShorthand({ foo: 'json' });
+        expect(_.isFunction(mapping.foo._serialize)).to.be(true);
+      });
+    });
+  });
+});