Merge branch 'master' into feature/ingest

CONTRIBUTING.md

@@ -55,7 +55,7 @@ Please make sure you have signed the [Contributor License Agreement](http://www.
   npm run elasticsearch
   ```
 
-- Start the development server.
+- Start the development server. _On Windows, you'll need you use Git Bash, Cygwin, or a similar shell that exposes the `sh` command._
 
   ```sh
   npm start
@@ -146,7 +146,8 @@ Run the tests for just your particular plugin. Assuming you plugin lives outside
 
 #### Running browser automation tests:
 
-*The Selenium server that is started currently only runs the tests in Firefox*
+*The Selenium server that is started currently only runs the tests in a recent version of Firefox.*
+*You can use the `PATH` environment variable to specify which version of Firefox to use.*
 
 The following will start Kibana, Elasticsearch and Selenium for you. To run the functional UI tests use the following commands
 
@@ -177,7 +178,7 @@ npm run test:ui:runner
 - These tests have been developed and tested with Chrome and Firefox browser.  In theory, they should work on all browsers (that's the benefit of Intern using Leadfoot).
 - These tests should also work with an external testing service like https://saucelabs.com/ or https://www.browserstack.com/ but that has not been tested.
 - https://theintern.github.io/
-- https://theintern.github.io/leadfoot/Element.html
+- https://theintern.github.io/leadfoot/module-leadfoot_Element.html
 
 #### Building OS packages
 

package.json

@@ -49,7 +49,7 @@
     "test:coverage": "grunt test:coverage",
     "build": "grunt build",
     "build:ospackages": "grunt build --os-packages",
-    "start": "./bin/kibana --dev",
+    "start": "sh ./bin/kibana --dev",
     "precommit": "grunt precommit",
     "karma": "karma start",
     "elasticsearch": "grunt esvm:dev:keepalive",
@@ -180,6 +180,7 @@
     "makelogs": "3.0.0-beta3",
     "marked-text-renderer": "0.1.0",
     "mocha": "2.3.0",
+    "ncp": "2.0.0",
     "nock": "2.10.0",
     "npm": "2.11.0",
     "portscanner": "1.0.0",

src/cli/serve/read_yaml_config.js

@@ -48,7 +48,14 @@ module.exports = function (path) {
       _.forOwn(val, function (subVal, subKey) {
         apply(config, subVal, key + '.' + subKey);
       });
-    } else {
+    }
+    else if (_.isArray(val)) {
+      config[key] = [];
+      val.forEach((subVal, i) => {
+        apply(config, subVal, key + '.' + i);
+      });
+    }
+    else {
       _.set(config, key, val);
     }
   }

src/plugins/kibana/server/lib/__tests__/create_mappings_from_pattern_fields.js

@@ -39,21 +39,21 @@ describe('createMappingsFromPatternFields', function () {
     let mappings = createMappingsFromPatternFields(testFields);
 
     _.forEach(mappings, function (mapping) {
-      if (mapping.type !== 'string') {
+      if (mapping.type !== 'text') {
         expect(_.isEqual(mapping, {
           type: mapping.type,
-          index: 'not_analyzed',
+          index: true,
           doc_values: true
         })).to.be.ok();
       }
     });
   });
 
-  it('should give strings a multi-field mapping', function () {
+  it('should give strings a multi-field mapping with a "text" base type', function () {
     let mappings = createMappingsFromPatternFields(testFields);
 
     _.forEach(mappings, function (mapping) {
-      if (mapping.type === 'string') {
+      if (mapping.type === 'text') {
         expect(mapping).to.have.property('fields');
       }
     });
@@ -68,7 +68,7 @@ describe('createMappingsFromPatternFields', function () {
     expect(mappings.geo.properties).to.have.property('coordinates');
     expect(_.isEqual(mappings.geo.properties.coordinates, {
       type: 'geo_point',
-      index: 'not_analyzed',
+      index: true,
       doc_values: true
     })).to.be.ok();
   });

src/plugins/kibana/server/lib/create_mappings_from_pattern_fields.js

@@ -13,10 +13,9 @@ module.exports = function createMappingsFromPatternFields(fields) {
 
     if (field.type === 'string') {
       mapping = {
-        type: 'string',
+        type: 'text',
-        index: 'analyzed',
         fields: {
-          raw: {type: 'string', index: 'not_analyzed', doc_values: true, ignore_above: 256}
+          raw: {type: 'keyword', ignore_above: 256}
         }
       };
     }
@@ -24,7 +23,7 @@ module.exports = function createMappingsFromPatternFields(fields) {
       const fieldType = field.type === 'number' ? 'double' : field.type;
       mapping = {
         type: fieldType,
-        index: 'not_analyzed',
+        index: true,
         doc_values: true
       };
     }

src/plugins/kibana/server/routes/api/ingest/register_post.js

@@ -101,10 +101,9 @@ module.exports = function registerPost(server) {
                   match: '*',
                   match_mapping_type: 'string',
                   mapping: {
-                    type: 'string',
+                    type: 'text',
-                    index: 'analyzed',
                     fields: {
-                      raw: {type: 'string', index: 'not_analyzed', doc_values: true, ignore_above: 256}
+                      raw: {type: 'keyword', ignore_above: 256}
                     }
                   }
                 }

src/server/status/__tests__/status.js

@@ -4,7 +4,6 @@ import Status from '../status';
 import ServerStatus from '../server_status';
 
 describe('Status class', function () {
-
   var server;
   var serverStatus;
 
@@ -59,6 +58,34 @@ describe('Status class', function () {
     expect(json.message).to.eql('Ready');
   });
 
+  it('should call on handler if status is already matched', function (done) {
+    var status = serverStatus.create('test');
+    var msg = 'Test Ready';
+    status.green(msg);
+
+    status.on('green', function (prev, prevMsg) {
+      expect(arguments.length).to.equal(2);
+      expect(prev).to.be('green');
+      expect(prevMsg).to.be(msg);
+      expect(status.message).to.equal(msg);
+      done();
+    });
+  });
+
+  it('should call once handler if status is already matched', function (done) {
+    var status = serverStatus.create('test');
+    var msg = 'Test Ready';
+    status.green(msg);
+
+    status.once('green', function (prev, prevMsg) {
+      expect(arguments.length).to.equal(2);
+      expect(prev).to.be('green');
+      expect(prevMsg).to.be(msg);
+      expect(status.message).to.equal(msg);
+      done();
+    });
+  });
+
   function testState(color) {
     it(`should change the state to ${color} when #${color}() is called`, function () {
       var status = serverStatus.create('test');

src/server/status/status.js

@@ -36,6 +36,22 @@ class Status extends EventEmitter {
       since: this.since
     };
   }
+
+  on(eventName, handler) {
+    super.on(eventName, handler);
+
+    if (eventName === this.state) {
+      setImmediate(() => handler(this.state, this.message));
+    }
+  }
+
+  once(eventName, handler) {
+    if (eventName === this.state) {
+      setImmediate(() => handler(this.state, this.message));
+    } else {
+      super.once(eventName, handler);
+    }
+  }
 }
 
 states.all.forEach(function (state) {

src/ui/public/chrome/api/__tests__/nav.js

@@ -36,5 +36,10 @@ describe('chrome nav apis', function () {
       const chrome = getChrome();
       expect(chrome.addBasePath('http://github.com/elastic/kibana')).to.be('http://github.com/elastic/kibana');
     });
+
+    it('includes the query string', function () {
+      const chrome = getChrome();
+      expect(chrome.addBasePath('/app/kibana?a=b')).to.be(`${basePath}/app/kibana?a=b`);
+    });
   });
 });

src/ui/public/chrome/api/nav.js

@@ -18,7 +18,7 @@ export default function (chrome, internals) {
     var isUrl = url && isString(url);
     if (!isUrl) return url;
 
-    var parsed = parse(url);
+    var parsed = parse(url, true);
     if (!parsed.host && parsed.pathname) {
       if (parsed.pathname[0] === '/') {
         parsed.pathname = chrome.getBasePath() + parsed.pathname;

src/ui/public/notify/__tests__/notifier.js

@@ -43,8 +43,8 @@ describe('Notifier', function () {
       expect(notify('error').title).to.equal('Error');
     });
 
-    it('sets lifetime to Infinity', function () {
+    it('sets lifetime to 5 minutes', function () {
-      expect(notify('error').lifetime).to.equal(Infinity);
+      expect(notify('error').lifetime).to.equal(300000);
     });
 
     it('allows reporting', function () {

src/ui/public/notify/notifier.js

@@ -229,7 +229,7 @@ Notifier.prototype.error = function (err, cb) {
     content: formatMsg(err, this.from),
     icon: 'warning',
     title: 'Error',
-    lifetime: Infinity,
+    lifetime: 300000,
     actions: ['report', 'accept'],
     stack: formatStack(err)
   }, cb);

src/ui/public/notify/partials/toaster.html

@@ -16,6 +16,13 @@
             ng-class="'btn-' + notif.type"
             ng-click="notif.showStack = true"
             >More Info</button>
+          <button
+            type="button"
+            ng-if="notif.stack && notif.showStack"
+            class="btn"
+            ng-class="'btn-' + notif.type"
+            ng-click="notif.showStack = false"
+            >Less Info</button>
           <button
             type="button"
             ng-if="notif.accept"

tasks/build/index.js

@@ -22,6 +22,7 @@ module.exports = function (grunt) {
       'stop:optimizeBuild',
       '_build:downloadNodeBuilds:finish',
       '_build:versionedLinks',
+      '_build:osShellScripts',
       '_build:archives',
       grunt.option('os-packages') ? [
         '_build:pleaseRun',

tasks/build/os_shell_scripts.js

@@ -0,0 +1,42 @@
+import {join, extname} from 'path';
+import {promisify} from 'bluebird';
+import {ncp} from 'ncp';
+import rimraf from 'rimraf';
+const pncp = promisify(ncp);
+const primraf = promisify(rimraf);
+
+export default function (grunt) {
+  grunt.registerTask('_build:osShellScripts', async function osShellScripts() {
+    const done = this.async();
+    const source = 'build/kibana/bin';
+    const platforms = grunt.config.get('platforms');
+    const allPlatforms = fn => invokeAllAsync(platforms, fn);
+
+    try {
+      await allPlatforms(platform => primraf(join(platform.buildDir, 'bin')));
+      await allPlatforms(platform => pncp(source, join(platform.buildDir, 'bin')));
+      await allPlatforms(platform => removeExtraneousShellScripts(grunt, platform));
+      done();
+    } catch (err) {
+      done(err);
+    }
+  });
+};
+
+function invokeAllAsync(all, fn) {
+  return Promise.all(all.map(fn));
+}
+
+function removeExtraneousShellScripts(grunt, platform) {
+  return Promise.all(grunt.file
+    .expand(join(platform.buildDir, 'bin', '*'))
+    .filter(file => isExtraneous(platform, file))
+    .map(file => primraf(file)));
+}
+
+function isExtraneous(platform, file) {
+  const ext = extname(file);
+  if (platform.win && ext === '') { return true; }
+  if (!platform.win && ext === '.bat') { return true; }
+  return false;
+}

test/functional/apps/visualize/_area_chart.js

@@ -150,7 +150,7 @@ define(function (require) {
             expect(labels).to.eql(yAxisLabels);
           })
           .then(function getAreaChartData() {
-            return visualizePage.getAreaChartData();
+            return visualizePage.getAreaChartData('Count');
           })
           .then(function (paths) {
             common.debug('expectedAreaChartData = ' + expectedAreaChartData);

test/functional/apps/visualize/_line_chart.js

@@ -122,7 +122,7 @@ define(function (require) {
           // sleep a bit before trying to get the chart data
           return common.sleep(3000)
           .then(function () {
-            return visualizePage.getLineChartData()
+            return visualizePage.getLineChartData('fill="#57c17b"')
             .then(function showData(data) {
               var tolerance = 10; // the y-axis scale is 10000 so 10 is 0.1%
               for (var x = 0; x < data.length; x++) {

test/support/pages/visualize_page.js

@@ -327,19 +327,53 @@ define(function (require) {
       });
     },
 
-    // saved visualizations are paginated 5 to a page!
+
-    loadSavedVisualization: function loadSavedVisualization(vizName) {
+    clickLoadSavedVisButton: function clickLoadSavedVisButton() {
-      var self = this;
       return this.remote
       .setFindTimeout(defaultTimeout)
       .findByCssSelector('button.ng-scope[aria-label="Load Saved Visualization"]')
+      .click();
+    },
+
+    filterVisByName: function filterVisByName(vizName) {
+      return this.remote
+      .findByCssSelector('input[name="filter"]')
       .click()
-      .then(function findVizByLinkedText() {
+      // can't uses dashes in saved visualizations when filtering
-        common.debug('Load Saved Vis button clicked');
+      // or extended character sets
-        return self.remote
+      // https://github.com/elastic/kibana/issues/6300
-        .setFindTimeout(defaultTimeout)
+      .type(vizName.replace('-',' '));
-        .findByLinkText(vizName)
+    },
-        .click();
+
+    clickVisualizationByLinkText: function clickVisualizationByLinkText(vizName) {
+      var self = this;
+      common.debug('clickVisualizationByLinkText(' + vizName + ')');
+      return this.remote
+      .setFindTimeout(defaultTimeout)
+      .findByLinkText(vizName)
+      .click();
+    },
+
+    // this starts by clicking the Load Saved Viz button, not from the
+    // bottom half of the "Create a new visualization      Step 1" page
+    loadSavedVisualization: function loadSavedVisualization(vizName) {
+      var self = this;
+      return this.clickLoadSavedVisButton()
+      .then(function filterVisualization() {
+        return self.openSavedVisualization(vizName);
+      });
+    },
+
+    // this is for starting on the
+    // bottom half of the "Create a new visualization      Step 1" page
+    openSavedVisualization: function openSavedVisualization(vizName) {
+      var self = this;
+      return self.filterVisByName(vizName)
+      .then(function () {
+        return common.sleep(1000);
+      })
+      .then(function clickDashboardByLinkedText() {
+        return self.clickVisualizationByLinkText(vizName);
       });
     },
 
@@ -385,7 +419,7 @@ define(function (require) {
      ** This method gets the chart data and scales it based on chart height and label.
      ** Returns an array of height values
      */
-    getAreaChartData: function getAreaChartData() {
+    getAreaChartData: function getAreaChartData(aggregateName) {
 
       var self = this.remote;
       var chartData = [];
@@ -400,11 +434,11 @@ define(function (require) {
       return this.remote
       .setFindTimeout(defaultTimeout)
       .findByCssSelector('div.y-axis-div-wrapper > div > svg > g > g:last-of-type')
-      .then(function setYAxisLabel(y) {
+      .getVisibleText()
-        return y.getVisibleText();
-      })
       .then(function (yLabel) {
-        yAxisLabel = yLabel.replace(',', '');
+        // since we're going to use the y-axis 'last' (top) label as a number to
+        // scale the chart pixel data, we need to clean out commas and % marks.
+        yAxisLabel = yLabel.replace(/(%|,)/g, '');
         common.debug('yAxisLabel = ' + yAxisLabel);
         return yLabel;
       })
@@ -412,10 +446,8 @@ define(function (require) {
       .then(function () {
         return self
         .setFindTimeout(defaultTimeout)
-        .findByCssSelector('rect.background'); // different here
+        .findByCssSelector('rect.background') // different here
-      })
+        .getAttribute('height');
-      .then(function (chartAreaObj) {
-        return chartAreaObj.getAttribute('height');
       })
       .then(function (chartH) {
         yAxisHeight = chartH;
@@ -423,43 +455,29 @@ define(function (require) {
       })
       .then(function () {
         return self.setFindTimeout(defaultTimeout * 2)
-        .findAllByCssSelector('path')
+        .findByCssSelector('path[data-label="' + aggregateName + '"]')
-        .then(function (chartTypes) {
+        .getAttribute('d');
-
-          function getChartType(chart) {
-            return chart
-            .getAttribute('data-label')
-            .then(function (chartString) {
-              //common.debug('data-label  = ' + chartString);
-              if (chartString === 'Count') {
-                return chart.getAttribute('d')
-                .then(function (data) {
-                  common.debug(data);
-                  tempArray = data.split('L');
-                  chartSections = tempArray.length / 2;
-                  common.debug('chartSections = ' + chartSections + ' height = ' + yAxisHeight + ' yAxisLabel = ' + yAxisLabel);
-                  chartData[0] = Math.round((yAxisHeight - tempArray[0].split(',')[1]) / yAxisHeight * yAxisLabel);
-                  common.debug('chartData[0] =' + chartData[0]);
-                  for (var i = 1; i < chartSections; i++) {
-                    chartData[i] = Math.round((yAxisHeight - tempArray[i].split(',')[1]) / yAxisHeight * yAxisLabel);
-                    common.debug('chartData[i] =' + chartData[i]);
-                  }
-                  return chartData;
-                });
-              }
-            });
-          }
-          var getChartTypesPromises = chartTypes.map(getChartType);
-          return Promise.all(getChartTypesPromises);
-        });
       })
-      .then(function (chartData) {
+      .then(function (data) {
-        return chartData[1]; // MAGIC NUMBER - we find multiple 'path's and only one of them is the right one.
+        common.debug(data);
+        // This area chart data starts with a 'M'ove to a x,y location, followed
+        // by a bunch of 'L'ines from that point to the next.  Those points are
+        // the values we're going to use to calculate the data values we're testing.
+        // So git rid of the one 'M' and split the rest on the 'L's.
+        tempArray = data.replace('M','').split('L');
+        chartSections = tempArray.length / 2;
+        common.debug('chartSections = ' + chartSections + ' height = ' + yAxisHeight + ' yAxisLabel = ' + yAxisLabel);
+        for (var i = 0; i < chartSections; i++) {
+          chartData[i] = Math.round((yAxisHeight - tempArray[i].split(',')[1]) / yAxisHeight * yAxisLabel);
+          common.debug('chartData[i] =' + chartData[i]);
+        }
+        return chartData;
       });
     },
 
+
     // The current test shows dots, not a line.  This function gets the dots and normalizes their height.
-    getLineChartData: function getLineChartData() {
+    getLineChartData: function getLineChartData(cssPart) {
       var self = this.remote;
       var yAxisLabel = 0;
       var yAxisHeight;
@@ -468,10 +486,7 @@ define(function (require) {
       return this.remote
         .setFindTimeout(defaultTimeout)
         .findByCssSelector('div.y-axis-div-wrapper > div > svg > g > g:last-of-type')
-        .then(function setYAxisLabel(y) {
+        .getVisibleText()
-          return y
-          .getVisibleText();
-        })
         .then(function (yLabel) {
           yAxisLabel = yLabel.replace(',', '');
           common.debug('yAxisLabel = ' + yAxisLabel);
@@ -482,10 +497,7 @@ define(function (require) {
           return self
           .setFindTimeout(defaultTimeout)
           .findByCssSelector('clipPath rect')
-          .then(function getRectHeight(chartAreaObj) {
+          .getAttribute('height')
-            return chartAreaObj
-            .getAttribute('height');
-          })
           .then(function (theHeight) {
             yAxisHeight = theHeight - 5; // MAGIC NUMBER - clipPath extends a bit above the top of the y-axis and below x-axis
             common.debug('theHeight = ' + theHeight);
@@ -502,7 +514,7 @@ define(function (require) {
             // 5). for each chart element, find the green circle, then the cy position
             function getChartType(chart) {
               return chart
-              .findByCssSelector('circle[fill="#57c17b"]')
+              .findByCssSelector('circle[' + cssPart + ']')
               .then(function (circleObject) {
                 // common.debug('circleObject = ' + circleObject + ' yAxisHeight= ' + yAxisHeight + ' yAxisLabel= ' + yAxisLabel);
                 return circleObject
@@ -636,6 +648,13 @@ define(function (require) {
       .getVisibleText();
     },
 
+    getMarkdownData: function getMarkdownData() {
+      return this.remote
+      .setFindTimeout(defaultTimeout)
+      .findByCssSelector('visualize.ng-isolate-scope')
+      .getVisibleText();
+    },
+
     clickColumns: function clickColumns() {
       return this.remote
       .setFindTimeout(defaultTimeout)

test/unit/api/ingest/_post.js

@@ -12,7 +12,12 @@ define(function (require) {
       });
 
       bdd.afterEach(function () {
-        return request.del('/kibana/ingest/logstash-*');
+        return request.del('/kibana/ingest/logstash-*')
+        .then(function () {
+          return scenarioManager.client.indices.delete({
+            index: 'logstash-*'
+          });
+        });
       });
 
       bdd.it('should return 400 for an invalid payload', function invalidPayload() {
@@ -62,6 +67,29 @@ define(function (require) {
           });
       });
 
+      bdd.it('should successfully create new indices based on the template', function newIndices() {
+        return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204)
+          .then(function () {
+            return scenarioManager.client.create({
+              index: 'logstash-1',
+              type: 'foo',
+              id: '1',
+              body: {
+                ip: '192.168.1.1',
+                '@timestamp': '2015-09-20T10:28:22.684Z',
+                agent: 'Jack',
+                bytes: 9001,
+                geo: {coordinates: {lat: 43.07260861, lon: -92.61077833}}
+              }
+            })
+            .then(function (response) {
+              expect(response.created).to.be.ok();
+            });
+          });
+      });
+
       bdd.it('should provide defaults for field properties', function createTemplate() {
         return request.post('/kibana/ingest')
           .send(createTestData())
@@ -97,15 +125,15 @@ define(function (require) {
             .then(function (template) {
               var mappings = template['kibana-logstash-*'].mappings._default_.properties;
               expect(mappings).to.be.ok();
-              expect(_.isEqual(mappings.ip, {index: 'not_analyzed', type: 'ip', doc_values: true})).to.be.ok();
+              expect(_.isEqual(mappings.ip, {index: true, type: 'ip', doc_values: true})).to.be.ok();
-              expect(_.isEqual(mappings['@timestamp'], {index: 'not_analyzed', type: 'date', doc_values: true})).to.be.ok();
+              expect(_.isEqual(mappings['@timestamp'], {index: true, type: 'date', doc_values: true})).to.be.ok();
-              expect(_.isEqual(mappings.bytes, {index: 'not_analyzed', type: 'double', doc_values: true})).to.be.ok();
+              expect(_.isEqual(mappings.bytes, {index: true, type: 'double', doc_values: true})).to.be.ok();
 
               // object fields are mapped as such, with individual mappings for each of their properties
               expect(_.isEqual(mappings.geo, {
                 properties: {
                   coordinates: {
-                    index: 'not_analyzed',
+                    index: true,
                     type: 'geo_point',
                     doc_values: true
                   }