Merge pull request #4904 from jbudz/functional-testing

Functional testing

.gitignore

@@ -17,3 +17,4 @@ disabledPlugins
 webpackstats.json
 config/kibana.dev.yml
 coverage
+selenium

Gruntfile.js

@@ -78,7 +78,7 @@ module.exports = function (grunt) {
     init: true,
     config: config,
     loadGruntTasks: {
-      pattern: ['grunt-*', '@*/grunt-*', 'gruntify-*', '@*/gruntify-*']
+      pattern: ['grunt-*', '@*/grunt-*', 'gruntify-*', '@*/gruntify-*', 'intern']
     }
   });
 

package.json

@@ -37,6 +37,9 @@
     "test:dev": "grunt test:dev",
     "test:quick": "grunt test:quick",
     "test:browser": "grunt test:browser",
+    "test:ui": "grunt test:ui",
+    "test:ui:server": "grunt test:ui:server",
+    "test:ui:runner": "grunt test:ui:runner",
     "test:server": "grunt test:server",
     "test:coverage": "grunt test:coverage",
     "build": "grunt build",
@@ -140,12 +143,13 @@
     "grunt-contrib-copy": "0.8.1",
     "grunt-esvm": "1.1.8",
     "grunt-karma": "0.12.0",
-    "grunt-run": "0.4.0",
+    "grunt-run": "0.5.0",
     "grunt-s3": "0.2.0-alpha.3",
     "grunt-simple-mocha": "0.4.0",
     "gruntify-eslint": "1.0.1",
     "html-entities": "1.1.3",
     "husky": "0.8.1",
+    "intern": "3.0.1",
     "istanbul-instrumenter-loader": "0.1.3",
     "karma": "0.13.9",
     "karma-chrome-launcher": "0.2.0",

tasks/config/downloadSelenium.js

@@ -0,0 +1,15 @@
+var path = require('path');
+
+
+module.exports = function (grunt) {
+  return {
+    options: {
+      selenium: {
+        filename: 'selenium-server-standalone-2.47.1.jar',
+        server: 'https://selenium-release.storage.googleapis.com/2.47/',
+        md5: 'e6cb10b8f0f353c6ca4a8f62fb5cb472',
+        directory: path.join(grunt.config.get('root'), 'selenium')
+      }
+    }
+  };
+};

tasks/config/esvm.js

@@ -36,6 +36,17 @@ module.exports = function (grunt) {
         directory: resolve(directory, 'test'),
         purge: true
       }
+    },
+    ui: {
+      options: {
+        directory: resolve(directory, 'test'),
+        purge: true,
+        config: {
+          http: {
+            port: 9220
+          }
+        }
+      }
     }
   };
 };

tasks/config/intern.js

@@ -0,0 +1,13 @@
+var path = require('path');
+
+
+module.exports = function (grunt) {
+  return {
+    options: {
+      runType: 'runner',
+      config: 'test/intern',
+      reporters: ['Console']
+    },
+    dev: {}
+  };
+};

tasks/config/loadFixtures.js

@@ -0,0 +1,11 @@
+var path = require('path');
+
+
+module.exports = function (grunt) {
+  return {
+    options: {
+      server: 'http://localhost:9220',
+      dataDir: path.join(grunt.config.get('root'), 'test/fixtures')
+    }
+  };
+};

tasks/config/run.js

@@ -22,6 +22,21 @@ module.exports = function (grunt) {
       ]
     },
 
+    testUIServer: {
+      options: {
+        wait: false,
+        ready: /Server running/,
+        quiet: false,
+        failOnError: false
+      },
+      cmd: /^win/.test(platform) ? '.\\bin\\kibana.bat' : './bin/kibana',
+      args: [
+        '--server.port=5620',
+        '--elasticsearch.url=http://localhost:9220',
+        '--logging.json=false'
+      ]
+    },
+
     testCoverageServer: {
       options: {
         wait: false,
@@ -60,6 +75,34 @@ module.exports = function (grunt) {
       ]
     },
 
+    seleniumServer: {
+      options: {
+        wait: false,
+        ready: /Selenium Server is up and running/,
+        quiet: true,
+        failOnError: false
+      },
+      cmd: 'java',
+      args: [
+        '-jar',
+        'selenium/selenium-server-standalone-2.47.1.jar'
+      ]
+    },
+
+    devSeleniumServer: {
+      options: {
+        wait: false,
+        ready: /Selenium Server is up and running/,
+        quiet: false,
+        failOnError: false
+      },
+      cmd: 'java',
+      args: [
+        '-jar',
+        'selenium/selenium-server-standalone-2.47.1.jar'
+      ]
+    },
+
     optimizeBuild: {
       options: {
         wait: false,

tasks/downloadSelenium.js

@@ -0,0 +1,60 @@
+var _ = require('lodash');
+var request = require('request');
+var fs = require('fs');
+var path = require('path');
+var colors = require('ansicolors');
+var crypto = require('crypto');
+var {spawn} = require('child_process');
+
+module.exports = function (grunt) {
+  grunt.registerTask('downloadSelenium', 'Download selenium standalone', function (keepalive) {
+    const done = this.async();
+    const config = this.options();
+
+    const SELENIUM_FILE_PATH = path.join(config.selenium.directory, config.selenium.filename);
+    const SELENIUM_DOWNLOAD_URL = config.selenium.server + config.selenium.filename;
+
+    function validateDownload(path, expectedHash, success) {
+      grunt.log.write('Validating hash...');
+      fs.readFile(path, function checkHash(err, data) {
+        if (err) grunt.fail.warn(err);
+
+        const calculatedHash = crypto.createHash('md5').update(data).digest('hex');
+        if (calculatedHash !== expectedHash) return grunt.fail.warn('Selenium download has an invalid hash');
+
+        grunt.log.writeln('done');
+        success();
+      });
+    }
+
+    function downloadSelenium(success) {
+      grunt.log.write(`Downloading ${SELENIUM_DOWNLOAD_URL}...`);
+      request.get(SELENIUM_DOWNLOAD_URL)
+      .pipe(fs.createWriteStream(SELENIUM_FILE_PATH))
+      .on('error', function downloadError(err) {
+        grunt.fail.warn(err);
+      })
+      .on('finish', function downloadFinish() {
+        grunt.log.writeln('done');
+        validateDownload(SELENIUM_FILE_PATH, config.selenium.md5, success);
+      });
+    }
+
+    function start() {
+      try {
+        fs.mkdirSync(config.selenium.directory);
+      } catch (err) {
+        if (err && err.code !== 'EEXIST') grunt.fail.warn(err);
+      }
+
+      if (fs.existsSync(SELENIUM_FILE_PATH)) {
+        validateDownload(SELENIUM_FILE_PATH, config.selenium.md5, done);
+      } else {
+        downloadSelenium(done);
+      }
+    }
+
+    start();
+
+  });
+};

tasks/loadFixtures.js

@@ -0,0 +1,31 @@
+var _ = require('lodash');
+var wreck = require('wreck');
+var fs = require('fs');
+var path = require('path');
+var colors = require('ansicolors');
+
+module.exports = function (grunt) {
+  grunt.registerTask('loadFixtures', 'Loads fixtures into elasticsearch', function () {
+    const config = this.options();
+    const done = this.async();
+    const files = fs.readdirSync(config.dataDir);
+    let doneProcessing = 0;
+
+    files.forEach(function (file) {
+      wreck.post(`${config.server}/_bulk`, {
+        payload: fs.createReadStream(path.join(config.dataDir, file)),
+        json: true
+      }, function bulkResponse(err, res, payload) {
+        var status;
+        if (err || res.statusCode !== 200) {
+          grunt.fail.warn(err || payload);
+          status = colors.red('error');
+        } else {
+          status = colors.green('success');
+        }
+        grunt.log.writeln(`[${status}] ${file}`);
+        if (++doneProcessing === files.length) done();
+      });
+    });
+  });
+};

tasks/test.js

@@ -6,6 +6,7 @@ module.exports = function (grunt) {
 
   grunt.registerTask('test:quick', [
     'test:server',
+    'test:ui',
     'test:browser'
   ]);
 
@@ -14,6 +15,30 @@ module.exports = function (grunt) {
     'karma:dev'
   ]);
 
+  grunt.registerTask('test:ui', [
+    'esvm:ui',
+    'loadFixtures',
+    'run:testUIServer',
+    'downloadSelenium',
+    'run:seleniumServer',
+    'intern:dev',
+    'esvm_shutdown:ui',
+    'stop:seleniumServer',
+    'stop:testUIServer'
+  ]);
+
+  grunt.registerTask('test:ui:server', [
+    'esvm:ui',
+    'loadFixtures',
+    'run:testUIServer',
+    'downloadSelenium',
+    'run:devSeleniumServer:keepalive'
+  ]);
+
+  grunt.registerTask('test:ui:runner', [
+    'intern:dev'
+  ]);
+
   grunt.registerTask('test', function (subTask) {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 

test/fixtures/test.json

@@ -0,0 +1,6 @@
+{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
+{ "field1" : "value1" }
+{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "2" } }
+{ "field1" : "value2" }
+{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "3" } }
+{ "field1" : "value3" }

test/functional/status.js

@@ -0,0 +1,21 @@
+define(function (require) {
+  var registerSuite = require('intern!object');
+  var expect = require('intern/dojo/node!expect.js');
+
+  registerSuite(function () {
+    var url = 'http://localhost:5620/status';
+    return {
+      'status': function () {
+        return this.remote
+          .get(url)
+          .refresh()
+          .setFindTimeout(60000)
+          .findByCssSelector('.plugin_status_breakdown')
+          .getVisibleText()
+          .then(function (text) {
+            expect(text.indexOf('plugin:kibana Ready')).to.be.above(-1);
+          });
+      }
+    };
+  });
+});

test/intern.js

@@ -0,0 +1,17 @@
+define({
+  capabilities: {
+    'selenium-version': '2.47.1',
+    'idle-timeout': 30
+  },
+  environments: [
+    {
+      browserName: 'firefox'
+    }
+  ],
+  webdriver: {
+    host: 'localhost',
+    port: 4444
+  },
+  functionalSuites: ['test/functional/status.js'],
+  excludeInstrumentation: /^(?:tests|node_modules)\//
+});