Run tests against an actual Kibana build (#11237)

* build before running selenium

* change directory into built snapshot

* WIP: what is in the build dir

* WIP: skip unit tests

* run selenium on build

* [functional tests on build] Derive version

* [functional tests on build] Missing folder name prefix

* WIP: debug on ubuntu VM

* WIP: double kibana stabilize timeout

* WIP: remove ubuntu VM debug change

* WIP: add logs for docParams

* WIP: add logs for es

* build release version without -SNAPSHOT-

* [functional test release] Cleanup

tasks/config/run.js

@@ -6,6 +6,8 @@ module.exports = function (grunt) {
   const root = p => resolve(__dirname, '../../', p);
   const binScript =  /^win/.test(platform) ? '.\\bin\\kibana.bat' : './bin/kibana';
   const buildScript =  /^win/.test(platform) ? '.\\build\\kibana\\bin\\kibana.bat' : './build/kibana/bin/kibana';
+  const pkgVersion = grunt.config.get('pkg.version');
+  const releaseBinScript = `./build/kibana-${pkgVersion}-linux-x86_64/bin/kibana`;
   const uiConfig = require(root('test/server_config'));
 
   const stdDevArgs = [
@@ -77,6 +79,22 @@ module.exports = function (grunt) {
       ]
     },
 
+    testUIReleaseServer: {
+      options: {
+        wait: false,
+        ready: /Server running/,
+        quiet: false,
+        failOnError: false
+      },
+      cmd: releaseBinScript,
+      args: [
+        ...stdDevArgs,
+        '--server.port=' + uiConfig.servers.kibana.port,
+        '--elasticsearch.url=' + format(uiConfig.servers.elasticsearch),
+        ...kbnServerFlags,
+      ]
+    },
+
     testUIDevServer: {
       options: {
         wait: false,

tasks/jenkins.js

@@ -38,7 +38,7 @@ module.exports = function (grunt) {
     'jenkins:env',
     'rejectRejFiles',
 
-    'test:ui'
+    'test:uiRelease'
   ]);
 
 };

tasks/test.js

@@ -64,6 +64,16 @@ module.exports = function (grunt) {
     'stop:testUIServer'
   ]);
 
+  grunt.registerTask('test:uiRelease', [
+    'checkPlugins',
+    'esvm:ui',
+    'run:testUIReleaseServer',
+    'clean:screenshots',
+    'functionalTestRunner',
+    'esvm_shutdown:ui',
+    'stop:testUIReleaseServer'
+  ]);
+
   grunt.registerTask('test:ui:server', [
     'checkPlugins',
     'esvm:ui',

test/scripts/jenkins_selenium.sh

@@ -3,4 +3,6 @@
 set -e
 source "$(dirname $0)/_jenkins_setup.sh"
 
+"$(npm bin)/grunt" build --release;
+
 xvfb-run "$(npm bin)/grunt" jenkins:selenium;