Run some functional tests against kibana in production mode (#21899)

Right now the functional tests are run against a distributable of Kibana in CI, but that distributable is running with `--env.name=development`. That causes the optimizer to run again before the tests can start and prevents the functional tests from running against the actual version of the application users will end up getting. This seems necessary for some tests, but not all of them, but I would like to get all of the tests running against the production version of Kibana soon.

This PR implements a second ftr config, `test/functional_production` that uses basically a copy of the `test/functional` config but with a few minor adjustments, removing the `--env.name=development` kbnServerArg and using a unique junit report name. To accomplish this I needed to modify the `@kbn/test` module to only pass the `--dev` flag to the Kibana server if it is being run in development mode, which it currently does by testing the args for `--env.name=development` or two args next to each other: `'--env.name'` and `'development'`. It does this by converting the `extraKbnOpts` option into an `addExtraKbnArgs` function, which is called with the final args just before passing them to the proc runner and given a chance to modify then after all other args are resolved (pulling from different places in config based on the build type, etc.)

Over the next couple weeks I'll push up PRs for individual test suites, migrating them over to the new production config, hopefully in a short period of time we will have all the function test suites back under that `test/function` config and can run them all against the Kibana server in production mode.

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -24,34 +24,34 @@ Options:
 
 exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "updateBaselines": true,
 }
 `;
 
 exports[`process options for run tests CLI accepts debug option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
   "debug": true,
-  "extraKbnOpts": undefined,
 }
 `;
 
 exports[`process options for run tests CLI accepts empty config value if default passed 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": "",
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
 }
 `;
 
@@ -60,78 +60,76 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": Object {
-    "server.foo": "bar",
-  },
 }
 `;
 
 exports[`process options for run tests CLI accepts quiet option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "quiet": true,
 }
 `;
 
 exports[`process options for run tests CLI accepts silent option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "silent": true,
 }
 `;
 
 exports[`process options for run tests CLI accepts source value for esFrom 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
   "esFrom": "source",
-  "extraKbnOpts": undefined,
 }
 `;
 
 exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "installDir": "foo",
 }
 `;
 
 exports[`process options for run tests CLI accepts value for grep 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "grep": "management",
 }
 `;
 
 exports[`process options for run tests CLI accepts verbose option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "configs": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "verbose": true,
 }
 `;

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -108,7 +108,7 @@ export function processOptions(userOptions, defaultConfigPaths) {
     ...userOptions,
     configs,
     createLogger,
-    extraKbnOpts: userOptions._,
+    addExtraKbnArgs: argv => argv.concat(userOptions._ || []),
   };
 }
 

packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap

@@ -21,22 +21,22 @@ Options:
 
 exports[`process options for start servers CLI accepts debug option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
   "debug": true,
-  "extraKbnOpts": undefined,
 }
 `;
 
 exports[`process options for start servers CLI accepts empty config value if default passed 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
 }
 `;
 
@@ -45,67 +45,65 @@ Object {
   "_": Object {
     "server.foo": "bar",
   },
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": Object {
-    "server.foo": "bar",
-  },
 }
 `;
 
 exports[`process options for start servers CLI accepts quiet option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "quiet": true,
 }
 `;
 
 exports[`process options for start servers CLI accepts silent option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "silent": true,
 }
 `;
 
 exports[`process options for start servers CLI accepts source value for esFrom 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
   "esFrom": "source",
-  "extraKbnOpts": undefined,
 }
 `;
 
 exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "installDir": "foo",
 }
 `;
 
 exports[`process options for start servers CLI accepts verbose option 1`] = `
 Object {
+  "addExtraKbnArgs": [Function],
   "config": Array [
     "foo",
   ],
   "createLogger": [Function],
-  "extraKbnOpts": undefined,
   "verbose": true,
 }
 `;

packages/kbn-test/src/functional_tests/cli/start_servers/args.js

@@ -95,7 +95,7 @@ export function processOptions(userOptions, defaultConfigPath) {
     ...userOptions,
     config,
     createLogger,
-    extraKbnOpts: userOptions._,
+    addExtraKbnArgs: argv => argv.concat(userOptions._ || []),
   };
 }
 

packages/kbn-test/src/functional_tests/lib/run_kibana_server.js

@@ -50,7 +50,7 @@ function getKibanaCmd(installDir) {
  * passed, we run from source code. We also allow passing in extra
  * Kibana server options, so we tack those on here.
  */
-function collectCliArgs(config, { installDir, extraKbnOpts }) {
+function collectCliArgs(config, { installDir, addExtraKbnArgs }) {
   const buildArgs = config.get('kbnTestServer.buildArgs') || [];
   const sourceArgs = config.get('kbnTestServer.sourceArgs') || [];
   const serverArgs = config.get('kbnTestServer.serverArgs') || [];
@@ -61,7 +61,7 @@ function collectCliArgs(config, { installDir, extraKbnOpts }) {
     args => {
       return installDir ? [...args, ...buildArgs] : [KIBANA_EXEC_PATH, ...args, ...sourceArgs];
     },
-    args => args.concat(extraKbnOpts || [])
+    addExtraKbnArgs || (args => args)
   );
 }
 

packages/kbn-test/src/functional_tests/tasks.js

@@ -35,6 +35,25 @@ in another terminal session by running this command from this directory:
 
 `;
 
+/**
+ * test if a list of argv flags tell the kibana server to run in the development environment
+ * @param {string[]} argv
+ * @return {boolean}
+ */
+function includesEnvNameDevelopmentFlag(argv) {
+  return argv.some((arg, i) => {
+    // find single-arg format
+    if (arg === '--env.name=development') {
+      return true;
+    }
+
+    // find split arg format
+    if (arg === '--env.name' && argv[i + 1] === 'development') {
+      return true;
+    }
+  });
+}
+
 /**
  * Run servers and tests for each config
  * @param {object} options                   Optional
@@ -76,7 +95,12 @@ export async function startServers(options) {
       config,
       options: {
         ...opts,
-        extraKbnOpts: [...options.extraKbnOpts, ...(options.installDir ? [] : ['--dev'])],
+        addExtraKbnArgs: argv => {
+          argv = options.addExtraKbnArgs(argv);
+          return !options.installDir && includesEnvNameDevelopmentFlag(argv)
+            ? argv.concat('--dev')
+            : argv;
+        },
       },
     });
 

scripts/functional_tests.js

@@ -19,6 +19,7 @@
 
 require('../src/setup_node_env');
 require('@kbn/test').runTestsCli([
+  require.resolve('../test/functional_production/config.js'),
   require.resolve('../test/functional/config.js'),
   require.resolve('../test/api_integration/config.js'),
   require.resolve('../test/panel_actions/config.js'),

src/functional_test_runner/lib/config/config.js

@@ -24,6 +24,14 @@ import { schema } from './schema';
 
 const $values = Symbol('values');
 
+function cloneConfigValue(value) {
+  return cloneDeep(value, (v) => (
+    typeof v === 'function'
+      ? v
+      : undefined // tells lodash to use default behavior
+  ));
+}
+
 export class Config {
   constructor(options = {}) {
     const {
@@ -86,15 +94,17 @@ export class Config {
     return recursiveHasCheck(path, this[$values], schema);
   }
 
+  getAll() {
+    return cloneConfigValue(this[$values]);
+  }
+
   get(key, defaultValue) {
     if (!this.has(key)) {
       throw new Error(`Unknown config key "${key}"`);
     }
 
-    return cloneDeep(get(this[$values], key, defaultValue), (v) => {
-      if (typeof v === 'function') {
-        return v;
-      }
-    });
+    return cloneConfigValue(
+      get(this[$values], key, defaultValue)
+    );
   }
 }

tasks/config/run.js

@@ -186,6 +186,18 @@ module.exports = function (grunt) {
       ],
     },
 
+    ossReleaseProductionSmokeTests: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/functional_production/config.js',
+        '--esFrom', 'source',
+        '--bail',
+        '--debug',
+        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
+      ],
+    },
+
     functionalTestsRelease: {
       cmd: process.execPath,
       args: [

tasks/jenkins.js

@@ -41,6 +41,7 @@ module.exports = function (grunt) {
 
   grunt.registerTask('jenkins:selenium', [
     'checkPlugins',
+    'run:ossReleaseProductionSmokeTests',
     'run:functionalTestsRelease',
   ]);
 };

test/common/config.js

@@ -49,7 +49,7 @@ export default function () {
         `--optimize.bundleDir=${OPTIMIZE_BUNDLE_DIR}`,
       ],
       serverArgs: [
-        '--env.name=development',
+        '--env.name=development', // this arg, in this format, is required by ../functional_production/config.js
         '--logging.json=false',
         `--server.port=${kbnTestConfig.getPort()}`,
         `--optimize.watchPort=${kbnTestConfig.getPort() + 10}`,

test/functional/config.js

@@ -64,12 +64,14 @@ export default async function ({ readConfigFile }) {
       require.resolve('./apps/context'),
       require.resolve('./apps/dashboard'),
       require.resolve('./apps/discover'),
-      require.resolve('./apps/home'),
       require.resolve('./apps/management'),
       require.resolve('./apps/status_page'),
       require.resolve('./apps/timelion'),
       require.resolve('./apps/visualize'),
       require.resolve('./apps/xpack'),
+
+      // migrated to functional_production config
+      // require.resolve('./apps/home')
     ],
     pageObjects: {
       common: CommonPageProvider,

test/functional_production/config.js

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default async function ({ readConfigFile }) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [
+      require.resolve('../functional/apps/home'),
+    ],
+
+    junit: {
+      reportName: `${functionalConfig.get('junit.reportName')} (Production)`
+    },
+
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+
+      serverArgs: functionalConfig.get('kbnTestServer.serverArgs').filter(arg => (
+        arg !== '--env.name=development'
+      )),
+    },
+  };
+}