[6.x] [optimizer] run webpack compilation ASAP (no more laziness) (#15795) (#15917)

* [optimizer] run webpack compilation ASAP (no more laziness)

* [optimize] fix variable reference

src/cli/serve/serve.js

@@ -39,7 +39,7 @@ function readServerSettings(opts, extraCliOptions) {
 
   if (opts.dev) {
     set('env', 'development');
-    set('optimize.lazy', true);
+    set('optimize.watch', true);
 
     if (opts.ssl) {
       set('server.ssl.enabled', true);

src/optimize/index.js

@@ -4,7 +4,7 @@ import { createBundlesRoute } from './bundles_route';
 export default async (kbnServer, server, config) => {
   if (!config.get('optimize.enabled')) return;
 
-  // the lazy optimizer sets up two threads, one is the server listening
+  // the watch optimizer sets up two threads, one is the server listening
   // on 5601 and the other is a server listening on 5602 that builds the
   // bundles in a "middleware" style.
   //
@@ -12,9 +12,9 @@ export default async (kbnServer, server, config) => {
   // on the watch setup managed by the cli. It proxies all bundles/* requests to
   // the other server. The server on 5602 is long running, in order to prevent
   // complete rebuilds of the optimize content.
-  const lazy = config.get('optimize.lazy');
-  if (lazy) {
-    return await kbnServer.mixin(require('./lazy/lazy'));
+  const watch = config.get('optimize.watch');
+  if (watch) {
+    return await kbnServer.mixin(require('./watch/watch'));
   }
 
   const { uiBundles } = kbnServer;

src/optimize/lazy/lazy_optimizer.js

@@ -1,119 +0,0 @@
-import BaseOptimizer from '../base_optimizer';
-import WeirdControlFlow from './weird_control_flow';
-import { once } from 'lodash';
-import { join } from 'path';
-
-import { createBundlesRoute } from '../bundles_route';
-
-export default class LazyOptimizer extends BaseOptimizer {
-  constructor(opts) {
-    super(opts);
-    this.log = opts.log || (() => null);
-    this.prebuild = opts.prebuild || false;
-
-    this.timer = {
-      ms: null,
-      start: () => this.timer.ms = Date.now(),
-      end: () => this.timer.ms = ((Date.now() - this.timer.ms) / 1000).toFixed(2)
-    };
-
-    this.build = new WeirdControlFlow();
-  }
-
-  async init() {
-    this.initializing = true;
-
-    await this.uiBundles.writeEntryFiles();
-    await this.initCompiler();
-
-    this.compiler.plugin('watch-run', (w, webpackCb) => {
-      this.build.work(once(() => {
-        this.timer.start();
-        this.logRunStart();
-        webpackCb();
-      }));
-    });
-
-    this.compiler.plugin('done', stats => {
-      if (!stats.hasErrors() && !stats.hasWarnings()) {
-        this.logRunSuccess();
-        this.build.success();
-        return;
-      }
-
-      const err = this.failedStatsToError(stats);
-      this.logRunFailure(err);
-      this.build.failure(err);
-      this.watching.invalidate();
-    });
-
-    this.watching = this.compiler.watch({ aggregateTimeout: 200 }, err => {
-      if (err) {
-        this.log('fatal', err);
-        process.exit(1);
-      }
-    });
-
-    const buildPromise = this.build.get();
-    if (this.prebuild) await buildPromise;
-
-    this.initializing = false;
-    this.log(['info', 'optimize'], {
-      tmpl: `Lazy optimization of ${this.uiBundles.getDescription()} ready`,
-      bundles: this.uiBundles.getIds()
-    });
-  }
-
-  async getPath(relativePath) {
-    await this.build.get();
-    return join(this.compiler.outputPath, relativePath);
-  }
-
-  bindToServer(server, basePath) {
-
-    // calling `build.get()` resolves when the build is
-    // "stable" (the compiler is not running) so this pauses
-    // all requests received while the compiler is running
-    // and lets the continue once it is done.
-    server.ext('onRequest', (request, reply) => {
-      this.build.get()
-        .then(() => reply.continue())
-        .catch(reply);
-    });
-
-    server.route(createBundlesRoute({
-      bundlesPath: this.compiler.outputPath,
-      basePublicPath: basePath
-    }));
-  }
-
-  logRunStart() {
-    this.log(['info', 'optimize'], {
-      tmpl: `Lazy optimization started`,
-      bundles: this.uiBundles.getIds()
-    });
-  }
-
-  logRunSuccess() {
-    this.log(['info', 'optimize'], {
-      tmpl: 'Lazy optimization <%= status %> in <%= seconds %> seconds',
-      bundles: this.uiBundles.getIds(),
-      status: 'success',
-      seconds: this.timer.end()
-    });
-  }
-
-  logRunFailure(err) {
-    // errors during initialization to the server, unlike the rest of the
-    // errors produced here. Lets not muddy the console with extra errors
-    if (this.initializing) return;
-
-    this.log(['fatal', 'optimize'], {
-      tmpl: 'Lazy optimization <%= status %> in <%= seconds %> seconds<%= err %>',
-      bundles: this.uiBundles.getIds(),
-      status: 'failed',
-      seconds: this.timer.end(),
-      err: err
-    });
-  }
-}

src/optimize/lazy/weird_control_flow.js

@@ -1,58 +0,0 @@
-import { fromNode } from 'bluebird';
-
-
-export default class WeirdControlFlow {
-  constructor() {
-    this.handlers = [];
-  }
-
-  get() {
-    return fromNode(cb => {
-      if (this.ready) return cb();
-      this.handlers.push(cb);
-      this.start();
-    });
-  }
-
-  work(work) {
-    this._work = work;
-    this.stop();
-
-    if (this.handlers.length) {
-      this.start();
-    }
-  }
-
-  start() {
-    if (this.running) return;
-    this.stop();
-    if (this._work) {
-      this.running = true;
-      this._work();
-    }
-  }
-
-  stop() {
-    this.ready = false;
-    this.error = false;
-    this.running = false;
-  }
-
-  success(...args) {
-    this.stop();
-    this.ready = true;
-    this._flush(args);
-  }
-
-  failure(err) {
-    this.stop();
-    this.error = err;
-    this._flush([err]);
-  }
-
-  _flush(args) {
-    for (const fn of this.handlers.splice(0)) {
-      fn.apply(null, args);
-    }
-  }
-}

src/optimize/watch/optmzr_role.js

@@ -1,17 +1,17 @@
-import LazyServer from './lazy_server';
-import LazyOptimizer from './lazy_optimizer';
+import WatchServer from './watch_server';
+import WatchOptimizer from './watch_optimizer';
 
 export default async (kbnServer, kibanaHapiServer, config) => {
-  const server = new LazyServer(
-    config.get('optimize.lazyHost'),
-    config.get('optimize.lazyPort'),
+  const server = new WatchServer(
+    config.get('optimize.watchHost'),
+    config.get('optimize.watchPort'),
     config.get('server.basePath'),
-    new LazyOptimizer({
+    new WatchOptimizer({
       log: (tags, data) => kibanaHapiServer.log(tags, data),
       uiBundles: kbnServer.uiBundles,
       profile: config.get('optimize.profile'),
       sourceMaps: config.get('optimize.sourceMaps'),
-      prebuild: config.get('optimize.lazyPrebuild'),
+      prebuild: config.get('optimize.watchPrebuild'),
       unsafeCache: config.get('optimize.unsafeCache'),
     })
   );

src/optimize/watch/proxy_role.js

@@ -8,8 +8,8 @@ export default (kbnServer, server, config) => {
     method: 'GET',
     handler: {
       proxy: {
-        host: config.get('optimize.lazyHost'),
-        port: config.get('optimize.lazyPort'),
+        host: config.get('optimize.watchHost'),
+        port: config.get('optimize.watchPort'),
         passThrough: true,
         xforward: true
       }
@@ -19,11 +19,11 @@ export default (kbnServer, server, config) => {
 
   return fromNode(cb => {
     const timeout = setTimeout(() => {
-      cb(new Error('Server timedout waiting for the optimizer to become ready'));
-    }, config.get('optimize.lazyProxyTimeout'));
+      cb(new Error('Timeout waiting for the optimizer to become ready'));
+    }, config.get('optimize.watchProxyTimeout'));
 
     const waiting = once(() => {
-      server.log(['info', 'optimize'], 'Waiting for optimizer completion');
+      server.log(['info', 'optimize'], 'Waiting for optimizer to be ready');
     });
 
     if (!process.connected) return;

src/optimize/watch/watch.js

@@ -2,21 +2,19 @@ import { isWorker } from 'cluster';
 
 export default async kbnServer => {
 
-
   if (!isWorker) {
-    throw new Error(`lazy optimization is only available in "watch" mode`);
+    throw new Error(`watch optimization is only available when using the "--dev" cli flag`);
   }
 
   /**
-   * When running in lazy mode two workers/threads run in one
-   * of the modes: 'optmzr' or 'server'
+   * When running in watch mode two processes run in one of the following modes:
    *
-   * optmzr: this thread runs the LiveOptimizer and the LazyServer
-   *   which serves the LiveOptimizer's output and blocks requests
+   * optmzr: this process runs the WatchOptimizer and the WatchServer
+   *   which serves the WatchOptimizer's output and blocks requests
    *   while the optimizer is running
    *
-   * server: this thread runs the entire kibana server and proxies
-   *   all requests for /bundles/* to the optmzr
+   * server: this process runs the entire kibana server and proxies
+   *   all requests for /bundles/* to the optmzr process
    *
    * @param  {string} process.env.kbnWorkerType
    */

src/optimize/watch/watch_optimizer.js

@@ -0,0 +1,155 @@
+import { Observable, ReplaySubject } from 'rxjs';
+
+import BaseOptimizer from '../base_optimizer';
+
+import { createBundlesRoute } from '../bundles_route';
+
+const STATUS = {
+  RUNNING: 'optimizer running',
+  SUCCESS: 'optimizer completed successfully',
+  FAILURE: 'optimizer failed with stats',
+  FATAL: 'optimizer failed without stats',
+};
+
+export default class WatchOptimizer extends BaseOptimizer {
+  constructor(opts) {
+    super(opts);
+    this.log = opts.log || (() => null);
+    this.prebuild = opts.prebuild || false;
+    this.status$ = new ReplaySubject(1);
+  }
+
+  async init() {
+    this.initializing = true;
+    this.initialBuildComplete = false;
+
+    // log status changes
+    this.status$.subscribe(this.onStatusChangeHandler);
+
+    await this.uiBundles.writeEntryFiles();
+    await this.initCompiler();
+
+    this.compiler.plugin('watch-run', this.compilerRunStartHandler);
+    this.compiler.plugin('done', this.compilerDoneHandler);
+    this.compiler.watch({ aggregateTimeout: 200 }, this.compilerWatchErrorHandler);
+
+    if (this.prebuild) {
+      await this.onceBuildOutcome();
+    }
+
+    this.initializing = false;
+  }
+
+  bindToServer(server, basePath) {
+    // pause all requests received while the compiler is running
+    // and continue once an outcome is reached (aborting the request
+    // with an error if it was a failure).
+    server.ext('onRequest', (request, reply) => {
+      this.onceBuildOutcome()
+        .then(() => reply.continue())
+        .catch(reply);
+    });
+
+    server.route(createBundlesRoute({
+      bundlesPath: this.compiler.outputPath,
+      basePublicPath: basePath
+    }));
+  }
+
+  async onceBuildOutcome() {
+    return await this.status$
+      .mergeMap(this.mapStatusToOutcomes)
+      .take(1)
+      .toPromise();
+  }
+
+  mapStatusToOutcomes({ type, error }) {
+    switch (type) {
+      case STATUS.RUNNING:
+        return [];
+
+      case STATUS.SUCCESS:
+        return [true];
+
+      case STATUS.FAILURE:
+      case STATUS.FATAL:
+        return Observable.throw(error);
+    }
+  }
+
+  compilerRunStartHandler = (watchingCompiler, cb) => {
+    this.status$.next({
+      type: STATUS.RUNNING
+    });
+
+    cb();
+  }
+
+  compilerWatchErrorHandler = (error) => {
+    if (error) {
+      this.status$.next({
+        type: STATUS.FATAL,
+        error
+      });
+    }
+  }
+
+  compilerDoneHandler = (stats) => {
+    this.initialBuildComplete = true;
+    const seconds = parseFloat((stats.endTime - stats.startTime) / 1000).toFixed(2);
+
+    if (stats.hasErrors() || stats.hasWarnings()) {
+      this.status$.next({
+        type: STATUS.FAILURE,
+        seconds,
+        error: this.failedStatsToError(stats)
+      });
+    } else {
+      this.status$.next({
+        type: STATUS.SUCCESS,
+        seconds,
+      });
+    }
+  }
+
+  onStatusChangeHandler = ({ type, seconds, error }) => {
+    switch (type) {
+      case STATUS.RUNNING:
+        if (!this.initialBuildComplete) {
+          this.log(['info', 'optimize'], {
+            tmpl: 'Optimization started',
+            bundles: this.uiBundles.getIds()
+          });
+        }
+        break;
+
+      case STATUS.SUCCESS:
+        this.log(['info', 'optimize'], {
+          tmpl: 'Optimization <%= status %> in <%= seconds %> seconds',
+          bundles: this.uiBundles.getIds(),
+          status: 'success',
+          seconds
+        });
+        break;
+
+      case STATUS.FAILURE:
+        // errors during initialization to the server, unlike the rest of the
+        // errors produced here. Lets not muddy the console with extra errors
+        if (!this.initializing) {
+          this.log(['fatal', 'optimize'], {
+            tmpl: 'Optimization <%= status %> in <%= seconds %> seconds<%= err %>',
+            bundles: this.uiBundles.getIds(),
+            status: 'failed',
+            seconds,
+            err: error
+          });
+        }
+        break;
+
+      case STATUS.FATAL:
+        this.log('fatal', error);
+        process.exit(1);
+        break;
+    }
+  }
+}

src/optimize/watch/watch_server.js

@@ -3,7 +3,7 @@ import { Server } from 'hapi';
 import { fromNode } from 'bluebird';
 import registerHapiPlugins from '../../server/http/register_hapi_plugins';
 
-export default class LazyServer {
+export default class WatchServer {
   constructor(host, port, basePath, optimizer) {
     this.basePath = basePath;
     this.optimizer = optimizer;

src/server/config/schema.js

@@ -133,11 +133,11 @@ export default () => Joi.object({
     bundleFilter: Joi.string().default('!tests'),
     bundleDir: Joi.string().default(fromRoot('optimize/bundles')),
     viewCaching: Joi.boolean().default(Joi.ref('$prod')),
-    lazy: Joi.boolean().default(false),
-    lazyPort: Joi.number().default(5602),
-    lazyHost: Joi.string().hostname().default('localhost'),
-    lazyPrebuild: Joi.boolean().default(false),
-    lazyProxyTimeout: Joi.number().default(5 * 60000),
+    watch: Joi.boolean().default(false),
+    watchPort: Joi.number().default(5602),
+    watchHost: Joi.string().hostname().default('localhost'),
+    watchPrebuild: Joi.boolean().default(false),
+    watchProxyTimeout: Joi.number().default(5 * 60000),
     useBundleCache: Joi.boolean().default(Joi.ref('$prod')),
     unsafeCache: Joi.when('$prod', {
       is: true,

src/server/config/transform_deprecations.js

@@ -30,6 +30,11 @@ const deprecations = [
   rename('server.ssl.cert', 'server.ssl.certificate'),
   unused('server.xsrf.token'),
   unused('uiSettings.enabled'),
+  rename('optimize.lazy', 'optimize.watch'),
+  rename('optimize.lazyPort', 'optimize.watchPort'),
+  rename('optimize.lazyHost', 'optimize.watchHost'),
+  rename('optimize.lazyPrebuild', 'optimize.watchPrebuild'),
+  rename('optimize.lazyProxyTimeout', 'optimize.watchProxyTimeout'),
   serverSslEnabled,
   savedObjectsIndexCheckTimeout,
 ];

src/server/kbn_server.js

@@ -62,7 +62,7 @@ export default class KbnServer {
       savedObjectsMixin,
 
       // ensure that all bundles are built, or that the
-      // lazy bundle server is running
+      // watch bundle server is running
       optimizeMixin,
 
       // initialize the plugins

tasks/config/run.js

@@ -141,8 +141,8 @@ module.exports = function (grunt) {
         '--elasticsearch.url=' + esTestConfig.getUrl(),
         '--dev',
         '--no-base-path',
-        '--optimize.lazyPort=5611',
-        '--optimize.lazyPrebuild=true',
+        '--optimize.watchPort=5611',
+        '--optimize.watchPrebuild=true',
         '--optimize.bundleDir=' + resolve(__dirname, '../../optimize/testUiServer'),
         ...kbnServerFlags,
       ]
@@ -178,8 +178,8 @@ module.exports = function (grunt) {
         '--no-watch',
         '--no-base-path',
         '--server.port=5610',
-        '--optimize.lazyPort=5611',
-        '--optimize.lazyPrebuild=true',
+        '--optimize.watchPort=5611',
+        '--optimize.watchPrebuild=true',
         '--optimize.bundleDir=' + resolve(__dirname, '../../optimize/testdev'),
         ...kbnServerFlags,
       ]